From abc448009eddd8563835f233e8e95095a7b32a15 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 10 Jan 2024 15:56:00 -0300 Subject: [PATCH 001/263] Linux: Add netfilter hooks enumeration plugin. --- .../framework/plugins/linux/netfilter.py | 715 ++++++++++++++++++ 1 file changed, 715 insertions(+) create mode 100644 volatility3/framework/plugins/linux/netfilter.py diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py new file mode 100644 index 000000000..cc4786b4e --- /dev/null +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -0,0 +1,715 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +from dataclasses import dataclass, field +from abc import ABC, abstractmethod +import logging + +from typing import Iterator, List, Tuple +from volatility3.framework import ( + class_subclasses, + constants, + interfaces, + renderers, +) +from volatility3.framework.renderers import format_hints +from volatility3.framework.configuration import requirements +from volatility3.framework.symbols import linux +from volatility3.plugins.linux import lsmod + +vollog = logging.getLogger(__name__) + + +@dataclass +class Proto: + name: str + hooks: Tuple[str] = field(default_factory=tuple) + + +PROTO_NOT_IMPLEMENTED = Proto(name="UNSPEC") + +NF_INET_HOOKS = ("PRE_ROUTING", "LOCAL_IN", "FORWARD", "LOCAL_OUT", "POST_ROUTING") +NF_DEC_HOOKS = ( + "PRE_ROUTING", + "LOCAL_IN", + "FORWARD", + "LOCAL_OUT", + "POST_ROUTING", + "HELLO", + "ROUTE", +) +NF_ARP_HOOKS = ("IN", "OUT", "FORWARD") +NF_NETDEV_HOOKS = ("INGRESS", "EGRESS") +LARGEST_HOOK_NUMBER = max( + len(NF_INET_HOOKS), len(NF_DEC_HOOKS), len(NF_ARP_HOOKS), len(NF_NETDEV_HOOKS) +) + + +class AbstractNetfilter(ABC): + """Netfilter Abstract Base Classes handling details across various + Netfilter implementations, including constants, helpers, and common + routines. + """ + + PROTO_HOOKS = ( + PROTO_NOT_IMPLEMENTED, # NFPROTO_UNSPEC + Proto(name="INET", hooks=NF_INET_HOOKS), # From kernels 3.14 + Proto(name="IPV4", hooks=NF_INET_HOOKS), + Proto(name="ARP", hooks=("IN", "OUT", "FORWARD")), + PROTO_NOT_IMPLEMENTED, + Proto(name="NETDEV", hooks=("INGRESS", "EGRESS")), + PROTO_NOT_IMPLEMENTED, + Proto(name="BRIDGE", hooks=NF_INET_HOOKS), + PROTO_NOT_IMPLEMENTED, + PROTO_NOT_IMPLEMENTED, + Proto(name="IPV6", hooks=NF_INET_HOOKS), + PROTO_NOT_IMPLEMENTED, + Proto(name="DECNET", hooks=NF_INET_HOOKS), # Removed in kernel 6.1 + ) + NF_MAX_HOOKS = LARGEST_HOOK_NUMBER + 1 + + def __init__( + self, + context: interfaces.context.ContextInterface, + config: interfaces.configuration.HierarchicalDict, + ): + self._context = context + self._config = config + symbol_table = self._config["kernel"] + self.vmlinux = context.modules[symbol_table] + self.layer_name = self.vmlinux.layer_name + + modules = lsmod.Lsmod.list_modules(context, symbol_table) + self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( + context, symbol_table, modules + ) + + self._set_data_sizes() + + def _set_data_sizes(self): + self.ptr_size = self.vmlinux.get_type("pointer").size + self.list_head_size = self.vmlinux.get_type("list_head").size + + @classmethod + def run_all( + cls, + context: interfaces.context.ContextInterface, + config: interfaces.configuration.HierarchicalDict, + ) -> Iterator[Tuple[int, str, str, int, int, str, bool]]: + """It calls each subclass symtab_checks() to test the required + conditions to that specific kernel implementation. + + Args: + context: The volatility3 context on which to operate + config: Core configuration + + Yields: + The kmsg records. Same as _run() + """ + vmlinux = context.modules[config["kernel"]] + + implementation_inst = None # type: ignore + for subclass in class_subclasses(cls): + if not subclass.symtab_checks(vmlinux=vmlinux): + vollog.log( + constants.LOGLEVEL_VVVV, + "Netfilter implementation '%s' doesn't match this memory dump", + subclass.__name__, + ) + continue + + vollog.log( + constants.LOGLEVEL_VVVV, + "Netfilter implementation '%s' matches!", + subclass.__name__, + ) + implementation_inst = subclass(context=context, config=config) + # More than one class could be executed for an specific kernel version + # For instance: Netfilter Ingress hooks + yield from implementation_inst._run() + + if implementation_inst is None: + vollog.error("Unsupported Netfilter kernel implementation") + + def _run(self) -> Iterator[Tuple[int, str, str, int, int, str, bool]]: + """Iterates over namespaces and protocols, executing various callbacks that + allow customization of the code to the specific data structure used in a + particular kernel implementation + + get_hooks_container_by_protocol(net, proto_name) + It returns the data structure used in a specific kernel implementation + to store the hooks for a respective namespace and protocol, basically: + For Ingress hooks: + network_namespace[] -> net_device[] -> nf_hooks_ingress[] + For all the other Netfilter hooks: + <= 4.2.8 + nf_hooks[] + >= 4.3 + network_namespace[] -> nf.hooks[] + + get_hook_ops(hook_container, proto_idx, hook_idx) + Give the 'hook_container' got in get_hooks_container_by_protocol(), it + returns an iterable of 'nf_hook_ops' elements for a respective protocol + and hook type. + + Returns: + netns [int]: Network namespace id + proto_name [str]: Protocol name + hook_name [str]: Hook name + priority [int]: Priority + hook_ops_hook [int]: Hook address + module_name [str]: Linux kernel module name + hooked [bool]: hooked? + """ + for netns, net in self.get_net_namespaces(): + for proto_idx, proto_name, hook_idx, hook_name in self._proto_hook_loop(): + hooks_container = self.get_hooks_container_by_protocol(net, proto_name) + + for hook_container in hooks_container: + for hook_ops in self.get_hook_ops( + hook_container, proto_idx, hook_idx + ): + if not hook_ops: + continue + + priority = int(hook_ops.priority) + hook_ops_hook = hook_ops.hook + module_name = self.get_module_name_for_address(hook_ops_hook) + hooked = module_name is not None + + yield netns, proto_name, hook_name, priority, hook_ops_hook, module_name, hooked + + @classmethod + @abstractmethod + def symtab_checks(cls, vmlinux: interfaces.context.ModuleInterface) -> bool: + """This method on each sublasss will be called to evaluate if the kernel + being analyzed fulfill the type & symbols requirements for the implementation. + The first class returning True will be instantiated and called via the + run() method. + + Returns: + bool: True if the kernel being analyzed fulfill the class requirements. + """ + + def _proto_hook_loop(self) -> Iterator[Tuple[int, str, int, str]]: + """Flattens the protocol families and hooks""" + for proto_idx, proto in enumerate(AbstractNetfilter.PROTO_HOOKS): + if proto == PROTO_NOT_IMPLEMENTED: + continue + if proto.name not in self.subscribed_protocols(): + # This protocol is not managed in this object + continue + for hook_idx, hook_name in enumerate(proto.hooks): + yield proto_idx, proto.name, hook_idx, hook_name + + def build_nf_hook_ops_array(self, nf_hook_entries): + """Function helper to build the nf_hook_ops array when it is not part of the + struct 'nf_hook_entries' definition. + + nf_hook_ops was stored adjacent in memory to the nf_hook_entry array, in the + new struct 'nf_hook_entries'. However, this 'nf_hooks_ops' array 'orig_ops' is + not part of the 'nf_hook_entries' struct. So, we need to calculate the offset. + + struct nf_hook_entries { + u16 num_hook_entries; /* plus padding */ + struct nf_hook_entry hooks[]; + //const struct nf_hook_ops *orig_ops[]; + } + """ + nf_hook_entry_size = self.vmlinux.get_type("nf_hook_entry").size + orig_ops_addr = ( + nf_hook_entries.hooks.vol.offset + + nf_hook_entry_size * nf_hook_entries.num_hook_entries + ) + orig_ops = self._context.object( + object_type=self.get_symbol_fullname("array"), + offset=orig_ops_addr, + subtype=self.vmlinux.get_type("pointer"), + layer_name=self.layer_name, + count=nf_hook_entries.num_hook_entries, + ) + + return orig_ops + + def subscribed_protocols(self) -> Tuple[str]: + """Allows to select which PROTO_HOOKS protocols will be processed by the + Netfiler subclass. + """ + + # Most implementation handlers respond to these protocols, except for + # the ingress hook, which specifically handles the 'NETDEV' protocol. + # However, there is no corresponding Netfilter hook implementation for + # the INET protocol in the kernel. AFAIU, this is used as + # 'NFPROTO_INET = NFPROTO_IPV4 || NFPROTO_IPV6' + # in other parts of the kernel source code. + return ("IPV4", "ARP", "BRIDGE", "IPV6", "DECNET") + + def get_module_name_for_address(self, addr) -> str: + """Helper to obtain the module and symbol name in the format needed for the + output of this plugin. + """ + module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( + self.vmlinux, self.handlers, addr + ) + + if module_name == "UNKNOWN": + module_name = None + + if symbol_name != "N/A": + module_name = f"[{symbol_name}]" + + return module_name + + def get_net_namespaces(self): + """Common function to retrieve the different namespaces. + From 4.3 on, all the implementations use network namespaces. + """ + nethead = self.vmlinux.object_from_symbol("net_namespace_list") + symbol_net_name = self.get_symbol_fullname("net") + for net in nethead.to_list(symbol_net_name, "list"): + net_ns_id = net.ns.inum + yield net_ns_id, net + + def get_hooks_container_by_protocol(self, net, proto_name): + """Returns the data structure used in a specific kernel implementation to store + the hooks for a respective namespace and protocol. + + Except for kernels < 4.3, all the implementations use network namespaces. + Also the data structure which contains the hooks, even though it changes its + implementation and/or data type, it is always in this location. + """ + yield net.nf.hooks + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + """Given the hook_container obtained from get_hooks_container_by_protocol(), it + returns an iterable of 'nf_hook_ops' elements for a corresponding protocol + and hook type. + + This is the most variable/unstable part of all Netfilter hook designs, it + changes almost in every single implementation. + """ + raise NotImplementedError("You must implement this method") + + def get_symbol_fullname(self, symbol_basename: str) -> str: + """Given a short symbol or type name, it returns its full name""" + return self.vmlinux.symbol_table_name + constants.BANG + symbol_basename + + @staticmethod + def get_member_type( + vol_type: interfaces.objects.Template, member_name: str + ) -> List[str]: + """Returns a list of types/subtypes belonging to the given type member. + + Args: + vol_type (interfaces.objects.Template): A vol3 type object + member_name (str): The member name + + Returns: + list: A list of types/subtypes + """ + _size, vol_obj = vol_type.vol.members[member_name] + type_name = vol_obj.type_name + type_basename = type_name.split(constants.BANG)[1] + member_type = [type_basename] + cur_type = vol_obj + while hasattr(cur_type, "subtype"): + subtype_name = cur_type.subtype.type_name + subtype_basename = subtype_name.split(constants.BANG)[1] + member_type.append(subtype_basename) + cur_type = cur_type.subtype + + return member_type + + +class NetfilterImp_to_4_3(AbstractNetfilter): + """At this point, Netfilter hooks were implemented as a linked list of struct + 'nf_hook_ops' type. One linked list per protocol per hook type. + It was like that until 4.2.8. + + struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return vmlinux.has_symbol("nf_hooks") + + def get_net_namespaces(self): + # In kernels <= 4.2.8 netfilter hooks are not implemented per namespaces + netns, net = renderers.NotAvailableValue(), renderers.NotAvailableValue() + yield netns, net + + def get_hooks_container_by_protocol(self, net, proto_name): + nf_hooks = self.vmlinux.object_from_symbol("nf_hooks") + if not nf_hooks: + return + + yield nf_hooks + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + list_head = hook_container[proto_idx][hook_idx] + nf_hooks_ops_name = self.get_symbol_fullname("nf_hook_ops") + return list_head.to_list(nf_hooks_ops_name, "list") + + +class NetfilterImp_4_3_to_4_9(AbstractNetfilter): + """Netfilter hooks were added to network namepaces in 4.3. + It is still implemented as a linked list of 'struct nf_hook_ops' type but inside a + network namespace. One linked list per protocol per hook type. + + struct net { ... struct netns_nf nf; ... } + struct netns_nf { ... + struct list_head hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; ... } + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("netns_nf") + and vmlinux.get_type("netns_nf").has_member("hooks") + and cls.get_member_type(vmlinux.get_type("netns_nf"), "hooks") + == ["array", "array", "list_head"] + ) + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + list_head = hook_container[proto_idx][hook_idx] + nf_hooks_ops_name = self.get_symbol_fullname("nf_hook_ops") + return list_head.to_list(nf_hooks_ops_name, "list") + + +class NetfilterImp_4_9_to_4_14(AbstractNetfilter): + """In this range of kernel versions, the doubly-linked lists of netfilter hooks were + replaced by an array of arrays of 'nf_hook_entry' pointers in a singly-linked lists. + struct net { ... struct netns_nf nf; ... } + struct netns_nf { .. + struct nf_hook_entry __rcu *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; ... } + + Also in v4.10 the struct nf_hook_entry changed, a hook function pointer was added to + it. However, for simplicity of this design, we will still take the hook address from + the 'nf_hook_ops'. As per v5.0-rc2, the hook address is duplicated in both sides. + - v4.9: + struct nf_hook_entry { + struct nf_hook_entry *next; + struct nf_hook_ops ops; + const struct nf_hook_ops *orig_ops; }; + - v4.10: + struct nf_hook_entry { + struct nf_hook_entry *next; + nf_hookfn *hook; + void *priv; + const struct nf_hook_ops *orig_ops; }; + (*) Even though the hook address is in the struct 'nf_hook_entry', we use the + original 'nf_hook_ops' hook address value, the one which was filled by the user, to + make it uniform to all the implementations. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + hooks_type = ["array", "array", "pointer", "nf_hook_entry"] + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("netns_nf") + and vmlinux.get_type("netns_nf").has_member("hooks") + and cls.get_member_type(vmlinux.get_type("netns_nf"), "hooks") == hooks_type + ) + + def _get_hook_ops(self, hook_container, proto_idx, hook_idx): + list_head = hook_container[proto_idx][hook_idx] + nf_hooks_ops_name = self.get_symbol_fullname("nf_hook_ops") + return list_head.to_list(nf_hooks_ops_name, "list") + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + nf_hook_entry_list = hook_container[proto_idx][hook_idx] + while nf_hook_entry_list: + yield nf_hook_entry_list.orig_ops + nf_hook_entry_list = nf_hook_entry_list.next + + +class NetfilterImp_4_14_to_4_16(AbstractNetfilter): + """'nf_hook_ops' was removed from struct 'nf_hook_entry'. Instead, it was stored + adjacent in memory to the 'nf_hook_entry' array, in the new struct 'nf_hook_entries' + However, 'orig_ops' is not part of the 'nf_hook_entries' struct definition. So, we + have to craft it by hand. + + struct net { ... struct netns_nf nf; ... } + struct netns_nf { + struct nf_hook_entries *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; ... } + struct nf_hook_entries { + u16 num_hook_entries; /* plus padding */ + struct nf_hook_entry hooks[]; + //const struct nf_hook_ops *orig_ops[]; } + struct nf_hook_entry { + nf_hookfn *hook; + void *priv; } + + (*) Even though the hook address is in the struct 'nf_hook_entry', we use the + original 'nf_hook_ops' hook address value, the one which was filled by the user, to + make it uniform to all the implementations. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + hooks_type = ["array", "array", "pointer", "nf_hook_entries"] + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("netns_nf") + and vmlinux.get_type("netns_nf").has_member("hooks") + and cls.get_member_type(vmlinux.get_type("netns_nf"), "hooks") == hooks_type + ) + + def get_nf_hook_entries(self, nf_hooks_addr, proto_idx, hook_idx): + """This allows to support different hook array implementations from this version + on. For instance, in kernels >= 4.16 this multi-dimensional array is split in + one-dimensional array of pointers to 'nf_hooks_entries' per each protocol.""" + return nf_hooks_addr[proto_idx][hook_idx] + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + nf_hook_entries = self.get_nf_hook_entries(hook_container, proto_idx, hook_idx) + if not nf_hook_entries: + return + + nf_hook_ops_name = self.get_symbol_fullname("nf_hook_ops") + nf_hook_ops_ptr_arr = self.build_nf_hook_ops_array(nf_hook_entries) + for nf_hook_ops_ptr in nf_hook_ops_ptr_arr: + nf_hook_ops = nf_hook_ops_ptr.dereference().cast(nf_hook_ops_name) + yield nf_hook_ops + + +class NetfilterImp_4_16_to_latest(NetfilterImp_4_14_to_4_16): + """The multidimensional array of nf_hook_entries was split in a one-dimensional + array per each protocol. + + struct net { + struct netns_nf nf; ... } + struct netns_nf { + struct nf_hook_entries * hooks_ipv4[NF_INET_NUMHOOKS]; + struct nf_hook_entries * hooks_ipv6[NF_INET_NUMHOOKS]; + struct nf_hook_entries * hooks_arp[NF_ARP_NUMHOOKS]; + struct nf_hook_entries * hooks_bridge[NF_INET_NUMHOOKS]; + struct nf_hook_entries * hooks_decnet[NF_DN_NUMHOOKS]; ... } + struct nf_hook_entries { + u16 num_hook_entries; /* plus padding */ + struct nf_hook_entry hooks[]; + //const struct nf_hook_ops *orig_ops[]; } + struct nf_hook_entry { + nf_hookfn *hook; + void *priv; } + + (*) Even though the hook address is in the struct nf_hook_entry, we use the original + nf_hook_ops hook address value, the one which was filled by the user, to make it + uniform to all the implementations. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("netns_nf") + and vmlinux.get_type("netns_nf").has_member("hooks_ipv4") + ) + + def get_hooks_container_by_protocol(self, net, proto_name): + try: + if proto_name == "IPV4": + net_nf_hooks = net.nf.hooks_ipv4 + elif proto_name == "ARP": + net_nf_hooks = net.nf.hooks_arp + elif proto_name == "BRIDGE": + net_nf_hooks = net.nf.hooks_bridge + elif proto_name == "IPV6": + net_nf_hooks = net.nf.hooks_ipv6 + elif proto_name == "DECNET": + net_nf_hooks = net.nf.hooks_decnet + else: + return + + yield net_nf_hooks + + except AttributeError: + # Protocol family disabled at kernel compilation + # CONFIG_NETFILTER_FAMILY_ARP=n || + # CONFIG_NETFILTER_FAMILY_BRIDGE=n || + # CONFIG_DECNET=n + pass + + def _get_nf_hook_entries_ptr(self, nf_hooks_addr, proto_idx, hook_idx): + nf_hook_entries_ptr = nf_hooks_addr[hook_idx] + return nf_hook_entries_ptr + + def get_nf_hook_entries(self, nf_hooks_addr, proto_idx, hook_idx): + return nf_hooks_addr[hook_idx] + + +class AbstractNetfilterNetDev(AbstractNetfilter): + """Base class to handle the Netfilter NetDev hooks. + It won't be executed. It has some common functions to all Netfilter NetDev hook + implementions. + + Netfilter NetDev hooks are set per network device which belongs to a network + namespace. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return False + + def subscribed_protocols(self): + return ("NETDEV",) + + def get_hooks_container_by_protocol(self, net, proto_name): + if proto_name != "NETDEV": + return + + net_device_type = self.vmlinux.get_type("net_device") + net_device_name = self.get_symbol_fullname("net_device") + for net_device in net.dev_base_head.to_list(net_device_name, "dev_list"): + if net_device_type.has_member("nf_hooks_ingress"): + # CONFIG_NETFILTER_INGRESS=y + yield net_device.nf_hooks_ingress + + if net_device_type.has_member("nf_hooks_egress"): + # CONFIG_NETFILTER_EGRESS=y + yield net_device.nf_hooks_egress + + +class NetfilterIngressImp_4_2_to_4_9(AbstractNetfilterNetDev): + """This is the first version of Netfilter Ingress hooks which was implemented using + a doubly-linked list of 'nf_hook_ops'. + struct list_head nf_hooks_ingress; + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + hooks_type = ["list_head"] + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("net_device") + and vmlinux.get_type("net_device").has_member("nf_hooks_ingress") + and cls.get_member_type(vmlinux.get_type("net_device"), "nf_hooks_ingress") + == hooks_type + ) + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + nf_hooks_ingress = hook_container + nf_hook_ops_name = self.get_symbol_fullname("nf_hook_ops") + return nf_hooks_ingress.to_list(nf_hook_ops_name, "list") + + +class NetfilterIngressImp_4_9_to_4_14(AbstractNetfilterNetDev): + """In 4.9 it was changed to a simple singly-linked list. + struct nf_hook_entry * nf_hooks_ingress; + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + hooks_type = ["pointer", "nf_hook_entry"] + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("net_device") + and vmlinux.get_type("net_device").has_member("nf_hooks_ingress") + and cls.get_member_type(vmlinux.get_type("net_device"), "nf_hooks_ingress") + == hooks_type + ) + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + nf_hooks_ingress_ptr = hook_container + if not nf_hooks_ingress_ptr: + return + + while nf_hooks_ingress_ptr: + nf_hook_entry = nf_hooks_ingress_ptr.dereference() + orig_ops = nf_hook_entry.orig_ops.dereference() + yield orig_ops + nf_hooks_ingress_ptr = nf_hooks_ingress_ptr.next + + +class NetfilterIngressImp_4_14_to_latest(AbstractNetfilterNetDev): + """In 4.14 the hook list was converted to an array of pointers inside the struct + 'nf_hook_entries': + struct nf_hook_entries * nf_hooks_ingress; + struct nf_hook_entries { + u16 num_hook_entries; + struct nf_hook_entry hooks[]; + //const struct nf_hook_ops *orig_ops[]; } + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + hooks_type = ["pointer", "nf_hook_entries"] + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("net_device") + and vmlinux.get_type("net_device").has_member("nf_hooks_ingress") + and cls.get_member_type(vmlinux.get_type("net_device"), "nf_hooks_ingress") + == hooks_type + ) + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + nf_hook_entries = hook_container + if not nf_hook_entries: + return + + nf_hook_ops_name = self.get_symbol_fullname("nf_hook_ops") + nf_hook_ops_ptr_arr = self.build_nf_hook_ops_array(nf_hook_entries) + for nf_hook_ops_ptr in nf_hook_ops_ptr_arr: + nf_hook_ops = nf_hook_ops_ptr.dereference().cast(nf_hook_ops_name) + yield nf_hook_ops + + +class Netfilter(interfaces.plugins.PluginInterface): + """Lists Netfilter hooks.""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + ] + + def _format_fields(self, fields): + ( + netns, + proto_name, + hook_name, + priority, + hook_func, + module_name, + hooked, + ) = fields + return ( + netns, + proto_name, + hook_name, + priority, + format_hints.Hex(hook_func), + module_name, + str(hooked), + ) + + def _generator(self): + for fields in AbstractNetfilter.run_all( + context=self.context, config=self.config + ): + yield (0, self._format_fields(fields)) + + def run(self): + headers = [ + ("Net NS", int), + ("Proto", str), + ("Hook", str), + ("Priority", int), + ("Handler", format_hints.Hex), + ("Module", str), + ("Is Hooked", str), + ] + return renderers.TreeGrid(headers, self._generator()) From 5ad7dd3bbcbad273e21610462e7e2517e87653ed Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 10 Jan 2024 16:06:55 -0300 Subject: [PATCH 002/263] Update netdev class names as it now supports both netdev hooks; ingress and egress --- volatility3/framework/plugins/linux/netfilter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index cc4786b4e..a9dbc742a 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -572,7 +572,7 @@ class AbstractNetfilterNetDev(AbstractNetfilter): yield net_device.nf_hooks_egress -class NetfilterIngressImp_4_2_to_4_9(AbstractNetfilterNetDev): +class NetfilterNetDevImp_4_2_to_4_9(AbstractNetfilterNetDev): """This is the first version of Netfilter Ingress hooks which was implemented using a doubly-linked list of 'nf_hook_ops'. struct list_head nf_hooks_ingress; @@ -595,7 +595,7 @@ class NetfilterIngressImp_4_2_to_4_9(AbstractNetfilterNetDev): return nf_hooks_ingress.to_list(nf_hook_ops_name, "list") -class NetfilterIngressImp_4_9_to_4_14(AbstractNetfilterNetDev): +class NetfilterNetDevImp_4_9_to_4_14(AbstractNetfilterNetDev): """In 4.9 it was changed to a simple singly-linked list. struct nf_hook_entry * nf_hooks_ingress; """ @@ -623,7 +623,7 @@ class NetfilterIngressImp_4_9_to_4_14(AbstractNetfilterNetDev): nf_hooks_ingress_ptr = nf_hooks_ingress_ptr.next -class NetfilterIngressImp_4_14_to_latest(AbstractNetfilterNetDev): +class NetfilterNetDevImp_4_14_to_latest(AbstractNetfilterNetDev): """In 4.14 the hook list was converted to an array of pointers inside the struct 'nf_hook_entries': struct nf_hook_entries * nf_hooks_ingress; From 5b5012d08ad7aff2dd48c5786e1bad2c62274989 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 10 Jan 2024 16:42:34 -0300 Subject: [PATCH 003/263] Fix. Use the global lists. DEC hooks were wrong --- volatility3/framework/plugins/linux/netfilter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index a9dbc742a..2b0a16a96 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -55,16 +55,16 @@ class AbstractNetfilter(ABC): PROTO_NOT_IMPLEMENTED, # NFPROTO_UNSPEC Proto(name="INET", hooks=NF_INET_HOOKS), # From kernels 3.14 Proto(name="IPV4", hooks=NF_INET_HOOKS), - Proto(name="ARP", hooks=("IN", "OUT", "FORWARD")), + Proto(name="ARP", hooks=NF_ARP_HOOKS), PROTO_NOT_IMPLEMENTED, - Proto(name="NETDEV", hooks=("INGRESS", "EGRESS")), + Proto(name="NETDEV", hooks=NF_NETDEV_HOOKS), PROTO_NOT_IMPLEMENTED, Proto(name="BRIDGE", hooks=NF_INET_HOOKS), PROTO_NOT_IMPLEMENTED, PROTO_NOT_IMPLEMENTED, Proto(name="IPV6", hooks=NF_INET_HOOKS), PROTO_NOT_IMPLEMENTED, - Proto(name="DECNET", hooks=NF_INET_HOOKS), # Removed in kernel 6.1 + Proto(name="DECNET", hooks=NF_DEC_HOOKS), # Removed in kernel 6.1 ) NF_MAX_HOOKS = LARGEST_HOOK_NUMBER + 1 From da6c2d863d17bdcef950cc8a8f9b7aa7c9e11b2a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 10 Jan 2024 16:43:36 -0300 Subject: [PATCH 004/263] Fix netdev egress hooks --- .../framework/plugins/linux/netfilter.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 2b0a16a96..33bb5c456 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -136,11 +136,13 @@ class AbstractNetfilter(ABC): allow customization of the code to the specific data structure used in a particular kernel implementation - get_hooks_container_by_protocol(net, proto_name) + get_hooks_container(net, proto_name, hook_name) It returns the data structure used in a specific kernel implementation to store the hooks for a respective namespace and protocol, basically: For Ingress hooks: network_namespace[] -> net_device[] -> nf_hooks_ingress[] + For egress hooks: + network_namespace[] -> net_device[] -> nf_hooks_egress[] For all the other Netfilter hooks: <= 4.2.8 nf_hooks[] @@ -148,7 +150,7 @@ class AbstractNetfilter(ABC): network_namespace[] -> nf.hooks[] get_hook_ops(hook_container, proto_idx, hook_idx) - Give the 'hook_container' got in get_hooks_container_by_protocol(), it + Give the 'hook_container' got in get_hooks_container(), it returns an iterable of 'nf_hook_ops' elements for a respective protocol and hook type. @@ -163,7 +165,7 @@ class AbstractNetfilter(ABC): """ for netns, net in self.get_net_namespaces(): for proto_idx, proto_name, hook_idx, hook_name in self._proto_hook_loop(): - hooks_container = self.get_hooks_container_by_protocol(net, proto_name) + hooks_container = self.get_hooks_container(net, proto_name, hook_name) for hook_container in hooks_container: for hook_ops in self.get_hook_ops( @@ -270,7 +272,7 @@ class AbstractNetfilter(ABC): net_ns_id = net.ns.inum yield net_ns_id, net - def get_hooks_container_by_protocol(self, net, proto_name): + def get_hooks_container(self, net, proto_name, hook_name): """Returns the data structure used in a specific kernel implementation to store the hooks for a respective namespace and protocol. @@ -281,7 +283,7 @@ class AbstractNetfilter(ABC): yield net.nf.hooks def get_hook_ops(self, hook_container, proto_idx, hook_idx): - """Given the hook_container obtained from get_hooks_container_by_protocol(), it + """Given the hook_container obtained from get_hooks_container(), it returns an iterable of 'nf_hook_ops' elements for a corresponding protocol and hook type. @@ -338,7 +340,7 @@ class NetfilterImp_to_4_3(AbstractNetfilter): netns, net = renderers.NotAvailableValue(), renderers.NotAvailableValue() yield netns, net - def get_hooks_container_by_protocol(self, net, proto_name): + def get_hooks_container(self, net, proto_name, hook_name): nf_hooks = self.vmlinux.object_from_symbol("nf_hooks") if not nf_hooks: return @@ -508,7 +510,7 @@ class NetfilterImp_4_16_to_latest(NetfilterImp_4_14_to_4_16): and vmlinux.get_type("netns_nf").has_member("hooks_ipv4") ) - def get_hooks_container_by_protocol(self, net, proto_name): + def get_hooks_container(self, net, proto_name, hook_name): try: if proto_name == "IPV4": net_nf_hooks = net.nf.hooks_ipv4 @@ -556,20 +558,19 @@ class AbstractNetfilterNetDev(AbstractNetfilter): def subscribed_protocols(self): return ("NETDEV",) - def get_hooks_container_by_protocol(self, net, proto_name): - if proto_name != "NETDEV": - return - + def get_hooks_container(self, net, proto_name, hook_name): net_device_type = self.vmlinux.get_type("net_device") net_device_name = self.get_symbol_fullname("net_device") for net_device in net.dev_base_head.to_list(net_device_name, "dev_list"): - if net_device_type.has_member("nf_hooks_ingress"): - # CONFIG_NETFILTER_INGRESS=y - yield net_device.nf_hooks_ingress + if hook_name == "INGRESS": + if net_device_type.has_member("nf_hooks_ingress"): + # CONFIG_NETFILTER_INGRESS=y + yield net_device.nf_hooks_ingress - if net_device_type.has_member("nf_hooks_egress"): - # CONFIG_NETFILTER_EGRESS=y - yield net_device.nf_hooks_egress + elif hook_name == "EGRESS": + if net_device_type.has_member("nf_hooks_egress"): + # CONFIG_NETFILTER_EGRESS=y + yield net_device.nf_hooks_egress class NetfilterNetDevImp_4_2_to_4_9(AbstractNetfilterNetDev): From 850b377dcf8b087d5296774b0803965271cda9b7 Mon Sep 17 00:00:00 2001 From: hsarkey Date: Fri, 9 Feb 2024 15:21:41 -0500 Subject: [PATCH 005/263] Updated windows.dlllist plugin to have --name and --base flags for filtering on dll name and base address. --- .../framework/plugins/windows/dlllist.py | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index f876dc1a2..4f50bdda5 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -5,6 +5,7 @@ import contextlib import datetime import logging import ntpath +import re from typing import List, Optional, Type from volatility3.framework import constants, exceptions, interfaces, renderers @@ -22,7 +23,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the loaded modules in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -53,6 +54,22 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): description="Process offset in the physical address space", optional=True, ), + requirements.StringRequirement( + name="name", + description="Specify a regular expression to match dll name(s)", + optional=True, + ), + requirements.IntRequirement( + name="base", + description="Specify a base address", + optional=True, + ), + requirements.BooleanRequirement( + name="ignore-case", + description="Specify case insensitivity for the regular expression name matching", + default=False, + optional=True, + ), requirements.BooleanRequirement( name="dump", description="Extract listed DLLs", @@ -144,9 +161,35 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): BaseDllName = FullDllName = renderers.UnreadableValue() with contextlib.suppress(exceptions.InvalidAddressException): BaseDllName = entry.BaseDllName.get_string() - # We assume that if the BaseDllName points to an invalid buffer, so will FullDllName + # We assume that if BaseDllName points to invalid buffer, so will FullDllName FullDllName = entry.FullDllName.get_string() + # Check if a name regex was passed and apply it to only show matches + if self.config["name"]: + try: + flags = re.I if self.config["ignore-case"] else 0 + mod_re = re.compile(self.config["name"], flags) + except re.error: + vollog.debug( + "Error parsing regular expression: %s", self.config["name"] + ) + return None + + # If Base or Full Dll Name are invalid, move on + if isinstance(BaseDllName, renderers.UnreadableValue) or isinstance( + FullDllName, renderers.UnreadableValue + ): + continue + + # If regex does not match, move on + if not mod_re.search(BaseDllName) and not mod_re.search( + FullDllName + ): + continue + + if self.config["base"] and self.config["base"] != entry.DllBase: + continue + if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute # and 32bit version shouldn't have the Quadpart according to MSDN From 47891c3477fbd7bd008e8b3b8f119936289104de Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Apr 2024 21:17:42 +0100 Subject: [PATCH 006/263] Core: Develop nested requirement files --- requirements-dev.txt | 21 ++++----------------- requirements.txt | 4 ++-- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c9b615cd8..ae3482290 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,22 +1,9 @@ -# The following packages are required for core functionality. -pefile>=2023.2.7 - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 - -# This is required for several plugins that perform malware analysis and disassemble code. -# It can also improve accuracy of Windows 8 and later memory samples. -capstone>=3.0.5 - -# This is required by plugins that decrypt passwords, password hashes, etc. -pycryptodome +-r requirements.txt # This can improve error messages regarding improperly configured ISF files, # but is only recommended for development jsonschema>=2.3.0 -# This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0 +# Used to build executable file +pyinstaller>=6.5.0 +pyinstaller-hooks-contrib>=2024.3 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4d09ff82a..c63dc0b36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -# The following packages are required for core functionality. -pefile>=2023.2.7 +# Include the minimal requirements +-r requirements-minimal.txt # The following packages are optional. # If certain packages are not necessary, place a comment (#) at the start of the line. From c1f239b8d2171e83e331ad51bf7ff6e48ed04e52 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 7 Apr 2024 19:34:17 +0100 Subject: [PATCH 007/263] Linux: Improve debugging of pslist dump feature --- volatility3/framework/plugins/linux/elfs.py | 1 + volatility3/framework/plugins/linux/pslist.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 6820576dc..22e39d127 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -84,6 +84,7 @@ class Elfs(plugins.PluginInterface): ) if not elf_object.is_valid(): + vollog.debug("ELF object to be dumped is not valid") return None sections = {} diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 046ee43d8..1888bd7b8 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -142,6 +142,8 @@ class PsList(interfaces.plugins.PluginInterface): file_output = str(file_handle.preferred_filename) file_handle.close() break + else: + file_output = "VMA start matching task start_code not found" return file_output def _generator( From 974e6a4107cda567ce24a3d6384c28939ae64dd1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 21 Apr 2024 18:28:13 +0100 Subject: [PATCH 008/263] Core: When clearing the cache, actually clear cached files --- volatility3/framework/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 1565b2267..422c083d0 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -225,6 +225,12 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: def clear_cache(complete=False): try: + if complete: + glob_pattern = "*.cache" + for cache_filename in glob.glob( + os.path.join(constants.CACHE_PATH, glob_pattern) + ): + os.unlink(cache_filename) os.unlink(os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)) except FileNotFoundError: vollog.log(constants.LOGLEVEL_VVVV, "Attempting to clear a non-existant cache") From 8f0d1b37cc2f88646b5858509e1bc0d5bf4941e0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Apr 2024 20:51:25 +0100 Subject: [PATCH 009/263] Core: Improve error handling for proxy authentication issue --- volatility3/framework/layers/resources.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index a64fa7d7a..2dba7caa8 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -151,6 +151,12 @@ class ResourceAccessor(object): raise excp else: raise excp + except ValueError as excp: + # Reraise errors such as proxy auth errors as offline exception errors + # Example Proxy auth error - ValueError: AbstractDigestAuthHandler does not support the following scheme: 'Negotiate' + vollog.info(f"Cannot access {url} due to {excp} - Setting OFFLINE") + constants.OFFLINE = True + raise exceptions.OfflineException(url) except exceptions.OfflineException: vollog.info(f"Not accessing {url} in offline mode") raise From 35c583721892b84ae4f9b151ddccbb2e4ef60e9f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 30 Apr 2024 12:34:46 +0100 Subject: [PATCH 010/263] Core: Default to completely clearing the cache when requested --- volatility3/framework/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 422c083d0..feb97810b 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -223,7 +223,7 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: return plugin_list -def clear_cache(complete=False): +def clear_cache(complete=True): try: if complete: glob_pattern = "*.cache" From 314725f0c53811b6652f90c4b8050982b8329ab7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 1 May 2024 04:30:36 +0900 Subject: [PATCH 011/263] Fix: typo for CITATION.cff --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index c36c3b7d5..ac45dfc18 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -14,7 +14,7 @@ authors: identifiers: - type: url value: 'https://github.com/volatilityfoundation/volatility3' - description: Volatility 3 source code respository + description: Volatility 3 source code repository repository-code: 'https://github.com/volatilityfoundation/volatility3' url: 'https://github.com/volatilityfoundation/volatility3' abstract: >- From c7f29936dcfb81bb9ae3fe33e261db2bb7c3be85 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:15:45 +1000 Subject: [PATCH 012/263] CommandLine: Fix issue when --output filename doesn't contain an extension. If the argument is i.e. "--output aaa" ... it returned ".aaa" (hidden filename in linux) then "-1.aaa", "-2.aaa", etc. --- volatility3/cli/__init__.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 457a49311..c0eed20ee 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -716,19 +716,17 @@ class CommandLine: """Gets the final filename""" if output_dir is None: raise TypeError("Output directory is not a string") + os.makedirs(output_dir, exist_ok=True) - pref_name_array = self.preferred_filename.split(".") - filename, extension = ( - os.path.join(output_dir, ".".join(pref_name_array[:-1])), - pref_name_array[-1], - ) - output_filename = f"{filename}.{extension}" + output_filename = os.path.join(output_dir, self.preferred_filename) + filename, extension = os.path.splitext(output_filename) counter = 1 while os.path.exists(output_filename): - output_filename = f"{filename}-{counter}.{extension}" + output_filename = f"{filename}-{counter}{extension}" counter += 1 + return output_filename class CLIMemFileHandler(io.BytesIO, CLIFileHandler): From 32a5f131fd22d5351eee264fd6e58420a1458e9c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:17:51 +1000 Subject: [PATCH 013/263] LayerWriter plugin: Fix log wrong (non-existent) variable --- volatility3/framework/plugins/layerwriter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 1bee5f20d..a8664b3f1 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -115,9 +115,7 @@ class LayerWriter(plugins.PluginInterface): ) file_handle.close() except IOError as excp: - yield 0, ( - f"Layer cannot be written to {self.config['output_name']}: {excp}", - ) + yield 0, (f"Layer cannot be written to {output_name}: {excp}",) yield 0, (f"Layer has been written to {output_name}",) From 1246b20b6521e11110038fe091af912e2b5f4439 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:25:45 +1000 Subject: [PATCH 014/263] LayerWriter plugin: Code improvements --- volatility3/framework/plugins/layerwriter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index a8664b3f1..021c44248 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -95,7 +95,7 @@ class LayerWriter(plugins.PluginInterface): if not self.config["layers"]: self.config["layers"] = [] for name in self.context.layers: - if not self.context.layers[name].metadata.get("mapped", False): + if "mapped" not in self.context.layers[name].metadata: self.config["layers"] = [name] for name in self.config["layers"]: @@ -103,7 +103,8 @@ class LayerWriter(plugins.PluginInterface): if name not in self.context.layers: yield 0, (f"Layer Name {name} does not exist",) else: - output_name = self.config.get("output", ".".join([name, "raw"])) + default_output_name = f"{name}.raw" + output_name = self.config.get("output", default_output_name) try: file_handle = self.write_layer( self.context, From 6d22347ce6dd0152746564a2d77022ca1b4d9045 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:27:15 +1000 Subject: [PATCH 015/263] LayerWriter plugin: Fix --output argument. It's referenced in the code but never mentioned as a requirement --- volatility3/framework/plugins/layerwriter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 021c44248..d60ead682 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -38,6 +38,10 @@ class LayerWriter(plugins.PluginInterface): default=False, optional=True, ), + requirements.StringRequirement( + name="output", + description="Output filename", + ), requirements.ListRequirement( name="layers", element_type=str, From e5a5b895771b655d21c36689c33a534034c31e36 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:06:02 +1000 Subject: [PATCH 016/263] Intel layer: Fix. This if statement will never be executed unless "minimum_address > maximum_address" which doesn't make sense to me. --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 75e561b33..8589cdbb6 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -180,7 +180,7 @@ class Intel(linear.LinearlyMappedLayer): position = self._initial_position entry = self._initial_entry - if self.minimum_address > offset > self.maximum_address: + if not (self.minimum_address <= offset <= self.maximum_address): raise exceptions.PagedInvalidAddressException( self.name, offset, From f116c08a7ec60f62e3ef931de7639e402f421d65 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:07:23 +1000 Subject: [PATCH 017/263] NonLinearlySegmentedLayer: Fix maximum addresses in segmented layers --- volatility3/framework/layers/segmented.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/segmented.py b/volatility3/framework/layers/segmented.py index 0d29d8bff..e8c067072 100644 --- a/volatility3/framework/layers/segmented.py +++ b/volatility3/framework/layers/segmented.py @@ -152,7 +152,7 @@ class NonLinearlySegmentedLayer( raise ValueError("SegmentedLayer must contain some segments") if self._maxaddr is None: mapped, _, length, _ = self._segments[-1] - self._maxaddr = mapped + length + self._maxaddr = mapped + length - 1 return self._maxaddr @property From a6c77c488436c7b05040b5bf475be79cb59457f3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:09:45 +1000 Subject: [PATCH 018/263] LayerWriter: Fix - Last chunk size is wrongly calculated --- volatility3/framework/plugins/layerwriter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index d60ead682..178d86ba9 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -18,7 +18,7 @@ class LayerWriter(plugins.PluginInterface): default_block_size = 0x500000 _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -81,7 +81,7 @@ class LayerWriter(plugins.PluginInterface): file_handle = open_method(preferred_name) for i in range(0, layer.maximum_address, chunk_size): - current_chunk_size = min(chunk_size, layer.maximum_address - i) + current_chunk_size = min(chunk_size, layer.maximum_address + 1 - i) data = layer.read(i, current_chunk_size, pad=True) file_handle.write(data) if progress_callback: From d7aae3a9828ba03bd5531aa385724a954e48501a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:11:52 +1000 Subject: [PATCH 019/263] Store the real/final output filename so that we can notify it correctly to the user --- volatility3/cli/__init__.py | 4 ++-- volatility3/framework/plugins/layerwriter.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index c0eed20ee..9873791e3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -790,8 +790,8 @@ class CommandLine: return None self._file.close() - output_filename = self._get_final_filename() - os.rename(self._name, output_filename) + self._output_filename = self._get_final_filename() + os.rename(self._name, self._output_filename) if direct: return CLIDirectFileHandler diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 178d86ba9..20d10d636 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -119,6 +119,7 @@ class LayerWriter(plugins.PluginInterface): progress_callback=self._progress_callback, ) file_handle.close() + output_name = file_handle._output_filename except IOError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) From 85052b6238414617cc87f6812894f14917e03393 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 7 May 2024 08:12:31 +1000 Subject: [PATCH 020/263] LayerWriter: Fix missing optional flag for the --output argument --- volatility3/framework/plugins/layerwriter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 20d10d636..6a06cc607 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -41,6 +41,7 @@ class LayerWriter(plugins.PluginInterface): requirements.StringRequirement( name="output", description="Output filename", + optional=True, ), requirements.ListRequirement( name="layers", From 3f3f1a9c0daae397a84a515e84b29ba9ef0a0300 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 8 May 2024 11:23:01 -0500 Subject: [PATCH 021/263] Prevent duplicate processing of the same file object --- volatility3/framework/plugins/windows/dumpfiles.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 48539c752..33d2d0d41 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -244,6 +244,8 @@ class DumpFiles(interfaces.plugins.PluginInterface): symbol_table=kernel.symbol_table_name, ) + dumped_files = set() + for proc in procs: try: object_table = proc.ObjectTable @@ -267,6 +269,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_re.search(name): continue + if file_obj.vol.offset in dumped_files: + continue + dumped_files.add(file_obj.vol.offset) + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): @@ -303,6 +309,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_re.search(name): continue + if file_obj.vol.offset in dumped_files: + continue + dumped_files.add(file_obj.vol.offset) + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): From 3933061551eb0f31b99229ac8346718f773f574c Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 9 May 2024 09:17:32 -0500 Subject: [PATCH 022/263] Add a default 0 value to macb columns to avoid mactime not reporting timeline entries --- volatility3/framework/plugins/timeliner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index d1c1c0f70..26a100f53 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -183,16 +183,16 @@ class Timeliner(interfaces.plugins.PluginInterface): plugin_name, self._sanitize_body_format(item), self._text_format( - times.get(TimeLinerType.ACCESSED, "") + times.get(TimeLinerType.ACCESSED, "0") ), self._text_format( - times.get(TimeLinerType.MODIFIED, "") + times.get(TimeLinerType.MODIFIED, "0") ), self._text_format( - times.get(TimeLinerType.CHANGED, "") + times.get(TimeLinerType.CHANGED, "0") ), self._text_format( - times.get(TimeLinerType.CREATED, "") + times.get(TimeLinerType.CREATED, "0") ), ) ) From 4c937a922d92175135e64be2192b94e8a232c4df Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 9 May 2024 14:47:56 -0500 Subject: [PATCH 023/263] Correctly check for a failed read --- volatility3/framework/plugins/windows/handles.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index d43d26ef1..93ca37c6b 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -161,8 +161,9 @@ class Handles(interfaces.plugins.PluginInterface): except exceptions.SymbolError: return None - data = self.context.layers.read(virtual_layer_name, kvo + func_addr, 0x200) - if data is None: + try: + data = self.context.layers.read(virtual_layer_name, kvo + func_addr, 0x200) + except exceptions.InvalidAddressException: return None md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) From 2b165a56be4d5c804ec6104e165cdf0f862b7204 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 9 May 2024 14:53:24 -0500 Subject: [PATCH 024/263] Fix pre-existing formatting issue from black checks --- volatility3/framework/plugins/windows/handles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 93ca37c6b..15f5f69df 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -162,7 +162,9 @@ class Handles(interfaces.plugins.PluginInterface): return None try: - data = self.context.layers.read(virtual_layer_name, kvo + func_addr, 0x200) + data = self.context.layers.read( + virtual_layer_name, kvo + func_addr, 0x200 + ) except exceptions.InvalidAddressException: return None From 5d5fa96e368f24e11f0860bdd43882da6677de25 Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 16 May 2024 09:19:59 +0100 Subject: [PATCH 025/263] Windows: add extra debugging messages to handles plugin, ref #1146 --- .../framework/plugins/windows/handles.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 15f5f69df..a19e7a397 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -25,7 +25,7 @@ class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -145,6 +145,9 @@ class Handles(interfaces.plugins.PluginInterface): if self._sar_value is None: if not has_capstone: + vollog.debug( + "capstone module is missing, unable to create disassembly of ObpCaptureHandleInformationEx" + ) return None kernel = self.context.modules[self.config["kernel"]] @@ -159,28 +162,46 @@ class Handles(interfaces.plugins.PluginInterface): try: func_addr = ntkrnlmp.get_symbol("ObpCaptureHandleInformationEx").address except exceptions.SymbolError: + vollog.debug("Unable to locate ObpCaptureHandleInformationEx symbol") return None try: + func_addr_to_read = kvo + func_addr + num_bytes_to_read = 0x200 + vollog.debug( + f"ObpCaptureHandleInformationEx symbol located at {hex(func_addr_to_read)}" + ) data = self.context.layers.read( - virtual_layer_name, kvo + func_addr, 0x200 + virtual_layer_name, func_addr_to_read, num_bytes_to_read ) except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}" + ) return None md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) + instruction_count = 0 for address, size, mnemonic, op_str in md.disasm_lite( data, kvo + func_addr ): # print("{} {} {} {}".format(address, size, mnemonic, op_str)) - + instruction_count += 1 if mnemonic.startswith("sar"): # if we don't want to parse op strings, we can disasm the # single sar instruction again, but we use disasm_lite for speed self._sar_value = int(op_str.split(",")[1].strip(), 16) + vollog.debug( + f"SAR located at {hex(address)} with value of {hex(self._sar_value)}" + ) break + if self._sar_value is None: + vollog.debug( + f"Failed to to locate SAR value having parsed {instruction_count} instructions" + ) + return self._sar_value @classmethod From c2b2321622ad1c170e10cf847b566796842a8020 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:09:14 -0500 Subject: [PATCH 026/263] Add a new getcellroutine plugin that reports hooked GetCellRoutine handlers of memory mapped Windows registry hives --- .../windows/registry/getcellroutine.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 volatility3/framework/plugins/windows/registry/getcellroutine.py diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py new file mode 100644 index 000000000..08523d2f1 --- /dev/null +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -0,0 +1,97 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import ssdt +from volatility3.plugins.windows.registry import hivelist + +vollog = logging.getLogger(__name__) + +class GetCellRoutine(interfaces.plugins.PluginInterface): + """ Reports registry hives with a hooked GetCellRoutine handler """ + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), + ] + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + collection = ssdt.SSDT.build_module_collection( + self.context, kernel.layer_name, kernel.symbol_table_name + ) + + # walk each hive and validate that the GetCellRoutine handler + # is inside of the kernel (ntoskrnl) + for hive_object in hivelist.HiveList.list_hives( + context=self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name + ): + hive = hive_object.hive + + try: + cellroutine = hive.GetCellRoutine + except exceptions.InvalidAddressException: + continue + + module_symbols = list( + collection.get_module_symbols_by_absolute_location(cellroutine) + ) + + if module_symbols: + for module_name, _ in module_symbols: + # GetCellRoutine handlers should only be in the kernel + if module_name not in constants.windows.KERNEL_MODULE_NAMES: + yield ( + 0, + ( + format_hints.Hex(hive.vol.offset), + hive_object.get_name() or "", + module_name, + format_hints.Hex(cellroutine) + ) + ) + # Doesn't map to any module... + else: + yield ( + 0, + ( + format_hints.Hex(hive.vol.offset), + hive_object.get_name() or "", + renderers.NotAvailableValue(), + format_hints.Hex(cellroutine) + ) + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Hive Offset", renderers.format_hints.Hex), + ("Hive Name", str), + ("GetCellRoutine Module", str), + ("GetCellRoutine Handler", renderers.format_hints.Hex) + ], + self._generator(), + ) From 47eb42204e19c482ff332ce9bb36fcbe54fa49d3 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:21:03 -0500 Subject: [PATCH 027/263] Fixes for black --- .../plugins/windows/registry/getcellroutine.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index 08523d2f1..2cbb87565 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -14,7 +14,7 @@ from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) class GetCellRoutine(interfaces.plugins.PluginInterface): - """ Reports registry hives with a hooked GetCellRoutine handler """ + """Reports registry hives with a hooked GetCellRoutine handler""" _required_framework_version = (2, 0, 0) @@ -32,7 +32,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) ), - ] + ] def _generator(self): kernel = self.context.modules[self.config["kernel"]] @@ -47,7 +47,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): context=self.context, base_config_path=self.config_path, layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name + symbol_table=kernel.symbol_table_name, ): hive = hive_object.hive @@ -70,7 +70,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): format_hints.Hex(hive.vol.offset), hive_object.get_name() or "", module_name, - format_hints.Hex(cellroutine) + format_hints.Hex(cellroutine), ) ) # Doesn't map to any module... @@ -81,7 +81,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): format_hints.Hex(hive.vol.offset), hive_object.get_name() or "", renderers.NotAvailableValue(), - format_hints.Hex(cellroutine) + format_hints.Hex(cellroutine), ) ) From 4ed7d40ecb61f1d4e671f3323a1b075ad2501bfc Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:23:21 -0500 Subject: [PATCH 028/263] Fixes for black --- .../framework/plugins/windows/registry/getcellroutine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index 2cbb87565..98b3c2517 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -71,7 +71,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): hive_object.get_name() or "", module_name, format_hints.Hex(cellroutine), - ) + ), ) # Doesn't map to any module... else: @@ -82,7 +82,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): hive_object.get_name() or "", renderers.NotAvailableValue(), format_hints.Hex(cellroutine), - ) + ), ) def run(self): @@ -91,7 +91,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): ("Hive Offset", renderers.format_hints.Hex), ("Hive Name", str), ("GetCellRoutine Module", str), - ("GetCellRoutine Handler", renderers.format_hints.Hex) + ("GetCellRoutine Handler", renderers.format_hints.Hex), ], self._generator(), ) From bc8666b64a4b722918879eda5efcadeab633bfd2 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:24:53 -0500 Subject: [PATCH 029/263] Fixes for black --- volatility3/framework/plugins/windows/registry/getcellroutine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index 98b3c2517..ed54a135d 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -13,6 +13,7 @@ from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) + class GetCellRoutine(interfaces.plugins.PluginInterface): """Reports registry hives with a hooked GetCellRoutine handler""" From 901b0fd6baeaf9779a7111172b579f13688e8ec8 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 15:11:06 -0500 Subject: [PATCH 030/263] Fix year in header --- .../framework/plugins/windows/registry/getcellroutine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index ed54a135d..200a45a82 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # From 3eab0671b48818aeb4af1873214bbe506fd7b7de Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 20 May 2024 20:47:12 +0100 Subject: [PATCH 031/263] Generic: Add vmscan plugin --- volatility3/framework/plugins/vmscan.py | 217 ++++++++++++++++++ .../generic/vmcs/haswell-architecture.json | 131 +++++++++++ .../generic/vmcs/skylake-architecture.json | 131 +++++++++++ 3 files changed, 479 insertions(+) create mode 100644 volatility3/framework/plugins/vmscan.py create mode 100644 volatility3/symbols/generic/vmcs/haswell-architecture.json create mode 100644 volatility3/symbols/generic/vmcs/skylake-architecture.json diff --git a/volatility3/framework/plugins/vmscan.py b/volatility3/framework/plugins/vmscan.py new file mode 100644 index 000000000..9d82ba214 --- /dev/null +++ b/volatility3/framework/plugins/vmscan.py @@ -0,0 +1,217 @@ +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import enum +import logging +import os +import struct +from typing import Dict, List + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import configuration, plugins +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed + +vollog = logging.getLogger(__name__) + + +class VMCSTest(enum.IntFlag): + VMCS_ABORT_INVALID = enum.auto() + VMCS_LINK_PTR_IS_NOT_FS = enum.auto() + VMCS_HOST_CR4_NO_VTX = enum.auto() + VMCS_CR3_IS_ZERO = enum.auto() + VMCS_GUEST_CR4_RESERVED = enum.auto() + + +class PageStartScanner(interfaces.layers.ScannerInterface): + def __init__(self, signatures: List[bytes], page_size: int = 0x1000): + super().__init__() + if not len(signatures): + raise ValueError("No signatures passed to constructor") + self._siglen = len(signatures[0]) + for item in signatures: + if len(item) != self._siglen: + raise ValueError( + "Signatures of different lengths passed to PageStartScanner" + ) + self._signatures = signatures + self._page_size = page_size + + def __call__(self, data: bytes, data_offset: int): + """Scans only the start of every page, to see whether a signature is present or not""" + for page_start in range( + data_offset % self._page_size, len(data), self._page_size + ): + if data[page_start : page_start + self._siglen] in self._signatures: + yield ( + page_start + data_offset, + data[page_start : page_start + self._siglen], + ) + + +class Vmscan(plugins.PluginInterface): + """Scans for Intel VT-d structues and generates VM volatility configs for them""" + + _required_framework_version = (2, 2, 0) + _version = (1, 0, 0) + + STRICTLY_REQUIRED_TESTS = { + VMCSTest.VMCS_ABORT_INVALID, + VMCSTest.VMCS_LINK_PTR_IS_NOT_FS, + VMCSTest.VMCS_HOST_CR4_NO_VTX, + } + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.TranslationLayerRequirement( + name="primary", description="Physical base memory layer" + ), + requirements.IntRequirement( + name="log-threshold", + description="Number of criteria failed to log to debug output", + default=2, + optional=True, + ), + ] + + # Scan for VMCS structures based on the known VMCS structures + # found in symbols/vmcs directory + + def _gather_vmcs_structures( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[bytes, str]: + """Enumerate all JSON files containing VMCS information and return the structures + Signatures can be generated using data extracted using the vmcs_layout tool at + https://github.com/google/rekall/tree/master/tools/linux/vmcs_layout + + Args: + context: The volatility context to work against + config_path: The location to store symbol table configurations under + + Returns: + A dictionary of pattern bytes to the string representation of the architecture + """ + filenames = intermed.IntermediateSymbolTable.file_symbol_url( + os.path.join("generic", "vmcs") + ) + table_names = [] + for filename in filenames: + base_name = os.path.basename(filename).split(".")[0] + table_name = intermed.IntermediateSymbolTable.create( + context, + configuration.path_join(config_path, "vmcs"), + os.path.join("generic", "vmcs"), + filename=base_name, + ) + table_names.append(table_name) + + result = {} + for table_name in table_names: + symbol_table = context.symbol_space[table_name] + revision_id = struct.pack( + " List[str]: + """Runs tests to verify whether a block of data is a VMCS page + Some tests based on the Hypervisor Memory Forensics paper by + Mariano Graziano, Andrea Lanzi and Davide Balzarotti + + Args: + context: The volatility context to be used for this call + vmcs: The instantiated VMCS object to verify + + Returns: + The list of failed criteria that the VMCS did not meet + """ + + # The VMCS should have been constructed on the physical layer (even a nested VMCS) + physical_layer_name = vmcs.vol.layer_name + + failed_tests: VMCSTest = VMCSTest(0) + # The abort field must be valid (generally 0, although other abort codes may exist) + if context.layers[physical_layer_name].read(vmcs.vol.offset + 4, 4) not in [ + b"\x00\x00\x00\x00" + ]: + failed_tests |= VMCSTest.VMCS_ABORT_INVALID + # The vmcs link pointer is supposed to always be set + if vmcs.vmcs_link_ptr != 0xFFFFFFFFFFFFFFFF: + failed_tests |= VMCSTest.VMCS_LINK_PTR_IS_NOT_FS + # To have a VMCS the host needs the VTx bit set in CR4, this can false positive often when all bits are set + if (vmcs.host_cr4 & 1 << 13) == 0: + failed_tests |= VMCSTest.VMCS_HOST_CR4_NO_VTX + # The guest CR3 is *exceptionally* unlikely to be 0 and the guest cr4 is likely to have some bits unset + if (vmcs.guest_cr3 == 0) or (vmcs.host_cr3 == 0): + failed_tests |= VMCSTest.VMCS_CR3_IS_ZERO + # CR4 registers have certain bits reserved that should not be set + if vmcs.guest_cr4 & 0xFFFFFFFFFF889000: + failed_tests |= VMCSTest.VMCS_GUEST_CR4_RESERVED + + if failed_tests and failed_tests.name: + failed_list = failed_tests.name.split("|") + return failed_list + + return [] + + def _generator(self): + # Gather VMCS structures + structures = self._gather_vmcs_structures(self.context, self.config_path) + # Scan memory for them + layer = self.context.layers[self.config["primary"]] + + # Try to move down to the highest physical layer + if layer.config.get("memory_layer"): + layer = self.context.layers[layer.config["memory_layer"]] + + # Run the scan + for offset, match in layer.scan( + self.context, + PageStartScanner(list(structures.keys())), + self._progress_callback, + ): + try: + vmcs = self.context.object( + structures[match] + constants.BANG + "_VMCS", + layer.name, + offset=offset, + ) + failed_list = self._verify_vmcs_page(self.context, vmcs) + if not failed_list: + yield ( + 0, + ( + structures[match], + format_hints.Hex(vmcs.vol.offset), + format_hints.Hex(vmcs.ept), + format_hints.Hex(vmcs.guest_cr3), + ), + ) + if len(failed_list) <= self.config["log-threshold"]: + vollog.debug( + f"Potential {structures[match]} VMCS found at {vmcs.vol.offset:x} with failed criteria: {failed_list}" + ) + except (exceptions.InvalidAddressException, AttributeError): + # Not what we're looking for + continue + + def run(self): + return renderers.TreeGrid( + [ + ("Architecture", str), + ("VMCS Physical offset", format_hints.Hex), + ("EPT", format_hints.Hex), + ("Guest CR3", format_hints.Hex), + ], + self._generator(), + ) diff --git a/volatility3/symbols/generic/vmcs/haswell-architecture.json b/volatility3/symbols/generic/vmcs/haswell-architecture.json new file mode 100644 index 000000000..33b2e55d8 --- /dev/null +++ b/volatility3/symbols/generic/vmcs/haswell-architecture.json @@ -0,0 +1,131 @@ +{ + "base_types": { + "pointer": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned char": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + } + }, + "enums": {}, + "metadata": { + "format": "6.1.0", + "producer": { + "datetime": "2021-07-31T17:37:28.313255", + "name": "vmextract-by-hand", + "version": "0.0.1" + } + }, + "symbols": { + "revision_id": { + "address": 0, + "constant_data": "MTg=" + } + }, + "user_types": { + "_VMCS": { + "fields": { + "ept": { + "offset": 320, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "executive_vmcs_ptr": { + "offset": 208, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr3": { + "offset": 528, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr4": { + "offset": 536, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_pdpte": { + "offset": 544, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "unsigned long long" + } + } + }, + "guest_physical_addr": { + "offset": 328, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr3": { + "offset": 816, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr4": { + "offset": 824, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vmcs_link_ptr": { + "offset": 248, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vpid": { + "offset": 206, + "type": { + "kind": "struct", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 4096 + } + } +} \ No newline at end of file diff --git a/volatility3/symbols/generic/vmcs/skylake-architecture.json b/volatility3/symbols/generic/vmcs/skylake-architecture.json new file mode 100644 index 000000000..78b110d94 --- /dev/null +++ b/volatility3/symbols/generic/vmcs/skylake-architecture.json @@ -0,0 +1,131 @@ +{ + "base_types": { + "pointer": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned char": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + } + }, + "enums": {}, + "metadata": { + "format": "6.1.0", + "producer": { + "datetime": "2021-07-16T16:21:01.062423", + "name": "vmextract-by-hand", + "version": "0.0.1" + } + }, + "symbols": { + "revision_id": { + "address": 0, + "constant_data": "NA==" + } + }, + "user_types": { + "_VMCS": { + "fields": { + "ept": { + "offset": 320, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "executive_vmcs_ptr": { + "offset": 208, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr3": { + "offset": 528, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr4": { + "offset": 536, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_pdpte": { + "offset": 544, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "unsigned long long" + } + } + }, + "guest_physical_addr": { + "offset": 328, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr3": { + "offset": 816, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr4": { + "offset": 824, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vmcs_link_ptr": { + "offset": 248, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vpid": { + "offset": 206, + "type": { + "kind": "struct", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 4096 + } + } +} From c5a45f91fc4968a46389781a4e21c34c771578a2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 20 May 2024 21:09:00 +0100 Subject: [PATCH 032/263] Linux: Replace uses of specific types with the more generic pointer --- volatility3/framework/automagic/linux.py | 2 +- volatility3/framework/plugins/linux/kmsg.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2eebcc2dc..7d7c563b6 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -159,7 +159,7 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): # This we get for free aslr_shift = ( - init_task.files.cast("long unsigned int") + init_task.files.cast("pointer") - module.get_symbol("init_files").address ) kaslr_shift = init_task_address - cls.virtual_to_physical_address( diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index c5e0fc302..e26d69543 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -66,7 +66,7 @@ class ABCKmsg(ABC): self._config = config self.vmlinux = context.modules[self._config["kernel"]] self.layer_name = self.vmlinux.layer_name # type: ignore - self.long_unsigned_int_size = self.vmlinux.get_type("long unsigned int").size + self.long_unsigned_int_size = self.vmlinux.get_type("pointer").size @classmethod def run_all( From c6d207d170c50ad460de4b9970e74405bd0e1d0d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 21 May 2024 09:06:01 +0100 Subject: [PATCH 033/263] Generic: Fix code scanning issue with vmscan --- volatility3/framework/plugins/vmscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/vmscan.py b/volatility3/framework/plugins/vmscan.py index 9d82ba214..64377d7d8 100644 --- a/volatility3/framework/plugins/vmscan.py +++ b/volatility3/framework/plugins/vmscan.py @@ -120,7 +120,7 @@ class Vmscan(plugins.PluginInterface): @classmethod def _verify_vmcs_page( - self, + cls, context: interfaces.context.ContextInterface, vmcs: interfaces.objects.ObjectInterface, ) -> List[str]: From 08493e9c3affda195c7c83a27c9e80bb020fe4c2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 21 May 2024 23:13:31 +0100 Subject: [PATCH 034/263] Automagic: Do some slight gymnastics to get the true value of the pointer --- volatility3/framework/automagic/linux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 7d7c563b6..52a73f45a 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -159,7 +159,10 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): # This we get for free aslr_shift = ( - init_task.files.cast("pointer") + int.from_bytes( + init_task.files.cast("bytes", length=init_task.files.vol.size), + byteorder=init_task.files.vol.data_format.byteorder, + ) - module.get_symbol("init_files").address ) kaslr_shift = init_task_address - cls.virtual_to_physical_address( From 57b3a99a1e42bfdb102f5038469999a815880330 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 May 2024 20:40:36 +0100 Subject: [PATCH 035/263] Infra: Upgrade deprecated githubs actions --- .github/workflows/build-pypi.yml | 6 +++--- .github/workflows/install.yml | 2 +- .github/workflows/test.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index f21898971..5a90364cd 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -20,9 +20,9 @@ jobs: matrix: python-version: ["3.7"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -37,7 +37,7 @@ jobs: python setup.py bdist_wheel - name: Archive dist - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: volatility3-pypi path: | diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index cc2a7fd3e..917685187 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b1a9dd31b..7ce6b13bb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,9 +8,9 @@ jobs: matrix: python-version: ["3.7"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From 35122df27d6d65c661c5fc43508e61609f262714 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 May 2024 20:42:25 +0100 Subject: [PATCH 036/263] Infra: Upgrade deprecated githubs actions 2 --- .github/workflows/black.yml | 2 +- .github/workflows/install.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 5f4523072..5adab3259 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -6,7 +6,7 @@ jobs: lint: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: options: "--check --diff --verbose" diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 917685187..e13161085 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -10,10 +10,10 @@ jobs: host: [ ubuntu-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 771ed10b44573a7f8baa32822f3bc524195fe0c9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 May 2024 20:44:04 +0100 Subject: [PATCH 037/263] Core: Bump to 2.7.1 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3c5e0eb2f..6963867f7 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 7 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 76c8067b21ccf4a596d06e6d2d470f01c39e1fa2 Mon Sep 17 00:00:00 2001 From: Branch Vincent Date: Sat, 1 Jun 2024 10:47:20 -0700 Subject: [PATCH 038/263] add full and dev extras --- requirements.txt | 2 +- setup.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index c63dc0b36..bd61edf14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ capstone>=3.0.5 pycryptodome # This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0 +leechcorepyc>=2.4.0; sys_platform != 'darwin' # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage gcsfs>=2023.1.0 diff --git a/setup.py b/setup.py index c2c55067d..6987c4359 100644 --- a/setup.py +++ b/setup.py @@ -10,12 +10,12 @@ with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() -def get_install_requires(): +def get_requires(filename): requirements = [] - with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: + with open(filename, "r", encoding="utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() - if stripped_line == "" or stripped_line.startswith("#"): + if stripped_line == "" or stripped_line.startswith(("#", "-r")): continue requirements.append(stripped_line) return requirements @@ -49,5 +49,9 @@ setuptools.setup( "volshell = volatility3.cli.volshell:main", ], }, - install_requires=get_install_requires(), + install_requires=get_requires("requirements-minimal.txt"), + extras_require={ + "dev": get_requires("requirements-dev.txt"), + "full": get_requires("requirements.txt"), + }, ) From 693eebc34b17fc067f8a91739b8aac884ce1481b Mon Sep 17 00:00:00 2001 From: Branch Vincent Date: Sat, 1 Jun 2024 15:29:01 -0700 Subject: [PATCH 039/263] migrate to pyproject.toml --- .github/workflows/build-pypi.yml | 5 ++- .github/workflows/test.yaml | 5 ++- .gitignore | 1 + README.md | 5 ++- pyproject.toml | 32 ++++++++++++++++++ setup.py | 33 ------------------- volatility3/framework/constants/__init__.py | 21 ++++-------- volatility3/framework/constants/_version.py | 13 ++++++++ volatility3/framework/interfaces/automagic.py | 2 +- 9 files changed, 60 insertions(+), 57 deletions(-) create mode 100644 pyproject.toml create mode 100644 volatility3/framework/constants/_version.py diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 5a90364cd..346d0337e 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -29,12 +29,11 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install build - name: Build PyPi packages run: | - python setup.py sdist --formats=gztar,zip - python setup.py bdist_wheel + python -m build - name: Archive dist uses: actions/upload-artifact@v4 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7ce6b13bb..55b2e4b60 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -18,13 +18,12 @@ jobs: run: | python -m pip install --upgrade pip pip install Cmake - pip install setuptools wheel + pip install build pip install -r ./test/requirements-testing.txt - name: Build PyPi packages run: | - python setup.py sdist --formats=gztar,zip - python setup.py bdist_wheel + python -m build - name: Download images run: | diff --git a/.gitignore b/.gitignore index 328ba5f83..c132736a9 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ config*.json # Pyinstaller files build dist +*.egg-info # Environments .env diff --git a/README.md b/README.md index f69acb3bb..d303d2dff 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,10 @@ Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of pip3 install -r requirements-minimal.txt ``` -Alternately, the minimal packages will be installed automatically when Volatility 3 is installed using setup.py. However, as noted in the Quick Start section below, Volatility 3 does not *need* to be installed via setup.py prior to using it. +Alternately, the minimal packages will be installed automatically when Volatility 3 is installed using pip. However, as noted in the Quick Start section below, Volatility 3 does not *need* to be installed prior to using it. ```shell -python3 setup.py build -python3 setup.py install +pip3 install . ``` To enable the full range of Volatility 3 functionality, use a command like the one below. For partial functionality, comment out any unnecessary packages in [requirements.txt](requirements.txt) prior to running the command. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..98c562a90 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "volatility3" +description = "Memory forensics framework" +keywords = ["volatility", "memory", "forensics", "framework", "windows", "linux", "volshell"] +readme = "README.md" +authors = [ + { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, +] +requires-python = ">=3.7.0" +license = { text = "VSL" } +dynamic = ["dependencies", "optional-dependencies", "version"] + +[project.urls] +Homepage = "https://github.com/volatilityfoundation/volatility3/" +"Bug Tracker" = "https://github.com/volatilityfoundation/volatility3/issues" +Documentation = "https://volatility3.readthedocs.io/" +"Source Code" = "https://github.com/volatilityfoundation/volatility3" + +[project.scripts] +vol = "volatility3.cli:main" +volshell = "volatility3.cli.volshell:main" + +[tool.setuptools.dynamic] +version = { attr = "volatility3.framework.constants._version.PACKAGE_VERSION" } +dependencies = { file = "requirements-minimal.txt" } + +[tool.setuptools.packages.find] +include = ["volatility3*"] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 6987c4359..3af033160 100644 --- a/setup.py +++ b/setup.py @@ -4,11 +4,6 @@ import setuptools -from volatility3.framework import constants - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - def get_requires(filename): requirements = [] @@ -22,34 +17,6 @@ def get_requires(filename): setuptools.setup( - name="volatility3", - description="Memory forensics framework", - version=constants.PACKAGE_VERSION, - license="VSL", - keywords="volatility memory forensics framework windows linux volshell", - author="Volatility Foundation", - long_description=long_description, - long_description_content_type="text/markdown", - author_email="volatility@volatilityfoundation.org", - url="https://github.com/volatilityfoundation/volatility3/", - project_urls={ - "Bug Tracker": "https://github.com/volatilityfoundation/volatility3/issues", - "Documentation": "https://volatility3.readthedocs.io/", - "Source Code": "https://github.com/volatilityfoundation/volatility3", - }, - packages=setuptools.find_namespace_packages( - include=["volatility3", "volatility3.*"] - ), - package_dir={"volatility3": "volatility3"}, - python_requires=">=3.7.0", - include_package_data=True, - entry_points={ - "console_scripts": [ - "vol = volatility3.cli:main", - "volshell = volatility3.cli.volshell:main", - ], - }, - install_requires=get_requires("requirements-minimal.txt"), extras_require={ "dev": get_requires("requirements-dev.txt"), "full": get_requires("requirements.txt"), diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 6963867f7..8743a64b0 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -14,6 +14,13 @@ from typing import Callable, Optional import volatility3.framework.constants.linux import volatility3.framework.constants.windows +from volatility3.framework.constants._version import ( + PACKAGE_VERSION, + VERSION_MAJOR, + VERSION_MINOR, + VERSION_PATCH, + VERSION_SUFFIX, +) PLUGINS_PATH = [ os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "plugins")), @@ -42,20 +49,6 @@ if hasattr(sys, "frozen") and sys.frozen: BANG = "!" """Constant used to delimit table names from type names when referring to a symbol""" -# We use the SemVer 2.0.0 versioning scheme -VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 7 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface -VERSION_SUFFIX = "" - -# TODO: At version 2.0.0, remove the symbol_shift feature - -PACKAGE_VERSION = ( - ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) - + VERSION_SUFFIX -) -"""The canonical version of the volatility3 package""" - AUTOMAGIC_CONFIG_PATH = "automagic" """The root section within the context configuration for automagic values""" diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py new file mode 100644 index 000000000..37c115431 --- /dev/null +++ b/volatility3/framework/constants/_version.py @@ -0,0 +1,13 @@ +# We use the SemVer 2.0.0 versioning scheme +VERSION_MAJOR = 2 # Number of releases of the library with a breaking change +VERSION_MINOR = 7 # Number of changes that only add to the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_SUFFIX = "" + +# TODO: At version 2.0.0, remove the symbol_shift feature + +PACKAGE_VERSION = ( + ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + + VERSION_SUFFIX +) +"""The canonical version of the volatility3 package""" diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index fe1361b30..0867b1608 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -50,7 +50,7 @@ class AutomagicInterface( context: interfaces.context.ContextInterface, config_path: str, *args, - **kwargs + **kwargs, ) -> None: super().__init__(context, config_path) for requirement in self.get_requirements(): From 0defe0f07e3801e4a833c05e37148aceb93fe305 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 21:08:22 +1000 Subject: [PATCH 040/263] Rename symbol_table to kernel_module_name --- volatility3/framework/plugins/linux/netfilter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 33bb5c456..2b6b67268 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -75,13 +75,13 @@ class AbstractNetfilter(ABC): ): self._context = context self._config = config - symbol_table = self._config["kernel"] - self.vmlinux = context.modules[symbol_table] + kernel_module_name = self._config["kernel"] + self.vmlinux = context.modules[kernel_module_name] self.layer_name = self.vmlinux.layer_name - modules = lsmod.Lsmod.list_modules(context, symbol_table) + modules = lsmod.Lsmod.list_modules(context, kernel_module_name) self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( - context, symbol_table, modules + context, kernel_module_name, modules ) self._set_data_sizes() From fefc6c4a4be2e1f12c81d04103d1e783303473b6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 21:27:27 +1000 Subject: [PATCH 041/263] Pass just the kernel module name instead of the whole config --- volatility3/framework/plugins/linux/netfilter.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 2b6b67268..c2af660b1 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -69,13 +69,9 @@ class AbstractNetfilter(ABC): NF_MAX_HOOKS = LARGEST_HOOK_NUMBER + 1 def __init__( - self, - context: interfaces.context.ContextInterface, - config: interfaces.configuration.HierarchicalDict, + self, context: interfaces.context.ContextInterface, kernel_module_name: str ): self._context = context - self._config = config - kernel_module_name = self._config["kernel"] self.vmlinux = context.modules[kernel_module_name] self.layer_name = self.vmlinux.layer_name @@ -106,7 +102,8 @@ class AbstractNetfilter(ABC): Yields: The kmsg records. Same as _run() """ - vmlinux = context.modules[config["kernel"]] + kernel_module_name = config["kernel"] + vmlinux = context.modules[kernel_module_name] implementation_inst = None # type: ignore for subclass in class_subclasses(cls): @@ -123,7 +120,9 @@ class AbstractNetfilter(ABC): "Netfilter implementation '%s' matches!", subclass.__name__, ) - implementation_inst = subclass(context=context, config=config) + implementation_inst = subclass( + context=context, kernel_module_name=kernel_module_name + ) # More than one class could be executed for an specific kernel version # For instance: Netfilter Ingress hooks yield from implementation_inst._run() From 0962f2638243517b2f5ff0a76423dfaeb7796071 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 21:33:59 +1000 Subject: [PATCH 042/263] class_subclasses is a function, soit's better import the volatility.framework to avoid people accidentally thinking it's defined here and then importing it from here. --- volatility3/framework/plugins/linux/netfilter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index c2af660b1..ad446123f 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -6,8 +6,8 @@ from abc import ABC, abstractmethod import logging from typing import Iterator, List, Tuple +from volatility3 import framework from volatility3.framework import ( - class_subclasses, constants, interfaces, renderers, @@ -106,7 +106,7 @@ class AbstractNetfilter(ABC): vmlinux = context.modules[kernel_module_name] implementation_inst = None # type: ignore - for subclass in class_subclasses(cls): + for subclass in framework.class_subclasses(cls): if not subclass.symtab_checks(vmlinux=vmlinux): vollog.log( constants.LOGLEVEL_VVVV, From 3793510f59adda53f25395bddd358a97e8397054 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 21:36:20 +1000 Subject: [PATCH 043/263] Add VersionRequirement for LinuxUtilities --- volatility3/framework/plugins/linux/netfilter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index ad446123f..b01ec25a7 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -674,6 +674,9 @@ class Netfilter(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) + ), ] def _format_fields(self, fields): From 9d5636b03a7cbee926f88954e58284de30d78f1e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 21:43:11 +1000 Subject: [PATCH 044/263] Move the data size setter to the __init__() --- volatility3/framework/plugins/linux/netfilter.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index b01ec25a7..d27106118 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -75,17 +75,15 @@ class AbstractNetfilter(ABC): self.vmlinux = context.modules[kernel_module_name] self.layer_name = self.vmlinux.layer_name + # Set data sizes + self.ptr_size = self.vmlinux.get_type("pointer").size + self.list_head_size = self.vmlinux.get_type("list_head").size + modules = lsmod.Lsmod.list_modules(context, kernel_module_name) self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( context, kernel_module_name, modules ) - self._set_data_sizes() - - def _set_data_sizes(self): - self.ptr_size = self.vmlinux.get_type("pointer").size - self.list_head_size = self.vmlinux.get_type("list_head").size - @classmethod def run_all( cls, From 23b9ed8c0ca6241db3b5cad6213337ee620c4b7a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 22:02:41 +1000 Subject: [PATCH 045/263] Extend the changes in fefc6c4a to the run_all() class method --- volatility3/framework/plugins/linux/netfilter.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index d27106118..60a71f798 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -86,21 +86,18 @@ class AbstractNetfilter(ABC): @classmethod def run_all( - cls, - context: interfaces.context.ContextInterface, - config: interfaces.configuration.HierarchicalDict, + cls, context: interfaces.context.ContextInterface, kernel_module_name: str ) -> Iterator[Tuple[int, str, str, int, int, str, bool]]: """It calls each subclass symtab_checks() to test the required conditions to that specific kernel implementation. Args: context: The volatility3 context on which to operate - config: Core configuration + kernel_module_name: The name of the table containing the kernel symbols Yields: The kmsg records. Same as _run() """ - kernel_module_name = config["kernel"] vmlinux = context.modules[kernel_module_name] implementation_inst = None # type: ignore @@ -698,8 +695,9 @@ class Netfilter(interfaces.plugins.PluginInterface): ) def _generator(self): + kernel_module_name = self.config["kernel"] for fields in AbstractNetfilter.run_all( - context=self.context, config=self.config + context=self.context, kernel_module_name=kernel_module_name ): yield (0, self._format_fields(fields)) From a72062f2889fdfe5d03e3974e4bc314a35e85700 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jun 2024 17:20:47 +1000 Subject: [PATCH 046/263] Revert "LayerWriter plugin: Fix --output argument. It's referenced in the code but never mentioned as a requirement" This reverts commit 6d22347ce6dd0152746564a2d77022ca1b4d9045. --- volatility3/framework/plugins/layerwriter.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 6a06cc607..319e4fd7a 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -38,11 +38,6 @@ class LayerWriter(plugins.PluginInterface): default=False, optional=True, ), - requirements.StringRequirement( - name="output", - description="Output filename", - optional=True, - ), requirements.ListRequirement( name="layers", element_type=str, From ab84070df30ddfaccc2e304e26fb7815a5555444 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jun 2024 17:25:05 +1000 Subject: [PATCH 047/263] Revert "Store the real/final output filename so that we can notify it correctly to the user" This reverts commit d7aae3a9828ba03bd5531aa385724a954e48501a. --- volatility3/cli/__init__.py | 4 ++-- volatility3/framework/plugins/layerwriter.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 9873791e3..c0eed20ee 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -790,8 +790,8 @@ class CommandLine: return None self._file.close() - self._output_filename = self._get_final_filename() - os.rename(self._name, self._output_filename) + output_filename = self._get_final_filename() + os.rename(self._name, output_filename) if direct: return CLIDirectFileHandler diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 319e4fd7a..61c3118b5 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -115,7 +115,6 @@ class LayerWriter(plugins.PluginInterface): progress_callback=self._progress_callback, ) file_handle.close() - output_name = file_handle._output_filename except IOError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) From 77fb0b7b26c2c52140380a2ce9f1a896a84c258a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jun 2024 18:37:38 +1000 Subject: [PATCH 048/263] Update the output filename (preferred_filename) so that we can notify it correctly to the user --- volatility3/cli/__init__.py | 10 +++++++++- volatility3/framework/plugins/layerwriter.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index c0eed20ee..6b17edac0 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -789,8 +789,16 @@ class CommandLine: if self._file.closed: return None - self._file.close() output_filename = self._get_final_filename() + + # Update the filename, which may have changed if a file with + # the same name already existed. This needs to be done before + # closing the file, otherwise FileHandlerInterface will raise + # an exception. Also, the preferred_filename setter only allows + # a specific set of characters, where '/' is not in that list + self.preferred_filename = os.path.basename(output_filename) + + self._file.close() os.rename(self._name, output_filename) if direct: diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 61c3118b5..24149a390 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -115,6 +115,10 @@ class LayerWriter(plugins.PluginInterface): progress_callback=self._progress_callback, ) file_handle.close() + + # Update the filename, which may have changed if a file + # with the same name already existed. + output_name = file_handle.preferred_filename except IOError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) From e4aac0c97634487db735f762bdc2f449fa8cd5ae Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 10 Jun 2024 11:30:15 +0100 Subject: [PATCH 049/263] Windows: ldrmodules update exception handling to InvalidAddressException --- .../framework/plugins/windows/ldrmodules.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 4c8456fa9..ffd84ea4a 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -1,3 +1,9 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -5,12 +11,14 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import pslist, vadinfo +vollog = logging.getLogger(__name__) + class LdrModules(interfaces.plugins.PluginInterface): """Lists the loaded modules in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -71,7 +79,11 @@ class LdrModules(interfaces.plugins.PluginInterface): # Filter out VADs that do not start with a MZ header if dos_header.e_magic != 0x5A4D: continue - except exceptions.PagedInvalidAddressException: + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping vad at {hex(dos_header.vol.offset)} due to InvalidAddressException", + ) continue mapped_files[vad.get_start()] = vad.get_file_name() From 0ee3573be239966a888df644d2aca3a5dd18b02d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jun 2024 23:14:45 +0100 Subject: [PATCH 050/263] Revert "Intel layer: Fix. This if statement will never be executed unless "minimum_address > maximum_address" which doesn't make sense to me." This reverts commit e5a5b895771b655d21c36689c33a534034c31e36. --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 8589cdbb6..75e561b33 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -180,7 +180,7 @@ class Intel(linear.LinearlyMappedLayer): position = self._initial_position entry = self._initial_entry - if not (self.minimum_address <= offset <= self.maximum_address): + if self.minimum_address > offset > self.maximum_address: raise exceptions.PagedInvalidAddressException( self.name, offset, From 04ca0214eb264268bd368401f8d5eaa940f2524e Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 13 Jun 2024 15:24:32 -0500 Subject: [PATCH 051/263] Prevent backtrace in ADS scanning when contents cannot be recovered. Fix missing parantheses as well as format_hints call on bad value --- volatility3/framework/plugins/windows/mftscan.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 85c072037..1b1a1b45e 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -264,9 +264,10 @@ class ADS(interfaces.plugins.PluginInterface): disasm = interfaces.renderers.Disassembly( content, 0, architecture.lower() ) + content = format_hints.HexBytes(content) else: - content = renderers.NotAvailableValue - disasm = interfaces.renderers.BaseAbsentValue + content = renderers.NotAvailableValue() + disasm = interfaces.renderers.BaseAbsentValue() yield 0, ( format_hints.Hex(attr_data.vol.offset), @@ -275,7 +276,7 @@ class ADS(interfaces.plugins.PluginInterface): attr.Attr_Header.AttrType.lookup(), file_name, ads_name, - format_hints.HexBytes(content), + content, disasm, ) else: From 59f6a050688c30e2b05fec2b0efe54f57e2f46f3 Mon Sep 17 00:00:00 2001 From: atcuno Date: Fri, 14 Jun 2024 16:45:54 -0500 Subject: [PATCH 052/263] The notes variable is not reset for each VAD, allowing bleed through of a previously set note value to VADs enumerated afterwards --- volatility3/framework/plugins/windows/malfind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 8c0cb68ac..079274d69 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -155,12 +155,12 @@ class Malfind(interfaces.plugins.PluginInterface): for proc in procs: # by default, "Notes" column will be set to N/A - notes = renderers.NotApplicableValue() process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( self.context, kernel.layer_name, kernel.symbol_table_name, proc ): + notes = renderers.NotApplicableValue() # Check for unique headers and update "Notes" column if criteria is met if data[0:2] in refined_criteria: notes = refined_criteria[data[0:2]] From aee81276a2d921e44e58db28174f7e4782fe7423 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 11:27:32 -0500 Subject: [PATCH 053/263] Add processghosting plugin to detect process ghosting and related techniques. Add process filter that gathers only active, non-smeared userland processes --- .../plugins/windows/processghosting.py | 99 +++++++++++++++++++ .../framework/plugins/windows/pslist.py | 21 ++++ 2 files changed, 120 insertions(+) create mode 100644 volatility3/framework/plugins/windows/processghosting.py diff --git a/volatility3/framework/plugins/windows/processghosting.py b/volatility3/framework/plugins/windows/processghosting.py new file mode 100644 index 000000000..6c311f555 --- /dev/null +++ b/volatility3/framework/plugins/windows/processghosting.py @@ -0,0 +1,99 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging + +from volatility3.framework import interfaces, exceptions +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + + +class ProcessGhosting(interfaces.plugins.PluginInterface): + """Lists processes whose DeletePending bit is set or whose FILE_OBJECT is set to 0""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + ] + + def _generator(self, procs): + # determine if we're on a 32 or 64 bit kernel + kernel = self.context.modules[self.config["kernel"]] + + if not kernel.get_type("_EPROCESS").has_member("ImageFilePointer"): + vollog.warning("This plugin only supports Windows 10 builds when the ImageFilePointer member of _EPROCESS is present") + return + + for proc in procs: + delete_pending = renderers.UnreadableValue() + process_name = utility.array_to_string(proc.ImageFileName) + + # if it is 0 then its a side effect of process ghosting + if proc.ImageFilePointer.vol.offset != 0: + try: + file_object = proc.ImageFilePointer + delete_pending = file_object.DeletePending + except exceptions.InvalidAddressException: + file_object = 0 + + # ImageFilePointer equal to 0 means process ghosting or similar techniques were used + else: + file_object = 0 + + # delete_pending besides 0 or 1 = smear + if file_object == 0 or delete_pending == 1: + path = renderers.UnreadableValue() + if file_object: + try: + path = file_object.FileName.String + except exceptions.InvalidAddressException: + path = renderers.UnreadableValue() + + yield ( + 0, + ( + proc.UniqueProcessId, + process_name, + format_hints.Hex(file_object), + delete_pending, + path + ), + ) + + def run(self): + filter_func = pslist.PsList.create_active_process_filter() + kernel = self.context.modules[self.config["kernel"]] + + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("FILE_OBJECT", format_hints.Hex), + ("DeletePending", str), + ("Path", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index e7a0d5dd4..8ad0f9a05 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -136,6 +136,27 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): filter_func = lambda x: x.UniqueProcessId not in filter_list return filter_func + @classmethod + def create_active_process_filter( + cls + ) -> Callable[[interfaces.objects.ObjectInterface], bool]: + """A factory for producing a filter function that only returns + active, userland processes. This prevents plugins from operating on terminated + processes that are still in the process list due to smear or handle leaks as well + as kernel processes (System, Registry, etc.). Use of this filter for plugins searching + for system state anomalies significantly reduces false positive in smeared and terminated + processes. + Returns: + Filter function for passing to the `list_processes` method + """ + + return lambda x: not (x.is_valid() and \ + x.ActiveThreads > 0 and \ + x.UniqueProcessId != 4 and \ + x.InheritedFromUniqueProcessId != 4 and \ + x.ExitTime.QuadPart == 0 and \ + x.get_handle_count() != renderers.UnreadableValue()) + @classmethod def create_name_filter( cls, name_list: List[str] = None, exclude: bool = False From 6c42c51a8a6201255f120321885309cd92611c77 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 11:31:43 -0500 Subject: [PATCH 054/263] Fixes for black --- .../framework/plugins/windows/processghosting.py | 7 ++++--- volatility3/framework/plugins/windows/pslist.py | 16 +++++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/processghosting.py b/volatility3/framework/plugins/windows/processghosting.py index 6c311f555..b29ff04f0 100644 --- a/volatility3/framework/plugins/windows/processghosting.py +++ b/volatility3/framework/plugins/windows/processghosting.py @@ -33,11 +33,12 @@ class ProcessGhosting(interfaces.plugins.PluginInterface): ] def _generator(self, procs): - # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] if not kernel.get_type("_EPROCESS").has_member("ImageFilePointer"): - vollog.warning("This plugin only supports Windows 10 builds when the ImageFilePointer member of _EPROCESS is present") + vollog.warning( + "This plugin only supports Windows 10 builds when the ImageFilePointer member of _EPROCESS is present" + ) return for proc in procs: @@ -72,7 +73,7 @@ class ProcessGhosting(interfaces.plugins.PluginInterface): process_name, format_hints.Hex(file_object), delete_pending, - path + path, ), ) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 8ad0f9a05..55a3fc280 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -138,7 +138,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def create_active_process_filter( - cls + cls, ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing a filter function that only returns active, userland processes. This prevents plugins from operating on terminated @@ -150,12 +150,14 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Filter function for passing to the `list_processes` method """ - return lambda x: not (x.is_valid() and \ - x.ActiveThreads > 0 and \ - x.UniqueProcessId != 4 and \ - x.InheritedFromUniqueProcessId != 4 and \ - x.ExitTime.QuadPart == 0 and \ - x.get_handle_count() != renderers.UnreadableValue()) + return lambda x: not ( + x.is_valid() and + x.ActiveThreads > 0 and + x.UniqueProcessId != 4 and + x.InheritedFromUniqueProcessId != 4 and + x.ExitTime.QuadPart == 0 and + x.get_handle_count() != renderers.UnreadableValue() + ) @classmethod def create_name_filter( From 4e15019c46ae24899ea0527dd4855ade5fd7af00 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 11:34:06 -0500 Subject: [PATCH 055/263] Fixes for black --- volatility3/framework/plugins/windows/pslist.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 55a3fc280..8234b210f 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -151,12 +151,12 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ return lambda x: not ( - x.is_valid() and - x.ActiveThreads > 0 and - x.UniqueProcessId != 4 and - x.InheritedFromUniqueProcessId != 4 and - x.ExitTime.QuadPart == 0 and - x.get_handle_count() != renderers.UnreadableValue() + x.is_valid() + and x.ActiveThreads > 0 + and x.UniqueProcessId != 4 + and x.InheritedFromUniqueProcessId != 4 + and x.ExitTime.QuadPart == 0 + and x.get_handle_count() != renderers.UnreadableValue() ) @classmethod From c4e7e50180a45dfb80b453b61e9b94860d5a01d3 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 14:42:23 -0500 Subject: [PATCH 056/263] Add svclist and svcdiff plugins. Make svcscan more modular to support inheritance and cleaner code --- .../framework/plugins/windows/svcdiff.py | 74 ++++++++++++ .../framework/plugins/windows/svclist.py | 86 ++++++++++++++ .../framework/plugins/windows/svcscan.py | 111 +++++++++++------- .../framework/symbols/windows/versions.py | 9 ++ 4 files changed, 235 insertions(+), 45 deletions(-) create mode 100644 volatility3/framework/plugins/windows/svcdiff.py create mode 100644 volatility3/framework/plugins/windows/svclist.py diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py new file mode 100644 index 000000000..03f0afcd3 --- /dev/null +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -0,0 +1,74 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +# This module attempts to locate skeleton-key like function hooks. +# It does this by locating the CSystems array through a variety of methods, +# and then validating the entry for RC4 HMAC (0x17 / 23) +# +# For a thorough walkthrough on how the R&D was performed to develop this plugin, +# please see our blogpost here: +# +# https://volatility-labs.blogspot.com/2021/10/memory-forensics-r-illustrated.html + +import logging + +from volatility3.framework import symbols +from volatility3.framework.configuration import requirements +from volatility3.plugins.windows import svclist, svcscan +from volatility3.framework.symbols.windows import versions + +vollog = logging.getLogger(__name__) + +class SvcDiff(svclist.SvcList, svcscan.SvcScan): + """Compares services found through list walking versus scanning to find rootkits""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="svclist", component=svclist.SvcList, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="svcscan", component=svcscan.SvcScan, version=(2, 0, 0) + ), + ] + + def _generator(self): + """ + Finds services by walking the services.exe list on supported Windows 10 versions + """ + kernel = self.context.modules[self.config["kernel"]] + + if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) or \ + not versions.is_win10_15063_or_later(context=self.context, symbol_table=kernel.symbol_table_name): + vollog.info("This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples") + return + + from_scan = set() + from_list = set() + records = {} + + service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + + # collect unique service names from scanning + for service in self.service_scan(service_table_name, service_binary_dll_map, filter_func): + from_scan.add(service[6]) + records[service[6]] = service + + # collect services from listing walking + for service in self.service_list(service_table_name, service_binary_dll_map, filter_func): + from_list.add(service[6]) + + # report services found from scanning but not list walking + for hidden_service in from_scan-from_list: + yield (0, records[hidden_service]) + diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py new file mode 100644 index 000000000..b4541981d --- /dev/null +++ b/volatility3/framework/plugins/windows/svclist.py @@ -0,0 +1,86 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import List + +from volatility3.framework import interfaces, exceptions, symbols +from volatility3.framework.configuration import requirements +from volatility3.framework.symbols.windows import versions +from volatility3.plugins.windows import svcscan, pslist +from volatility3.framework.layers import scanners + +vollog = logging.getLogger(__name__) + + +class SvcList(svcscan.SvcScan): + """Lists services contained with the services.exe doubly linked list of services""" + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.PluginRequirement( + name="svcscan", plugin=svcscan.SvcScan, version=(2, 0, 0) + ), + ] + + def _get_exe_range(self, proc): + """ + Returns a tuple of starting,ending address for + the VAD containing services.exe + """ + + vad_root = proc.get_vad_root() + for vad in vad_root.traverse(): + filename = vad.get_file_name() + if isinstance(filename, str) and filename.lower().endswith("\\services.exe"): + return [(vad.get_start(), vad.get_size())] + + return None + + def service_list(self, service_table_name, service_binary_dll_map, filter_func): + kernel = self.context.modules[self.config["kernel"]] + + if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) or \ + not versions.is_win10_15063_or_later(context=self.context, symbol_table=kernel.symbol_table_name): + vollog.info("This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples") + return + + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ): + try: + layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + vollog.warning("Unable to access memory of services.exe running with PID: {}".format(proc.UniqueProcessId)) + continue + + layer = self.context.layers[layer_name] + + exe_range = self._get_exe_range(proc) + if not exe_range: + vollog.warning("Could not find the application executable VAD for services.exe. Unable to proceed.") + continue + + for offset in layer.scan( + context=self.context, + scanner=scanners.BytesScanner(needle = b"Sc27"), + sections=exe_range, + ): + for record in self.enumerate_vista_or_later_header(service_table_name, service_binary_dll_map, layer_name, offset): + yield record + + def _generator(self): + service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + + for record in self.service_list(service_table_name, service_binary_dll_map, filter_func): + yield (0, record) + diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 10de46e2a..08d11a946 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -19,7 +19,7 @@ from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions -from volatility3.framework.symbols.windows.extensions import services +from volatility3.framework.symbols.windows.extensions import services as services_types from volatility3.plugins.windows import poolscanner, pslist, vadyarascan from volatility3.plugins.windows.registry import hivelist @@ -140,7 +140,7 @@ class SvcScan(interfaces.plugins.PluginInterface): config_path, os.path.join("windows", "services"), symbol_filename, - class_types=services.class_types, + class_types=services_types.class_types, native_types=native_types, ) @@ -232,28 +232,44 @@ class SvcScan(interfaces.plugins.PluginInterface): for service_key in services } - def _generator(self): + def enumerate_vista_or_later_header( + self, + service_table_name, + service_binary_dll_map, + proc_layer_name, + offset + ): + if offset % 8: + return + + service_header = self.context.object( + service_table_name + constants.BANG + "_SERVICE_HEADER", + offset=offset, + layer_name=proc_layer_name, + ) + + if not service_header.is_valid(): + return + + # since we walk the s-list backwards, if we've seen + # an object, then we've also seen all objects that + # exist before it, thus we can break at that time. + for service_record in service_header.ServiceRecord.traverse(): + service_info = service_binary_dll_map.get( + service_record.get_name(), + ServiceBinaryInfo( + renderers.UnreadableValue(), renderers.UnreadableValue() + ), + ) + yield self.get_record_tuple(service_record, service_info) + + def service_scan(self, service_table_name, service_binary_dll_map, filter_func): kernel = self.context.modules[self.config["kernel"]] - service_table_name = self.create_service_table( - self.context, kernel.symbol_table_name, self.config_path - ) - - # Building the dictionary ahead of time is much better for performance - # vs looking up each service's DLL individually. - services_key = self._get_service_key(kernel) - service_binary_dll_map = ( - self._get_service_binary_map(services_key) - if services_key is not None - else {} - ) - relative_tag_offset = self.context.symbol_space.get_type( service_table_name + constants.BANG + "_SERVICE_RECORD" ).relative_child_offset("Tag") - filter_func = pslist.PsList.create_name_filter(["services.exe"]) - is_vista_or_later = versions.is_vista_or_later( context=self.context, symbol_table=kernel.symbol_table_name ) @@ -306,37 +322,42 @@ class SvcScan(interfaces.plugins.PluginInterface): renderers.UnreadableValue(), renderers.UnreadableValue() ), ) - yield ( - 0, - self.get_record_tuple(service_record, service_info), - ) + yield self.get_record_tuple(service_record, service_info) else: - service_header = self.context.object( - service_table_name + constants.BANG + "_SERVICE_HEADER", - offset=offset, - layer_name=proc_layer_name, - ) - - if not service_header.is_valid(): - continue - - # since we walk the s-list backwards, if we've seen - # an object, then we've also seen all objects that - # exist before it, thus we can break at that time. - for service_record in service_header.ServiceRecord.traverse(): + for service_record in self.enumerate_vista_or_later_header(service_table_name, service_binary_dll_map, proc_layer_name, offset): if service_record in seen: break seen.append(service_record) - service_info = service_binary_dll_map.get( - service_record.get_name(), - ServiceBinaryInfo( - renderers.UnreadableValue(), renderers.UnreadableValue() - ), - ) - yield ( - 0, - self.get_record_tuple(service_record, service_info), - ) + yield service_record + + + def get_prereq_info(self): + """ + Data structures and information needed to analyze service information + """ + kernel = self.context.modules[self.config["kernel"]] + + service_table_name = self.create_service_table( + self.context, kernel.symbol_table_name, self.config_path + ) + + services_key = self._get_service_key(kernel) + + service_binary_dll_map = ( + self._get_service_binary_map(services_key) + if services_key is not None + else {} + ) + + filter_func = pslist.PsList.create_name_filter(["services.exe"]) + + return service_table_name, service_binary_dll_map, filter_func + + def _generator(self): + service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + + for record in self.service_scan(service_table_name, service_binary_dll_map, filter_func): + yield (0, record) def run(self): return renderers.TreeGrid( diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index e1e74afc0..d8964f575 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -141,6 +141,15 @@ is_win10_15063 = OsDistinguisher( ], ) +is_win10_15063_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 15063), + fallback_checks=[ + ("ObHeaderCookie", None, True), + ("_HANDLE_TABLE", "HandleCount", False), + ("_EPROCESS", "KeepAliveCounter", False), + ], +) + is_win10_16299_or_later = OsDistinguisher( version_check=lambda x: x >= (10, 0, 16299), fallback_checks=[ From 91184c7d92f84cdd3371d3a2371cf1784a2eb1a2 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 14:54:20 -0500 Subject: [PATCH 057/263] Format fixes --- .../framework/plugins/windows/svcdiff.py | 23 ++++++++---- .../framework/plugins/windows/svclist.py | 36 +++++++++++++------ .../framework/plugins/windows/svcscan.py | 18 +++++----- 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py index 03f0afcd3..71aa03636 100644 --- a/volatility3/framework/plugins/windows/svcdiff.py +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -48,27 +48,36 @@ class SvcDiff(svclist.SvcList, svcscan.SvcScan): """ kernel = self.context.modules[self.config["kernel"]] - if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) or \ - not versions.is_win10_15063_or_later(context=self.context, symbol_table=kernel.symbol_table_name): - vollog.info("This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples") + if not symbols.symbol_table_is_64bit( + self.context, kernel.symbol_table_name + ) or not versions.is_win10_15063_or_later( + context=self.context, symbol_table=kernel.symbol_table_name + ): + vollog.info( + "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" + ) return from_scan = set() from_list = set() records = {} - + service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() # collect unique service names from scanning - for service in self.service_scan(service_table_name, service_binary_dll_map, filter_func): + for service in self.service_scan( + service_table_name, service_binary_dll_map, filter_func + ): from_scan.add(service[6]) records[service[6]] = service # collect services from listing walking - for service in self.service_list(service_table_name, service_binary_dll_map, filter_func): + for service in self.service_list( + service_table_name, service_binary_dll_map, filter_func + ): from_list.add(service[6]) # report services found from scanning but not list walking - for hidden_service in from_scan-from_list: + for hidden_service in from_scan - from_list: yield (0, records[hidden_service]) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index b4541981d..53ca68da7 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -6,7 +6,7 @@ import logging from typing import List -from volatility3.framework import interfaces, exceptions, symbols +from volatility3.framework import interfaces, exceptions, symbols from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows import versions from volatility3.plugins.windows import svcscan, pslist @@ -38,7 +38,9 @@ class SvcList(svcscan.SvcScan): vad_root = proc.get_vad_root() for vad in vad_root.traverse(): filename = vad.get_file_name() - if isinstance(filename, str) and filename.lower().endswith("\\services.exe"): + if isinstance(filename, str) and filename.lower().endswith( + "\\services.exe" + ): return [(vad.get_start(), vad.get_size())] return None @@ -46,9 +48,14 @@ class SvcList(svcscan.SvcScan): def service_list(self, service_table_name, service_binary_dll_map, filter_func): kernel = self.context.modules[self.config["kernel"]] - if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) or \ - not versions.is_win10_15063_or_later(context=self.context, symbol_table=kernel.symbol_table_name): - vollog.info("This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples") + if not symbols.symbol_table_is_64bit( + self.context, kernel.symbol_table_name + ) or not versions.is_win10_15063_or_later( + context=self.context, symbol_table=kernel.symbol_table_name + ): + vollog.info( + "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" + ) return for proc in pslist.PsList.list_processes( @@ -60,27 +67,34 @@ class SvcList(svcscan.SvcScan): try: layer_name = proc.add_process_layer() except exceptions.InvalidAddressException: - vollog.warning("Unable to access memory of services.exe running with PID: {}".format(proc.UniqueProcessId)) + vollog.warning( + "Unable to access memory of services.exe running with PID: {}".format( + proc.UniqueProcessId + ) + ) continue layer = self.context.layers[layer_name] exe_range = self._get_exe_range(proc) if not exe_range: - vollog.warning("Could not find the application executable VAD for services.exe. Unable to proceed.") + vollog.warning( + "Could not find the application executable VAD for services.exe. Unable to proceed." + ) continue for offset in layer.scan( context=self.context, - scanner=scanners.BytesScanner(needle = b"Sc27"), + scanner=scanners.BytesScanner(needle=b"Sc27"), sections=exe_range, + ): + for record in self.enumerate_vista_or_later_header( + service_table_name, service_binary_dll_map, layer_name, offset ): - for record in self.enumerate_vista_or_later_header(service_table_name, service_binary_dll_map, layer_name, offset): yield record def _generator(self): service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - + for record in self.service_list(service_table_name, service_binary_dll_map, filter_func): yield (0, record) - diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 08d11a946..c991ec943 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -233,11 +233,7 @@ class SvcScan(interfaces.plugins.PluginInterface): } def enumerate_vista_or_later_header( - self, - service_table_name, - service_binary_dll_map, - proc_layer_name, - offset + self, service_table_name, service_binary_dll_map, proc_layer_name, offset ): if offset % 8: return @@ -324,13 +320,17 @@ class SvcScan(interfaces.plugins.PluginInterface): ) yield self.get_record_tuple(service_record, service_info) else: - for service_record in self.enumerate_vista_or_later_header(service_table_name, service_binary_dll_map, proc_layer_name, offset): + for service_record in self.enumerate_vista_or_later_header( + service_table_name, + service_binary_dll_map, + proc_layer_name, + offset + ): if service_record in seen: break seen.append(service_record) yield service_record - def get_prereq_info(self): """ Data structures and information needed to analyze service information @@ -356,7 +356,9 @@ class SvcScan(interfaces.plugins.PluginInterface): def _generator(self): service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - for record in self.service_scan(service_table_name, service_binary_dll_map, filter_func): + for record in self.service_scan( + service_table_name, service_binary_dll_map, filter_func + ): yield (0, record) def run(self): From d42fb0a1451ca0a0cc00ee4a2a431be092c193b9 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 14:56:57 -0500 Subject: [PATCH 058/263] Format fixes --- volatility3/framework/plugins/windows/svclist.py | 4 +++- volatility3/framework/plugins/windows/svcscan.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index 53ca68da7..1938dd182 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -96,5 +96,7 @@ class SvcList(svcscan.SvcScan): def _generator(self): service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - for record in self.service_list(service_table_name, service_binary_dll_map, filter_func): + for record in self.service_list( + service_table_name, service_binary_dll_map, filter_func + ): yield (0, record) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index c991ec943..07274f03d 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -324,7 +324,7 @@ class SvcScan(interfaces.plugins.PluginInterface): service_table_name, service_binary_dll_map, proc_layer_name, - offset + offset, ): if service_record in seen: break From f6a053d7b3db2421c9d4d8501eff1fcb78001f1f Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 14:58:05 -0500 Subject: [PATCH 059/263] Format fixes --- volatility3/framework/plugins/windows/svcdiff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py index 71aa03636..809064946 100644 --- a/volatility3/framework/plugins/windows/svcdiff.py +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -20,6 +20,7 @@ from volatility3.framework.symbols.windows import versions vollog = logging.getLogger(__name__) + class SvcDiff(svclist.SvcList, svcscan.SvcScan): """Compares services found through list walking versus scanning to find rootkits""" @@ -80,4 +81,3 @@ class SvcDiff(svclist.SvcList, svcscan.SvcScan): # report services found from scanning but not list walking for hidden_service in from_scan - from_list: yield (0, records[hidden_service]) - From 3d1b9ef2bc1e968eed6f41195834752d2d328ec9 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 20:09:05 -0500 Subject: [PATCH 060/263] Add new plugin to detect hollowed processes using a variety of techniques and allowing for easy additions of future detection techniques --- .../plugins/windows/hollowprocesses.py | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 volatility3/framework/plugins/windows/hollowprocesses.py diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py new file mode 100644 index 000000000..a832e89e1 --- /dev/null +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -0,0 +1,175 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import NamedTuple + +from volatility3.framework import interfaces, exceptions, constants +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.plugins.windows import pslist, vadinfo + +vollog = logging.getLogger(__name__) + +VadInfo = NamedTuple( + "VadInfo", + [ + ("protection", str), + ("path", str), + ], +) + +DLLInfo = NamedTuple( + "DLLInfo", + [ + ("path", str), + ], +) + +class HollowProcesses(interfaces.plugins.PluginInterface): + """Lists hollowed processes""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + ] + + def _get_vads_map(self, proc): + vads = {} + + kernel = self.context.modules[self.config["kernel"]] + + for vad in proc.get_vad_root().traverse(): + protection_string = vad.get_protection( + vadinfo.VadInfo.protect_values( + self.context, kernel.layer_name, kernel.symbol_table_name + ), + vadinfo.winnt_protections, + ) + + fn = vad.get_file_name() + if not fn or not isinstance(fn, str): + fn = "" + + vads[vad.get_start()] = VadInfo(protection_string, fn) + + return vads + + def _get_dlls_map(self, proc): + dlls = {} + + for entry in proc.load_order_modules(): + try: + base = entry.DllBase + except exceptions.InvalidAddressException: + continue + + try: + FullDllName = entry.FullDllName.get_string() + except exceptions.InvalidAddressException: + FullDllName = renderers.UnreadableValue() + + dlls[base] = DLLInfo(FullDllName) + + return dlls + + def _get_image_base(self, proc): + kernel = self.context.modules[self.config["kernel"]] + + try: + proc_layer_name = proc.add_process_layer() + peb = self.context.object( + kernel.symbol_table_name + constants.BANG + "_PEB", + layer_name=proc_layer_name, + offset=proc.Peb, + ) + return peb.ImageBaseAddress + except exceptions.InvalidAddressException: + return None + + def _check_load_address(self, proc, _, __): + image_base = self._get_image_base(proc) + if image_base is not None and image_base != proc.SectionBaseAddress: + yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format(image_base, proc.SectionBaseAddress) + + def _check_exe_protection(self, proc, vads, __): + base = proc.SectionBaseAddress + + if base not in vads: + yield "There is no VAD starting at the base address of the process executable ({:#x})".format(base) + elif vads[base].protection != "PAGE_EXECUTE_WRITECOPY": + yield "Unexpected protection ({}) for VAD hosting the process executable ({:#x}) with path {}".format(vads[base].protection, base, vads[base].path) + + def _check_dlls_protection(self, _, vads, dlls): + for dll_base in dlls: + # could be malicious but triggers too many FPs from smear + if dll_base not in vads: + continue + + if vads[dll_base].protection != "PAGE_EXECUTE_WRITECOPY": + yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format(vads[dll_base].protection, dll_base, dlls[dll_base].path) + + def _generator(self, procs): + checks = [self._check_load_address, self._check_exe_protection, self._check_dlls_protection] + + for proc in procs: + proc_name = utility.array_to_string(proc.ImageFileName) + pid = proc.UniqueProcessId + + # smear and/or terminated process + dlls = self._get_dlls_map(proc) + if len(dlls) < 3: + continue + + vads = self._get_vads_map(proc) + if len(vads) < 5: + continue + + for check in checks: + for note in check(proc, vads, dlls): + yield 0, ( + pid, + proc_name, + note, + ) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] + + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Notes", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) From 02bda980d429a49c18a1b908b5fcbfda8de5ad09 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 20:12:07 -0500 Subject: [PATCH 061/263] formatting fixes --- .../plugins/windows/hollowprocesses.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index a832e89e1..18fb24c8f 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging @@ -17,14 +17,14 @@ VadInfo = NamedTuple( [ ("protection", str), ("path", str), - ], + ], ) DLLInfo = NamedTuple( "DLLInfo", [ ("path", str), - ], + ], ) class HollowProcesses(interfaces.plugins.PluginInterface): @@ -111,15 +111,21 @@ class HollowProcesses(interfaces.plugins.PluginInterface): def _check_load_address(self, proc, _, __): image_base = self._get_image_base(proc) if image_base is not None and image_base != proc.SectionBaseAddress: - yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format(image_base, proc.SectionBaseAddress) + yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format( + image_base, proc.SectionBaseAddress + ) def _check_exe_protection(self, proc, vads, __): base = proc.SectionBaseAddress if base not in vads: - yield "There is no VAD starting at the base address of the process executable ({:#x})".format(base) + yield "There is no VAD starting at the base address of the process executable ({:#x})".format( + base + ) elif vads[base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for VAD hosting the process executable ({:#x}) with path {}".format(vads[base].protection, base, vads[base].path) + yield "Unexpected protection ({}) for VAD hosting the process executable ({:#x}) with path {}".format( + vads[base].protection, base, vads[base].path + ) def _check_dlls_protection(self, _, vads, dlls): for dll_base in dlls: @@ -128,10 +134,16 @@ class HollowProcesses(interfaces.plugins.PluginInterface): continue if vads[dll_base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format(vads[dll_base].protection, dll_base, dlls[dll_base].path) + yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format( + vads[dll_base].protection, dll_base, dlls[dll_base].path + ) def _generator(self, procs): - checks = [self._check_load_address, self._check_exe_protection, self._check_dlls_protection] + checks = [ + self._check_load_address, + self._check_exe_protection, + self._check_dlls_protection + ] for proc in procs: proc_name = utility.array_to_string(proc.ImageFileName) From 178f7c45f7ed1413277baba486929e5f2afae7fc Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 20:13:04 -0500 Subject: [PATCH 062/263] formatting fixes --- volatility3/framework/plugins/windows/hollowprocesses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index 18fb24c8f..9d7e800b7 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -27,6 +27,7 @@ DLLInfo = NamedTuple( ], ) + class HollowProcesses(interfaces.plugins.PluginInterface): """Lists hollowed processes""" @@ -142,7 +143,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): checks = [ self._check_load_address, self._check_exe_protection, - self._check_dlls_protection + self._check_dlls_protection, ] for proc in procs: From a68a4826329dbd44e95d4e173ea467be24dbc99b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 20 Jun 2024 04:00:20 +0200 Subject: [PATCH 063/263] fix #509 and simplify logic in _load_segments --- volatility3/framework/layers/crash.py | 105 +++++++++++++++----------- 1 file changed, 62 insertions(+), 43 deletions(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 042b18ddc..2d0f37008 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -96,12 +96,13 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): def get_summary_header(self) -> interfaces.objects.ObjectInterface: return self.context.object( self._crash_common_table_name + constants.BANG + "_SUMMARY_DUMP", - offset=0x1000 * self.headerpages, + offset=self._page_size * self.headerpages, layer_name=self._base_layer, ) def _load_segments(self) -> None: - """Loads up the segments from the meta_layer.""" + """Loads up the segments from the meta_layer. + A segment is a set of contiguous memory pages.""" segments = [] @@ -119,70 +120,88 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): for run in header.PhysicalMemoryBlockBuffer.Run: segments.append( ( - run.BasePage * 0x1000, - offset * 0x1000, - run.PageCount * 0x1000, - run.PageCount * 0x1000, + run.BasePage * self._page_size, + offset * self._page_size, + run.PageCount * self._page_size, + run.PageCount * self._page_size, ) ) offset += run.PageCount elif self.dump_type == 0x05: summary_header = self.get_summary_header() - first_bit = None # First bit in a run - first_offset = 0 # File offset of first bit - last_bit_seen = 0 # Most recent bit processed - offset = summary_header.HeaderSize # Size of file headers - buffer_char = summary_header.get_buffer_char() - buffer_long = summary_header.get_buffer_long() - - for outer_index in range(0, ((summary_header.BitmapSize + 31) // 32)): - if buffer_long[outer_index] == 0: - if first_bit is not None: - last_bit = ((outer_index - 1) * 32) + 31 - segment_length = (last_bit - first_bit + 1) * 0x1000 + seg_first_bit = None # First bit in a run + seg_first_offset = 0 # File offset of first bit + offset = ( + summary_header.HeaderSize + ) # Offset to the start of actual memory dump + ulong_bitmap_array = summary_header.get_buffer_long() + # outer_index points to a 32 bits array inside a list of arrays, + # each bit indicating a page mapping state + for outer_index in range(0, ulong_bitmap_array.vol.count): + ulong_bitmap = ulong_bitmap_array[outer_index] + # All pages in this 32 bits array are mapped (speedup iteration process) + if ulong_bitmap == 0xFFFFFFFF: + # New segment + if seg_first_bit is None: + seg_first_offset = offset + seg_first_bit = outer_index * 32 + offset += 32 * self._page_size + # No pages in this 32 bits array are mapped (speedup iteration process) + elif ulong_bitmap == 0: + # End of segment + if seg_first_bit is not None: + last_bit = (outer_index - 1) * 32 + 31 + segment_length = ( + last_bit - seg_first_bit + 1 + ) * self._page_size segments.append( ( - first_bit * 0x1000, - first_offset, + seg_first_bit * self._page_size, + seg_first_offset, segment_length, segment_length, ) ) - first_bit = None - elif buffer_long[outer_index] == 0xFFFFFFFF: - if first_bit is None: - first_offset = offset - first_bit = outer_index * 32 - offset = offset + (32 * 0x1000) + seg_first_bit = None + # Some pages in this 32 bits array are mapped and some aren't else: - for inner_index in range(0, 32): - bit_addr = outer_index * 32 + inner_index - if (buffer_char[bit_addr >> 3] >> (bit_addr & 0x7)) & 1: - if first_bit is None: - first_offset = offset - first_bit = bit_addr - offset = offset + 0x1000 + for inner_bit_position in range(0, 32): + current_bit = outer_index * 32 + inner_bit_position + page_mapped = ulong_bitmap & (1 << inner_bit_position) + if page_mapped: + # New segment + if seg_first_bit is None: + seg_first_offset = offset + seg_first_bit = current_bit + offset += self._page_size else: - if first_bit is not None: + # End of segment + if seg_first_bit is not None: segment_length = ( - (bit_addr - 1) - first_bit + 1 - ) * 0x1000 + current_bit - 1 - seg_first_bit + 1 + ) * self._page_size segments.append( ( - first_bit * 0x1000, - first_offset, + seg_first_bit * self._page_size, + seg_first_offset, segment_length, segment_length, ) ) - first_bit = None - last_bit_seen = (outer_index * 32) + 31 + seg_first_bit = None + else: + last_bit_seen = outer_index * 32 + 31 - if first_bit is not None: - segment_length = (last_bit_seen - first_bit + 1) * 0x1000 + if seg_first_bit is not None: + segment_length = (last_bit_seen - seg_first_bit + 1) * self._page_size segments.append( - (first_bit * 0x1000, first_offset, segment_length, segment_length) + ( + seg_first_bit * self._page_size, + seg_first_offset, + segment_length, + segment_length, + ) ) else: vollog.log( From c688744f66ae6af49123a530dd45abc29ed9abba Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 20 Jun 2024 04:16:32 +0200 Subject: [PATCH 064/263] revert useless for-else --- volatility3/framework/layers/crash.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 2d0f37008..3cfc0a25b 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -190,7 +190,6 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): ) ) seg_first_bit = None - else: last_bit_seen = outer_index * 32 + 31 if seg_first_bit is not None: From ed140e9ac451ab3f83f39f9788c344bfa658cf18 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 20 Jun 2024 13:06:43 -0500 Subject: [PATCH 065/263] placeholder --- .../plugins/windows/suspicious_threads.py | 201 ++++++++++++++++++ .../framework/plugins/windows/thrdscan.py | 59 +++-- .../framework/plugins/windows/threads.py | 80 +++++++ 3 files changed, 310 insertions(+), 30 deletions(-) create mode 100644 volatility3/framework/plugins/windows/suspicious_threads.py create mode 100644 volatility3/framework/plugins/windows/threads.py diff --git a/volatility3/framework/plugins/windows/suspicious_threads.py b/volatility3/framework/plugins/windows/suspicious_threads.py new file mode 100644 index 000000000..b24f3f2c2 --- /dev/null +++ b/volatility3/framework/plugins/windows/suspicious_threads.py @@ -0,0 +1,201 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List +from volatility3.framework import renderers, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist, threads, vadinfo, thrdscan + +vollog = logging.getLogger(__name__) + + +class SupsiciousThreads(interfaces.plugins.PluginInterface): + """Lists suspicious userland process threads""" + + _required_framework_version = (2, 4, 0) + _version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.PluginRequirement( + name="threads", plugin=threads.Threads, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + ] + + def _get_ranges(self, kernel, all_ranges, proc): + """ + Maintains a hash table so each process' VADs + are only enumerated once per plugin run + """ + key = proc.vol.offset + + if key not in all_ranges: + all_ranges[key] = [] + + for vad in proc.get_vad_root().traverse(): + fn = vad.get_file_name() + if not isinstance(fn, str) or not fn: + fn = None + + protection_string = vad.get_protection( + vadinfo.VadInfo.protect_values( + self.context, kernel.layer_name, kernel.symbol_table_name + ), + vadinfo.winnt_protections, + ) + + all_ranges[key].append( + (vad.get_start(), vad.get_end(), protection_string, fn) + ) + + return all_ranges[key] + + def _get_range(self, ranges, address): + for start, end, protection_string, fn in ranges: + if start <= address < end: + return start, protection_string, fn + + return None, None, None + + def _check_thread_address(self, exe_path, ranges, thread_address): + vad_base, prot, vad_path = self._get_range(ranges, thread_address) + + # threads outside of a VAD means either smear from this thread or this process' VAD tree + if vad_base is None: + return + + if vad_path is None: + # set this so checks after report the non file backed region in the path column + vad_path = "" + + yield ( + vad_path, + "This thread started execution in the VAD starting at base address ({:#x}), which is not backed by a file".format( + vad_base + ), + ) + + if prot != "PAGE_EXECUTE_WRITECOPY": + yield ( + vad_path, + "VAD at base address ({:#x}) hosting this thread has an unexpected starting protection {}".format( + vad_base, prot + ), + ) + + if ( + exe_path + and vad_path.lower().endswith(".exe") + and (vad_path.lower() != exe_path.lower()) + ): + yield ( + vad_path, + "VAD at base address ({:#x}) hosting this thread maps an application executable that is not the process exectuable".format( + vad_base + ), + ) + + def _enumerate_processes(self, kernel, all_ranges): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ): + ranges = self._get_ranges(kernel, all_ranges, proc) + + # smeared vads or process is terminating + if len(all_ranges[proc.vol.offset]) < 5: + continue + + pid = proc.UniqueProcessId + proc_name = utility.array_to_string(proc.ImageFileName) + + _, __, exe_path = self._get_range(ranges, proc.SectionBaseAddress) + if not isinstance(exe_path, str): + exe_path = None + + yield proc, pid, proc_name, exe_path, ranges + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + all_ranges = {} + + for proc, pid, proc_name, exe_path, ranges in self._enumerate_processes( + kernel, all_ranges + ): + # processes often schedule multiple threads at the same address + # there is no benefit to checking the same address more than once per process + checked = set() + + for thread in threads.Threads.list_threads(kernel, proc): + # do not process if a thread is exited or terminated (4 = Terminated) + if thread.ExitTime.QuadPart > 0 or thread.Tcb.State == 4: + continue + + # bail if accessing the threads members causes a page fault + info = thrdscan.ThrdScan.gather_thread_info(thread) + if not info: + continue + + tid, start_address = info[2], info[3] + + addresses = [ + (start_address, "Start"), + (thread.Win32StartAddress, "Win32Start"), + ] + + for address, context in addresses: + if address in checked: + continue + checked.add(address) + + for vad_path, note in self._check_thread_address( + exe_path, ranges, address + ): + yield 0, ( + proc_name, + pid, + tid, + context, + format_hints.Hex(address), + vad_path, + note, + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("TID", int), + ("Context", str), + ("Address", format_hints.Hex), + ("VAD Path", str), + ("Note", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 80906b3b9..bbf65cd6c 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -62,42 +62,41 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) _constraint, mem_object, _header = result yield mem_object + @classmethod + def gather_thread_info(cls, ethread): + try: + thread_offset = ethread.vol.offset + owner_proc_pid = ethread.Cid.UniqueProcess + thread_tid = ethread.Cid.UniqueThread + thread_start_addr = ethread.StartAddress + thread_create_time = ( + ethread.get_create_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_exit_time = ( + ethread.get_exit_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + except exceptions.InvalidAddressException: + vollog.debug("Thread invalid address {:#x}".format(thread.vol.offset)) + return None + + return ( + format_hints.Hex(thread_offset), + owner_proc_pid, + thread_tid, + format_hints.Hex(thread_start_addr), + thread_create_time, + thread_exit_time, + ) + def _generator(self): kernel = self.context.modules[self.config["kernel"]] for ethread in self.scan_threads( self.context, kernel.layer_name, kernel.symbol_table_name ): - try: - thread_offset = ethread.vol.offset - owner_proc_pid = ethread.Cid.UniqueProcess - thread_tid = ethread.Cid.UniqueThread - thread_start_addr = ethread.StartAddress - thread_create_time = ( - ethread.get_create_time() - ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object - thread_exit_time = ( - ethread.get_exit_time() - ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object - except (ValueError, exceptions.InvalidAddressException): - vollog.debug( - "Thread :{}, invalid address {} in layer {}".format( - thread_tid, thread_start_addr, kernel.layer_name - ) - ) - continue - - yield ( - 0, - ( - format_hints.Hex(thread_offset), - owner_proc_pid, - thread_tid, - format_hints.Hex(thread_start_addr), - thread_create_time, - thread_exit_time, - ), - ) + info = self.gather_thread_info(ethread) + if info: + yield (0, info) def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py new file mode 100644 index 000000000..80ff458cd --- /dev/null +++ b/volatility3/framework/plugins/windows/threads.py @@ -0,0 +1,80 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import Callable, List, Generator, Iterable, Type, Optional + +from volatility3.framework import renderers, interfaces, constants, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist, thrdscan + +vollog = logging.getLogger(__name__) + + +class Threads(thrdscan.ThrdScan): + """Lists process threads""" + + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.PluginRequirement( + name="thrdscan", plugin=thrdscan.ThrdScan, version=(1, 0, 0) + ), + ] + + @classmethod + def list_threads( + cls, kernel, proc: interfaces.objects.ObjectInterface + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Lists the Threads of a specific process. + + Args: + proc: _EPROCESS object from which to list the VADs + filter_func: Function to take a virtual address descriptor value and return True if it should be filtered out + + Returns: + A list of threads based on the process and filtered based on the filter function + """ + seen = set() + for thread in proc.ThreadListHead.to_list( + f"{kernel.symbol_table_name}{constants.BANG}_ETHREAD", "ThreadListEntry" + ): + if thread.vol.offset in seen: + break + seen.add(thread.vol.offset) + yield thread + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + kernel_layer = self.context.layers[kernel.layer_name] + + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ): + for thread in self.list_threads(kernel, proc): + info = self.gather_thread_info(thread) + if info: + yield (0, info) From b601250279fe206b2586d82820b38e1c2ea6fd83 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 08:29:50 -0500 Subject: [PATCH 066/263] #1175 - initial unloadedmodules plugin --- .../plugins/windows/unloadedmodules.py | 159 ++++++++++++++++++ .../symbols/windows/unloadedmodules-x64.json | 109 ++++++++++++ .../symbols/windows/unloadedmodules-x86.json | 109 ++++++++++++ 3 files changed, 377 insertions(+) create mode 100644 volatility3/framework/plugins/windows/unloadedmodules.py create mode 100644 volatility3/framework/symbols/windows/unloadedmodules-x64.json create mode 100644 volatility3/framework/symbols/windows/unloadedmodules-x86.json diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py new file mode 100644 index 000000000..1fd4914ac --- /dev/null +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -0,0 +1,159 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import datetime +from typing import List, Iterable + +from volatility3.framework import constants +from volatility3.framework import interfaces, symbols +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.symbols import intermed +from volatility3.plugins import timeliner + +vollog = logging.getLogger(__name__) + + +class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """Lists the unloaded kernel modules.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + ] + + @staticmethod + def create_unloadedmodules_table( + context: interfaces.context.ContextInterface, + symbol_table: str, + config_path: str, + ) -> str: + """Creates a symbol table for the unloaded modules. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of an existing symbol table containing the kernel symbols + config_path: The configuration path within the context of the symbol table to create + + Returns: + The name of the constructed unloaded modules table + """ + native_types = context.symbol_space[symbol_table].natives + is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) + table_mapping = {"nt_symbols": symbol_table} + + if is_64bit: + symbol_filename = "unloadedmodules-x64" + else: + symbol_filename = "unloadedmodules-x86" + + return intermed.IntermediateSymbolTable.create( + context, + config_path, + "windows", + symbol_filename, + native_types=native_types, + table_mapping=table_mapping, + ) + + @classmethod + def list_unloadedmodules( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + unloadedmodule_table_name: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Lists all the unloaded modules in the primary layer. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + + Returns: + A list of Unloaded Modules as retrieved from MmUnloadedDrivers + """ + + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) + unloadedmodules_offset = ntkrnlmp.get_symbol("MmUnloadedDrivers").address + unloadedmodules = ntkrnlmp.object( + object_type="pointer", + offset=unloadedmodules_offset, + subtype="array", + ) + is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) + + if is_64bit: + unloaded_count_type = "unsigned long long" + else: + unloaded_count_type = "unsigned long" + + last_unloadedmodule_offset = ntkrnlmp.get_symbol("MmLastUnloadedDriver").address + unloaded_count = ntkrnlmp.object( + object_type=unloaded_count_type, offset=last_unloadedmodule_offset + ) + + unloadedmodules_array = context.object( + object_type=unloadedmodule_table_name + + constants.BANG + + "_UNLOADED_DRIVERS", + layer_name=layer_name, + offset=unloadedmodules, + ) + unloadedmodules_array.UnloadedDrivers.count = unloaded_count + + for mod in unloadedmodules_array.UnloadedDrivers: + yield mod + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + unloadedmodule_table_name = self.create_unloadedmodules_table( + self.context, kernel.symbol_table_name, self.config_path + ) + + for mod in self.list_unloadedmodules( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + unloadedmodule_table_name, + ): + yield ( + 0, + ( + mod.Name.String, + format_hints.Hex(mod.StartAddress), + format_hints.Hex(mod.EndAddress), + conversion.wintime_to_datetime(mod.CurrentTime), + ), + ) + + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + description = f"Unloaded Module: {row_data[0]}" + yield (description, timeliner.TimeLinerType.CHANGED, row_data[3]) + + def run(self): + return renderers.TreeGrid( + [ + ("Name", str), + ("StartAddress", format_hints.Hex), + ("EndAddress", format_hints.Hex), + ("Time", datetime.datetime), + ], + self._generator(), + ) diff --git a/volatility3/framework/symbols/windows/unloadedmodules-x64.json b/volatility3/framework/symbols/windows/unloadedmodules-x64.json new file mode 100644 index 000000000..75ab7c690 --- /dev/null +++ b/volatility3/framework/symbols/windows/unloadedmodules-x64.json @@ -0,0 +1,109 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_UNLOADED_DRIVER": { + "fields": { + "Name": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "StartAddress": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 16 + }, + "EndAddress": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 24 + }, + "CurrentTime": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 40 + }, + "_UNLOADED_DRIVERS": { + "fields": { + "UnloadedDrivers": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_UNLOADED_DRIVER" + } + } + } + }, + "kind": "struct", + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle by hand", + "datetime": "2024-06-19T17:57:16.394003" + }, + "format": "4.0.0" + } +} \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/unloadedmodules-x86.json b/volatility3/framework/symbols/windows/unloadedmodules-x86.json new file mode 100644 index 000000000..ff9e78965 --- /dev/null +++ b/volatility3/framework/symbols/windows/unloadedmodules-x86.json @@ -0,0 +1,109 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_UNLOADED_DRIVER": { + "fields": { + "Name": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "StartAddress": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 8 + }, + "EndAddress": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 12 + }, + "CurrentTime": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_UNLOADED_DRIVERS": { + "fields": { + "UnloadedDrivers": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_UNLOADED_DRIVER" + } + } + } + }, + "kind": "struct", + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle by hand", + "datetime": "2024-06-19T17:57:16.394003" + }, + "format": "4.0.0" + } +} \ No newline at end of file From e172676d3caaecd3592211cd561aa612d934bcbc Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 15:07:18 -0500 Subject: [PATCH 067/263] #118 - initial timers plugin --- volatility3/framework/objects/utility.py | 21 ++ .../framework/plugins/windows/timers.py | 264 ++++++++++++++++++ .../framework/symbols/windows/__init__.py | 1 + .../symbols/windows/extensions/__init__.py | 80 ++++++ 4 files changed, 366 insertions(+) create mode 100644 volatility3/framework/plugins/windows/timers.py diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index 0292608c1..8aa527cdb 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -7,6 +7,27 @@ from typing import Optional, Union from volatility3.framework import interfaces, objects, constants +def rol(value: int, count: int, max_bits: int = 64) -> int: + """A rotate-left instruction in Python""" + max_bits_mask = (1 << max_bits) - 1 + return (value << count % max_bits) & max_bits_mask | ( + (value & max_bits_mask) >> (max_bits - (count % max_bits)) + ) + + +def bswap_32(value: int) -> int: + value = ((value << 8) & 0xFF00FF00) | ((value >> 8) & 0x00FF00FF) + + return ((value << 16) | (value >> 16)) & 0xFFFFFFFF + + +def bswap_64(value: int) -> int: + low = bswap_32((value >> 32)) + high = bswap_32((value & 0xFFFFFFFF)) + + return ((high << 32) | low) & 0xFFFFFFFFFFFFFFFF + + def array_to_string( array: "objects.Array", count: Optional[int] = None, errors: str = "replace" ) -> interfaces.objects.ObjectInterface: diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py new file mode 100644 index 000000000..19d52d9bf --- /dev/null +++ b/volatility3/framework/plugins/windows/timers.py @@ -0,0 +1,264 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterator, List, Tuple, Iterable + +from volatility3.framework import exceptions, layers, renderers, interfaces, constants, symbols +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols.windows import versions +from volatility3.plugins.windows import ssdt + +vollog = logging.getLogger(__name__) + + +class Timers(interfaces.plugins.PluginInterface): + """Print kernel timers and associated module DPCs""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), + ] + + @classmethod + def get_kernel_module( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ): + """Returns the kernel module based on the layer and symbol_table""" + virtual_layer = context.layers[layer_name] + if not isinstance(virtual_layer, layers.intel.Intel): + raise TypeError("Virtual Layer is not an intel layer") + + kvo = virtual_layer.config["kernel_virtual_offset"] + + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) + return ntkrnlmp + + @classmethod + def get_kpcrs( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> interfaces.objects.ObjectInterface: + """Returns the KPCR structure for each processor + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of an existing symbol table containing the kernel symbols + config_path: The configuration path within the context of the symbol table to create + + Returns: + The _KPCR structure for each processor + """ + + ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table) + cpu_count_offset = ntkrnlmp.get_symbol("KeNumberProcessors").address + cpu_count = ntkrnlmp.object( + object_type="unsigned int", layer_name=layer_name, offset=cpu_count_offset + ) + processor_block = ntkrnlmp.object( + object_type="pointer", + layer_name=layer_name, + offset=ntkrnlmp.get_symbol("KiProcessorBlock").address, + ) + processor_pointers = utility.array_of_pointers( + context=context, + array=processor_block, + count=cpu_count, + subtype=symbol_table + constants.BANG + "_KPRCB", + ) + for pointer in processor_pointers: + kprcb = pointer.dereference() + reloff = ntkrnlmp.get_type("_KPCR").relative_child_offset("Prcb") + kpcr = context.object( + symbol_table + constants.BANG + "_KPCR", + offset=kprcb.vol.offset - reloff, + layer_name=layer_name, + ) + yield kpcr + + @classmethod + def list_timers( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[Tuple[str, int, str]]: + """Lists all kernel timers. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + + Yields: + A _KTIMER entry + """ + ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table) + + if versions.is_windows_7( + context=context, symbol_table=symbol_table + ) or versions.is_windows_8_or_later( + context=context, symbol_table=symbol_table + ): + # Starting with Windows 7, there is no more KiTimerTableListHead. The list is + # at _KPCR.PrcbData.TimerTable.TimerEntries + # See http://pastebin.com/FiRsGW3f + for kpcr in cls.get_kpcrs(context, layer_name, symbol_table): + if hasattr(kpcr.Prcb.TimerTable, "TableState"): + for timer_entries in kpcr.Prcb.TimerTable.TimerEntries: + for timer_entry in timer_entries: + for timer in timer_entry.Entry.to_list( + symbol_table + constants.BANG + "_KTIMER", + "TimerListEntry", + ): + yield timer + + else: + for timer_entries in kpcr.Prcb.TimerTable.TimerEntries: + for timer in timer_entries.Entry.to_list( + symbol_table + constants.BANG + "_KTIMER", + "TimerListEntry", + ): + yield timer + + elif versions.is_xp_or_2003( + context=context, symbol_table=symbol_table + ) or versions.is_vista_or_later( + context=context, symbol_table=symbol_table + ): + is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) + if is_64bit or versions.is_vista_or_later(context=context, symbol_table=symbol_table): + # On XP x64, Windows 2003 SP1-SP2, and Vista SP0-SP2, KiTimerTableListHead + # is an array of 512 _KTIMER_TABLE_ENTRY structs. + array_size = 512 + else: + # On XP SP0-SP3 x86 and Windows 2003 SP0, KiTimerTableListHead + # is an array of 256 _LIST_ENTRY for _KTIMERs. + array_size = 256 + + timer_table_list_head = ntkrnlmp.object( + object_type="array", + offset=ntkrnlmp.get_symbol("KiTimerTableListHead").address, + subtype=ntkrnlmp.get_type("_LIST_ENTRY"), + count=array_size, + ) + for table in timer_table_list_head: + for timer in table.to_list( + symbol_table + constants.BANG + "_KTIMER", + "TimerListEntry", + ): + yield timer + + else: + raise NotImplementedError("This version of Windows is not supported!") + + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config["kernel"]] + layer_name = kernel.layer_name + symbol_table = kernel.symbol_table_name + + collection = ssdt.SSDT.build_module_collection( + self.context, kernel.layer_name, kernel.symbol_table_name + ) + + for timer in self.list_timers(self.context, layer_name, symbol_table): + if not timer.valid_type(): + continue + try: + dpc = timer.get_dpc() + if dpc == 0: + continue + if dpc.DeferredRoutine == 0: + continue + deferred_routine = dpc.DeferredRoutine + except Exception as e: + continue + + module_symbols = list( + collection.get_module_symbols_by_absolute_location(deferred_routine) + ) + + if module_symbols: + for module_name, symbol_generator in module_symbols: + symbols_found = False + + # we might have multiple symbols pointing to the same location + for symbol in symbol_generator: + symbols_found = True + yield ( + 0, + ( + format_hints.Hex(timer.vol.offset), + timer.get_due_time(), + timer.Period, + timer.get_signaled(), + format_hints.Hex(deferred_routine), + module_name, + symbol.split(constants.BANG)[1], + ), + ) + + # no symbols, but we at least can report the module name + if not symbols_found: + yield ( + 0, + ( + format_hints.Hex(timer.vol.offset), + timer.get_due_time(), + timer.Period, + timer.get_signaled(), + format_hints.Hex(deferred_routine), + module_name, + renderers.NotAvailableValue(), + ), + ) + else: + # no module was found at the absolute location + yield ( + 0, + ( + format_hints.Hex(timer.vol.offset), + timer.get_due_time(), + timer.Period, + timer.get_signaled(), + format_hints.Hex(deferred_routine), + renderers.NotAvailableValue(), + renderers.NotAvailableValue(), + ), + ) + + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("DueTime", str), + ("Period(ms)", int), + ("Signaled", str), + ("Routine", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index abf9f6da3..4aeb22dcf 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -39,6 +39,7 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("_VACB", extensions.VACB) self.set_type_class("_POOL_TRACKER_BIG_PAGES", pool.POOL_TRACKER_BIG_PAGES) self.set_type_class("_IMAGE_DOS_HEADER", pe.IMAGE_DOS_HEADER) + self.set_type_class("_KTIMER", extensions.KTIMER) # Might not necessarily defined in every version of windows self.optional_set_type_class("_IMAGE_NT_HEADERS", pe.IMAGE_NT_HEADERS) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index a8fa7b2ff..040c57bae 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -20,6 +20,7 @@ from volatility3.framework import ( ) from volatility3.framework.interfaces.objects import ObjectInterface from volatility3.framework.layers import intel +from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic from volatility3.framework.symbols.windows.extensions import kdbg, pe, pool @@ -994,6 +995,85 @@ class TOKEN(objects.StructType): vollog.log(constants.LOGLEVEL_VVVV, "Broken Token Privileges.") +class KTIMER(objects.StructType): + """A class for Kernel Timers""" + + VALID_TYPES = { + 8: "TimerNotificationObject", + 9: "TimerSynchronizationObject", + } + + def get_signaled(self): + if self.Header.SignalState: + return "Yes" + return "-" + + def get_raw_dpc(self): + """Returns the encoded DPC since it may not look like a pointer after encoding""" + symbol_table_name = self.get_symbol_table_name() + ulonglong_type = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "unsigned long long" + ) + + return self._context.object( + object_type=ulonglong_type, + layer_name=self.vol.layer_name, + offset=self.Dpc.vol.offset, + ) + def valid_type(self): + return self.Header.Type in self.VALID_TYPES + + def get_due_time(self): + return "{0:#010x}:{1:#010x}".format(self.DueTime.HighPart, self.DueTime.LowPart) + + def get_dpc(self): + """Return Dpc, and if Windows 7 or later, decode it""" + symbol_table_name = self.get_symbol_table_name() + kvo = self._context.layers[self.vol.native_layer_name].config[ + "kernel_virtual_offset" + ] + ntkrnlmp = self._context.module( + symbol_table_name, + layer_name=self.vol.native_layer_name, + offset=kvo, + native_layer_name=self.vol.native_layer_name, + ) + + try: + wait_never = ntkrnlmp.object( + object_type="unsigned long long", + offset=ntkrnlmp.get_symbol("KiWaitNever").address, + ) + + wait_always = ntkrnlmp.object( + object_type="unsigned long long", + offset=ntkrnlmp.get_symbol("KiWaitAlways").address, + ) + except exceptions.SymbolError: + wait_never = None + wait_always = None + + if wait_never is None or wait_always is None: + return self.Dpc + else: + low_byte = (wait_never) & 0xFF + entry = utility.rol(self.get_raw_dpc() ^ wait_never, low_byte) + swap_xor = self.vol.offset | 0xFFFF000000000000 + entry = utility.bswap_64(entry ^ swap_xor) + dpc = entry ^ wait_always + + symbol_table_name = self.get_symbol_table_name() + kdpc_type = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "_KDPC" + ) + + return self._context.object( + object_type=kdpc_type, + layer_name=self.vol.layer_name, + offset=dpc, + ) + + class KTHREAD(objects.StructType): """A class for thread control block objects.""" From c79dc52a854d2a5be41283a720081f63471c0df8 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 15:09:21 -0500 Subject: [PATCH 068/263] #118 - black formatted --- .../framework/plugins/windows/timers.py | 23 +++++++++++-------- .../symbols/windows/extensions/__init__.py | 7 +++--- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index 19d52d9bf..e54b2a13f 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -6,7 +6,14 @@ import logging from typing import Iterator, List, Tuple, Iterable -from volatility3.framework import exceptions, layers, renderers, interfaces, constants, symbols +from volatility3.framework import ( + exceptions, + layers, + renderers, + interfaces, + constants, + symbols, +) from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints @@ -117,9 +124,7 @@ class Timers(interfaces.plugins.PluginInterface): if versions.is_windows_7( context=context, symbol_table=symbol_table - ) or versions.is_windows_8_or_later( - context=context, symbol_table=symbol_table - ): + ) or versions.is_windows_8_or_later(context=context, symbol_table=symbol_table): # Starting with Windows 7, there is no more KiTimerTableListHead. The list is # at _KPCR.PrcbData.TimerTable.TimerEntries # See http://pastebin.com/FiRsGW3f @@ -143,11 +148,11 @@ class Timers(interfaces.plugins.PluginInterface): elif versions.is_xp_or_2003( context=context, symbol_table=symbol_table - ) or versions.is_vista_or_later( - context=context, symbol_table=symbol_table - ): + ) or versions.is_vista_or_later(context=context, symbol_table=symbol_table): is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) - if is_64bit or versions.is_vista_or_later(context=context, symbol_table=symbol_table): + if is_64bit or versions.is_vista_or_later( + context=context, symbol_table=symbol_table + ): # On XP x64, Windows 2003 SP1-SP2, and Vista SP0-SP2, KiTimerTableListHead # is an array of 512 _KTIMER_TABLE_ENTRY structs. array_size = 512 @@ -172,7 +177,6 @@ class Timers(interfaces.plugins.PluginInterface): else: raise NotImplementedError("This version of Windows is not supported!") - def _generator(self) -> Iterator[Tuple]: kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name @@ -248,7 +252,6 @@ class Timers(interfaces.plugins.PluginInterface): ), ) - def run(self): return renderers.TreeGrid( [ diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 040c57bae..b830a9d9f 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -453,9 +453,9 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[str, interfaces.renderers.BaseAbsentValue] = ( - renderers.UnreadableValue() - ) + name: Union[ + str, interfaces.renderers.BaseAbsentValue + ] = renderers.UnreadableValue() # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. @@ -1020,6 +1020,7 @@ class KTIMER(objects.StructType): layer_name=self.vol.layer_name, offset=self.Dpc.vol.offset, ) + def valid_type(self): return self.Header.Type in self.VALID_TYPES From 7065446e8ab9efb02254e4415235072d30422596 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 15:17:40 -0500 Subject: [PATCH 069/263] #118 - fix black issues --- volatility3/framework/plugins/windows/timers.py | 1 - volatility3/framework/symbols/windows/extensions/__init__.py | 1 - 2 files changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index e54b2a13f..ec8bc17e9 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -7,7 +7,6 @@ import logging from typing import Iterator, List, Tuple, Iterable from volatility3.framework import ( - exceptions, layers, renderers, interfaces, diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b830a9d9f..895a8c094 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1045,7 +1045,6 @@ class KTIMER(objects.StructType): object_type="unsigned long long", offset=ntkrnlmp.get_symbol("KiWaitNever").address, ) - wait_always = ntkrnlmp.object( object_type="unsigned long long", offset=ntkrnlmp.get_symbol("KiWaitAlways").address, From 2d7789f8509664f3930705bf83524d6b5ec8de33 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 15:49:03 -0500 Subject: [PATCH 070/263] #118 - fix black issues --- .../framework/symbols/windows/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 895a8c094..4d4ffc055 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -453,9 +453,9 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[ - str, interfaces.renderers.BaseAbsentValue - ] = renderers.UnreadableValue() + name: Union[str, interfaces.renderers.BaseAbsentValue] = ( + renderers.UnreadableValue() + ) # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. From 146baab14c4a17bdcbc35715b30a4d190ed82214 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 15:54:16 -0500 Subject: [PATCH 071/263] #118 - refactor get_dpc --- .../framework/symbols/windows/extensions/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 4d4ffc055..b9eabcda6 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1040,7 +1040,7 @@ class KTIMER(objects.StructType): native_layer_name=self.vol.native_layer_name, ) - try: + if ntkrnlmp.has_symbol("KiWaitNever") and ntkrnlmp.has_symbol("KiWaitAlways"): wait_never = ntkrnlmp.object( object_type="unsigned long long", offset=ntkrnlmp.get_symbol("KiWaitNever").address, @@ -1049,13 +1049,7 @@ class KTIMER(objects.StructType): object_type="unsigned long long", offset=ntkrnlmp.get_symbol("KiWaitAlways").address, ) - except exceptions.SymbolError: - wait_never = None - wait_always = None - if wait_never is None or wait_always is None: - return self.Dpc - else: low_byte = (wait_never) & 0xFF entry = utility.rol(self.get_raw_dpc() ^ wait_never, low_byte) swap_xor = self.vol.offset | 0xFFFF000000000000 @@ -1072,6 +1066,8 @@ class KTIMER(objects.StructType): layer_name=self.vol.layer_name, offset=dpc, ) + else: + return self.Dpc class KTHREAD(objects.StructType): From 898c0844c464f87ab7bbd1cae6756a69b2a8048e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 21 Jun 2024 16:15:17 -0500 Subject: [PATCH 072/263] Windows: fixes scanner bug for versions < win10 This commit fixes a bug where the `layer_name` gets discarded when constructing objects. Previously, it was assumed that we would not want to construct an object for a module with a layer_name different from that of the module. However, because we switch to scanning the memory layer on samples where the version is < 10, but still construct kernel executive objects based on the result of the memory layer scan, we actually do sometimes need to specify a different layer. --- volatility3/framework/contexts/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 73868a58f..4c169728c 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -256,12 +256,13 @@ class Module(interfaces.context.ModuleInterface): if not absolute: offset += self._offset - # Ensure we don't use a layer_name other than the module's, why would anyone do that? - if "layer_name" in kwargs: - del kwargs["layer_name"] + # We have to allow using an alternative layer name due to pool scanners switching + # to the memory layer for scanning samples prior to Windows 10. + layer_name = kwargs.pop("layer_name", self._layer_name) + return self._context.object( object_type=object_type, - layer_name=self._layer_name, + layer_name=layer_name, offset=offset, native_layer_name=native_layer_name or self._native_layer_name, **kwargs, From 7ece5fb1bf2f27d2731d09ed8329fb20552c713c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 23 Jun 2024 00:16:23 +0100 Subject: [PATCH 073/263] Windows: Improve Virtmap error messages slightly --- volatility3/framework/plugins/windows/virtmap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/virtmap.py b/volatility3/framework/plugins/windows/virtmap.py index 5190bec8d..3f3f270e2 100644 --- a/volatility3/framework/plugins/windows/virtmap.py +++ b/volatility3/framework/plugins/windows/virtmap.py @@ -78,7 +78,7 @@ class VirtMap(interfaces.plugins.PluginInterface): ) else: raise exceptions.SymbolError( - None, module.name, "Required structures not found" + "SystemVaRegions", module.name, "Required structures not found" ) elif module.has_symbol("MiSystemVaType"): system_range_start = module.object( @@ -99,7 +99,7 @@ class VirtMap(interfaces.plugins.PluginInterface): ) else: raise exceptions.SymbolError( - None, module.name, "Required structures not found" + "MiVisibleState", module.name, "Required structures not found" ) return result From 8c1a5c46e32c8ffaacd6507ec036cbd07c9072b7 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 27 Jun 2024 17:13:37 -0500 Subject: [PATCH 074/263] Add timeliner support to userassist --- .../plugins/windows/registry/userassist.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 70c75b50b..54ec2fc74 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -17,11 +17,12 @@ from volatility3.framework.layers.registry import RegistryHive from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.plugins.windows.registry import hivelist +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class UserAssist(interfaces.plugins.PluginInterface): +class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Print userassist registry keys and information.""" _required_framework_version = (2, 0, 0) @@ -335,6 +336,19 @@ class UserAssist(interfaces.plugins.PluginInterface): ) yield result + + def generate_timeline(self): + self._reg_table_name = intermed.IntermediateSymbolTable.create( + self.context, self._config_path, "windows", "registry" + ) + + for row in self._generator(): + _depth, row_data = row + # check the name and the timestamp to not be empty + if isinstance(row_data[5], str) and not isinstance(row_data[10], renderers.NotApplicableValue): + description = f"UserAssist: {row_data[5]} {row_data[2]} ({row_data[7]})" + yield (description, timeliner.TimeLinerType.MODIFIED, row_data[10]) + def run(self): self._reg_table_name = intermed.IntermediateSymbolTable.create( self.context, self._config_path, "windows", "registry" From 8525edd3331c6ccd967c8048d61adf5d2c3e1015 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 27 Jun 2024 17:15:49 -0500 Subject: [PATCH 075/263] Formatting fixes --- volatility3/framework/plugins/windows/registry/userassist.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 54ec2fc74..cf345c901 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -336,7 +336,6 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac ) yield result - def generate_timeline(self): self._reg_table_name = intermed.IntermediateSymbolTable.create( self.context, self._config_path, "windows", "registry" @@ -345,7 +344,9 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac for row in self._generator(): _depth, row_data = row # check the name and the timestamp to not be empty - if isinstance(row_data[5], str) and not isinstance(row_data[10], renderers.NotApplicableValue): + if isinstance(row_data[5], str) and not isinstance( + row_data[10], renderers.NotApplicableValue + ): description = f"UserAssist: {row_data[5]} {row_data[2]} ({row_data[7]})" yield (description, timeliner.TimeLinerType.MODIFIED, row_data[10]) From 857cd8df491310d4c8ed56833a9fae6714f9c582 Mon Sep 17 00:00:00 2001 From: atcuno Date: Tue, 2 Jul 2024 19:14:15 -0500 Subject: [PATCH 076/263] Updates from ikelos' feedback --- .../plugins/windows/hollowprocesses.py | 67 ++++++++++++++++--- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index 9d7e800b7..3158667ac 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import NamedTuple +from typing import NamedTuple, Dict from volatility3.framework import interfaces, exceptions, constants from volatility3.framework import renderers @@ -12,21 +12,25 @@ from volatility3.plugins.windows import pslist, vadinfo vollog = logging.getLogger(__name__) -VadInfo = NamedTuple( - "VadInfo", +VadData = NamedTuple( + "VadData", [ ("protection", str), ("path", str), ], ) -DLLInfo = NamedTuple( - "DLLInfo", +DLLData = NamedTuple( + "DLLData", [ ("path", str), ], ) +### Useful references on process hollowing +# https://cysinfo.com/detecting-deceptive-hollowing-techniques/ +# https://github.com/m0n0ph1/Process-Hollowing + class HollowProcesses(interfaces.plugins.PluginInterface): """Lists hollowed processes""" @@ -56,7 +60,16 @@ class HollowProcesses(interfaces.plugins.PluginInterface): ), ] - def _get_vads_map(self, proc): + def _get_vads_data( + self, proc: interfaces.objects.ObjectInterface + ) -> Dict[int, VadData]: + """ + Returns a dictionary of: + base address -> (protection string, file name) + For each mapped VAD in the process. This is used + for quick lookups of data and matching the DLL + at the same base address as the VAD + """ vads = {} kernel = self.context.modules[self.config["kernel"]] @@ -73,11 +86,23 @@ class HollowProcesses(interfaces.plugins.PluginInterface): if not fn or not isinstance(fn, str): fn = "" - vads[vad.get_start()] = VadInfo(protection_string, fn) + vads[vad.get_start()] = VadData(protection_string, fn) return vads - def _get_dlls_map(self, proc): + def _get_dlls_map( + self, proc: interfaces.objects.ObjectInterface + ) -> Dict[int, DLLData]: + """ + Returns a dictionary of: + base address -> path + for each DLL loaded in the process + + This is used to cross compare with + the corresponding VAD and to have a + backup path source in case of smear + in the VAD + """ dlls = {} for entry in proc.load_order_modules(): @@ -91,11 +116,14 @@ class HollowProcesses(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: FullDllName = renderers.UnreadableValue() - dlls[base] = DLLInfo(FullDllName) + dlls[base] = DLLData(FullDllName) return dlls - def _get_image_base(self, proc): + def _get_image_base(self, proc: interfaces.objects.ObjectInterface) -> int: + """ + Uses the PEB to get the image base of the process + """ kernel = self.context.modules[self.config["kernel"]] try: @@ -110,6 +138,12 @@ class HollowProcesses(interfaces.plugins.PluginInterface): return None def _check_load_address(self, proc, _, __): + """ + Detects when the image base in the PEB, which is writable by process malware, + does not match the section base address - whose value lives in kernel memory. + Many malware samples will manipulate their image base to fool AVs/EDRs and + as a necessary part of certain hollowing techniques + """ image_base = self._get_image_base(proc) if image_base is not None and image_base != proc.SectionBaseAddress: yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format( @@ -117,6 +151,16 @@ class HollowProcesses(interfaces.plugins.PluginInterface): ) def _check_exe_protection(self, proc, vads, __): + """ + Legitimately mapped application executables and DLLs + will have a VAD present and its initial protection will be + PAGE_EXECUTE_WRITECOPY. + Many process hollowing and code injection techniques will + unmap the real executable and/or map in executables with + incorrect permissions. + This check verifies the VAD for the application exe. + `_check_dlls_protection` checks for DLLs mapped in the process. + """ base = proc.SectionBaseAddress if base not in vads: @@ -134,6 +178,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): if dll_base not in vads: continue + # PAGE_EXECUTE_WRITECOPY is the only valid permission for mapped DLLs and .exe files if vads[dll_base].protection != "PAGE_EXECUTE_WRITECOPY": yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format( vads[dll_base].protection, dll_base, dlls[dll_base].path @@ -155,7 +200,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): if len(dlls) < 3: continue - vads = self._get_vads_map(proc) + vads = self._get_vads_data(proc) if len(vads) < 5: continue From 020005f43cf32e9a38622926c184be1b18d9b18e Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 3 Jul 2024 09:55:07 -0500 Subject: [PATCH 077/263] Address feedback from ikelos --- .../plugins/windows/suspicious_threads.py | 45 ++++++++++++------- .../framework/plugins/windows/threads.py | 7 +-- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspicious_threads.py b/volatility3/framework/plugins/windows/suspicious_threads.py index b24f3f2c2..2679b191a 100644 --- a/volatility3/framework/plugins/windows/suspicious_threads.py +++ b/volatility3/framework/plugins/windows/suspicious_threads.py @@ -3,7 +3,7 @@ # import logging -from typing import List +from typing import List, Dict, Tuple, Generator from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -42,7 +42,12 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): ), ] - def _get_ranges(self, kernel, all_ranges, proc): + def _get_ranges( + self, + kernel: interfaces.context.ModuleInterface, + all_ranges: Dict[int, List[Tuple[int, int, str, str]]], + proc, + ) -> Tuple[int, int, str, str]: """ Maintains a hash table so each process' VADs are only enumerated once per plugin run @@ -70,14 +75,24 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): return all_ranges[key] - def _get_range(self, ranges, address): + def _get_range( + self, ranges: Dict[int, List[Tuple[int, int, str, str]]], address: int + ) -> Tuple[int, str, str]: + """ + Walks a process' VADs looking for the one + containing `address` + + Returns its base address, protection string, and mapped file, if any + """ for start, end, protection_string, fn in ranges: if start <= address < end: return start, protection_string, fn return None, None, None - def _check_thread_address(self, exe_path, ranges, thread_address): + def _check_thread_address( + self, exe_path: str, ranges, thread_address: int + ) -> Generator[Tuple[str, str], None, None]: vad_base, prot, vad_path = self._get_range(ranges, thread_address) # threads outside of a VAD means either smear from this thread or this process' VAD tree @@ -90,19 +105,17 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): yield ( vad_path, - "This thread started execution in the VAD starting at base address ({:#x}), which is not backed by a file".format( - vad_base - ), + f"This thread started execution in the VAD starting at base address ({vad_base:#x}), which is not backed by a file", ) + # All threads should point to PAGE_EXECUTE_WRITECOPY mapped regions if prot != "PAGE_EXECUTE_WRITECOPY": yield ( vad_path, - "VAD at base address ({:#x}) hosting this thread has an unexpected starting protection {}".format( - vad_base, prot - ), + f"VAD at base address ({vad_base:#x}) hosting this thread has an unexpected starting protection {prot}", ) + # check for process hollowing type techniques that mapped in a second, malicious exe file if ( exe_path and vad_path.lower().endswith(".exe") @@ -110,12 +123,12 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): ): yield ( vad_path, - "VAD at base address ({:#x}) hosting this thread maps an application executable that is not the process exectuable".format( - vad_base - ), + "VAD at base address ({vad_base:#x}) hosting this thread maps an application executable that is not the process exectuable", ) - def _enumerate_processes(self, kernel, all_ranges): + def _enumerate_processes( + self, kernel: interfaces.context.ModuleInterface, all_ranges + ): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for proc in pslist.PsList.list_processes( @@ -147,7 +160,7 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): for proc, pid, proc_name, exe_path, ranges in self._enumerate_processes( kernel, all_ranges ): - # processes often schedule multiple threads at the same address + # processes often create multiple threads at the same address # there is no benefit to checking the same address more than once per process checked = set() @@ -161,7 +174,7 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): if not info: continue - tid, start_address = info[2], info[3] + _, _, tid, start_address, _, _ = info addresses = [ (start_address, "Start"), diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index 80ff458cd..83d231abb 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -3,12 +3,10 @@ # import logging -from typing import Callable, List, Generator, Iterable, Type, Optional +from typing import List, Generator -from volatility3.framework import renderers, interfaces, constants, exceptions +from volatility3.framework import interfaces, constants from volatility3.framework.configuration import requirements -from volatility3.framework.objects import utility -from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist, thrdscan vollog = logging.getLogger(__name__) @@ -64,7 +62,6 @@ class Threads(thrdscan.ThrdScan): def _generator(self): kernel = self.context.modules[self.config["kernel"]] - kernel_layer = self.context.layers[kernel.layer_name] filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) From acc41c2b8635df059dc77738121edd7b043d52a0 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 20 Feb 2024 16:59:51 -0600 Subject: [PATCH 078/263] Windows: Callbacks - update symbol files Updates the windows callbacks symbol files to include structures that were missing from the original volatility plugin. --- .../symbols/windows/callbacks-x64.json | 184 ++++++++++++++++-- .../symbols/windows/callbacks-x86.json | 179 ++++++++++++++++- 2 files changed, 342 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/symbols/windows/callbacks-x64.json b/volatility3/framework/symbols/windows/callbacks-x64.json index 87682cb92..3f891b94f 100644 --- a/volatility3/framework/symbols/windows/callbacks-x64.json +++ b/volatility3/framework/symbols/windows/callbacks-x64.json @@ -1,6 +1,18 @@ { "symbols": {}, - "enums": {}, + "enums": { + "EventCategory": { + "base": "long", + "constants": { + "EventCategoryReserved": 0, + "EventCategoryHardwareProfileChange": 1, + "EventCategoryDeviceInterfaceChange": 2, + "EventCategoryTargetDeviceChange": 3, + "EventCategoryKernelSoftRestart": 4 + }, + "size": 4 + } + }, "base_types": { "unsigned long": { "kind": "int", @@ -48,19 +60,161 @@ "user_types": { "_GENERIC_CALLBACK": { "fields": { - "Callback": { - "type": { - "kind": "pointer", - "subtype": { - "kind": "base", - "name": "void" - } + "Callback": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 8 }, - "offset": 8 - } + "Associated": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + } }, "kind": "struct", - "size": 16 + "size": 24 + }, + "_NOTIFICATION_PACKET": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "DriverObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DRIVER_OBJECT" + } + }, + "offset": 16 + }, + "NotificationRoutine": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 48 + }, + "_SHUTDOWN_PACKET": { + "fields": { + "Entry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "DeviceObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DEVICE_OBJECT" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_DBGPRINT_CALLBACK": { + "fields": { + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_NOTIFY_ENTRY_HEADER": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "EventCategory": { + "type": { + "kind": "enum", + "name": "EventCategory" + }, + "offset": 16 + }, + "CallbackRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "DriverObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DRIVER_OBJECT" + } + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 64 + }, + "_REGISTRY_CALLBACK": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 48 }, "_KBUGCHECK_CALLBACK_RECORD": { "fields": { @@ -184,10 +338,10 @@ }, "metadata": { "producer": { - "version": "0.0.1", - "name": "mhl by hand", - "datetime": "2019-08-27T18:17:16.417006" + "version": "0.0.2", + "name": "dgmcdona by hand", + "datetime": "2024-02-02T19:32:00.000000" }, "format": "4.0.0" } -} \ No newline at end of file +} diff --git a/volatility3/framework/symbols/windows/callbacks-x86.json b/volatility3/framework/symbols/windows/callbacks-x86.json index 702b68a65..63b8c69b4 100644 --- a/volatility3/framework/symbols/windows/callbacks-x86.json +++ b/volatility3/framework/symbols/windows/callbacks-x86.json @@ -1,6 +1,18 @@ { "symbols": {}, - "enums": {}, + "enums": { + "EventCategory": { + "base": "long", + "constants": { + "EventCategoryReserved": 0, + "EventCategoryHardwareProfileChange": 1, + "EventCategoryDeviceInterfaceChange": 2, + "EventCategoryTargetDeviceChange": 3, + "EventCategoryKernelSoftRestart": 4 + }, + "size": 4 + } + }, "base_types": { "unsigned long": { "kind": "int", @@ -46,6 +58,151 @@ } }, "user_types": { + "_NOTIFICATION_PACKET": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "DriverObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DRIVER_OBJECT" + } + }, + "offset": 8 + }, + "NotificationRoutine": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_SHUTDOWN_PACKET": { + "fields": { + "Entry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "DeviceObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DEVICE_OBJECT" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 12 + }, + "_REGISTRY_CALLBACK": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 28 + } + }, + "kind": "struct", + "size": 32 + }, + "_REGISTRY_CALLBACK_LEGACY": { + "fields": { + "CreateTime": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 28 + } + }, + "kind": "struct", + "size": 32 + }, + "_DBGPRINT_CALLBACK": { + "fields": { + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 20 + }, + "_NOTIFY_ENTRY_HEADER": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "EventCategory": { + "type": { + "kind": "enum", + "name": "EventCategory" + }, + "offset": 8 + }, + "CallbackRoutine": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + }, + "DriverObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DRIVER_OBJECT" + } + }, + "offset": 28 + } + }, + "kind": "struct", + "size": 32 + }, "_GENERIC_CALLBACK": { "fields": { "Callback": { @@ -57,10 +214,20 @@ } }, "offset": 4 + }, + "Associated": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 8 } }, "kind": "struct", - "size": 8 + "size": 12 }, "_KBUGCHECK_CALLBACK_RECORD": { "fields": { @@ -184,10 +351,10 @@ }, "metadata": { "producer": { - "version": "0.0.1", - "name": "mhl by hand", - "datetime": "2019-08-27T18:17:16.417006" + "version": "0.0.2", + "name": "dgmcdona by hand", + "datetime": "2024-02-02T19:32:00.000000" }, "format": "4.0.0" } -} \ No newline at end of file +} From 89b024d8f58894008bf26238e24b512c4840a66c Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 20 Feb 2024 17:00:38 -0600 Subject: [PATCH 079/263] Framework: Fix bad format string Fixes a bad format string inside of a raised exception --- volatility3/framework/symbols/intermed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 8c40e466a..751f88e39 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -282,7 +282,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): urls = list(cls.file_symbol_url(sub_path, filename)) if not urls: raise FileNotFoundError( - "No symbol files found at provided filename: {}", filename + f"No symbol files found at provided filename: {filename}", ) table_name = context.symbol_space.free_table_name(filename) table = cls( From 55881535bb39dc610119155a098f2aaa1d259adf Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 20 Feb 2024 17:01:02 -0600 Subject: [PATCH 080/263] Windows: Callbacks - parse missing callback types Updates the Windows callbacks plugin to support several types of callbacks that were present in the original volatility framework, but were missing in volatility3. Adds an extension for `_SHUTDOWN_PACKET` structures for determining validity of structure. --- .../framework/plugins/windows/callbacks.py | 347 ++++++++++++++++-- .../symbols/windows/extensions/callbacks.py | 51 +++ 2 files changed, 368 insertions(+), 30 deletions(-) create mode 100644 volatility3/framework/symbols/windows/extensions/callbacks.py diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index fcc333b9f..d5eeda1ea 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -2,16 +2,24 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import logging import contextlib -from typing import List, Iterable, Tuple, Optional, Union +import logging +from typing import Dict, Iterable, List, Optional, Tuple, Union, cast -from volatility3.framework import constants, exceptions, renderers, interfaces, symbols +from volatility3.framework import ( + constants, + exceptions, + interfaces, + objects, + renderers, + symbols, +) from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions -from volatility3.plugins.windows import ssdt +from volatility3.framework.symbols.windows.extensions import callbacks +from volatility3.plugins.windows import driverirp, handles, poolscanner, ssdt vollog = logging.getLogger(__name__) @@ -20,7 +28,7 @@ class Callbacks(interfaces.plugins.PluginInterface): """Lists kernel callbacks and notification routines.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -33,27 +41,154 @@ class Callbacks(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="driverirp", plugin=driverirp.DriverIrp, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="handles", plugin=handles.Handles, version=(1, 0, 0) + ), ] - @staticmethod - def create_callback_table( + @classmethod + def create_callback_scan_constraints( + cls, context: interfaces.context.ContextInterface, symbol_table: str, - config_path: str, - ) -> str: - """Creates a symbol table for a set of callbacks. + is_vista_or_above: bool, + ) -> List[poolscanner.PoolConstraint]: + """Creates a list of Pool Tag Constraints for callback objects. Args: context: The context to retrieve required elements (layers, symbol tables) from - symbol_table: The name of an existing symbol table containing the kernel symbols - config_path: The configuration path within the context of the symbol table to create + symbol_table: The name of an existing symbol table containing the symbols / types + is_vista_or_above: A boolean indicating whether the OS version is Vista or newer. Returns: - The name of the constructed callback table + The list containing the built constraints. """ - native_types = context.symbol_space[symbol_table].natives - is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) - table_mapping = {"nt_symbols": symbol_table} + constraints = cls._create_default_scan_constraints(context, symbol_table) + if is_vista_or_above: + constraints += cls._create_scan_constraints_vista_and_above(symbol_table) + return constraints + + @staticmethod + def _create_default_scan_constraints( + context: interfaces.context.ContextInterface, symbol_table: str + ) -> List[poolscanner.PoolConstraint]: + + shutdown_packet_size = context.symbol_space.get_type( + symbol_table + constants.BANG + "_SHUTDOWN_PACKET" + ).size + generic_callback_size = context.symbol_space.get_type( + symbol_table + constants.BANG + "_GENERIC_CALLBACK" + ).size + notification_packet_size = context.symbol_space.get_type( + symbol_table + constants.BANG + "_NOTIFICATION_PACKET" + ).size + + return [ + poolscanner.PoolConstraint( + b"IoFs", + type_name=symbol_table + constants.BANG + "_NOTIFICATION_PACKET", + size=(notification_packet_size, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + ), + poolscanner.PoolConstraint( + b"IoSh", + type_name=symbol_table + constants.BANG + "_SHUTDOWN_PACKET", + size=(shutdown_packet_size, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + index=(0, 0), + ), + poolscanner.PoolConstraint( + b"Cbrb", + type_name=symbol_table + constants.BANG + "_GENERIC_CALLBACK", + size=(generic_callback_size, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + ), + ] + + @staticmethod + def _create_scan_constraints_vista_and_above( + symbol_table: str, + ) -> List[poolscanner.PoolConstraint]: + """Creates a list of Pool Tag Constraints for callback objects. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of an existing symbol table containing the symbols / types + + Returns: + The list containing the built constraints. + """ + + return [ + poolscanner.PoolConstraint( + b"DbCb", + type_name=symbol_table + constants.BANG + "_DBGPRINT_CALLBACK", + size=(0x20, 0x40), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + ), + poolscanner.PoolConstraint( + b"Pnp9", + type_name=symbol_table + constants.BANG + "_NOTIFY_ENTRY_HEADER", + size=(0x30, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + index=(1, 1), + ), + poolscanner.PoolConstraint( + b"PnpD", + type_name=symbol_table + constants.BANG + "_NOTIFY_ENTRY_HEADER", + size=(0x40, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + index=(1, 1), + ), + poolscanner.PoolConstraint( + b"PnpC", + type_name=symbol_table + constants.BANG + "_NOTIFY_ENTRY_HEADER", + size=(0x38, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + index=(1, 1), + ), + ] + + @classmethod + def create_callback_symbol_table( + cls, + context: interfaces.context.ContextInterface, + nt_symbol_table: str, + config_path: str, + ) -> str: + """Creates a symbol table for kernel callback objects. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + nt_symbol_table: The name of the table containing the kernel symbols + config_path: The config path where to find symbol files + + Returns: + The name of the constructed symbol table + """ + native_types = context.symbol_space[nt_symbol_table].natives + is_64bit = symbols.symbol_table_is_64bit(context, nt_symbol_table) + table_mapping = {"nt_symbols": nt_symbol_table} if is_64bit: symbol_filename = "callbacks-x64" @@ -65,10 +200,143 @@ class Callbacks(interfaces.plugins.PluginInterface): config_path, "windows", symbol_filename, + class_types=callbacks.class_types_x86 if not is_64bit else None, native_types=native_types, table_mapping=table_mapping, ) + @classmethod + def scan( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + nt_symbol_table: str, + callback_symbol_table: str, + ) -> Iterable[ + Tuple[ + Union[str, interfaces.renderers.BaseAbsentValue], + int, + Union[str, interfaces.renderers.BaseAbsentValue], + ] + ]: + """Scans for callback objects using the poolscanner module and constraints. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + nt_symbol_table: The name of the table containing the kernel symbols + callback_symbol_table: The name of the table containing the callback object symbols (_SHUTDOWN_PACKET etc.) + + Returns: + A list of callback objects found by scanning the `layer_name` layer for callback pool signatures + """ + is_vista_or_later = versions.is_vista_or_later( + context=context, symbol_table=nt_symbol_table + ) + + type_map = handles.Handles.get_type_map(context, layer_name, nt_symbol_table) + + constraints = cls.create_callback_scan_constraints( + context, callback_symbol_table, is_vista_or_later + ) + + for ( + _constraint, + mem_object, + _header, + ) in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, nt_symbol_table, constraints + ): + try: + if hasattr(mem_object, "is_valid") and not mem_object.is_valid(): + continue + + yield cls._process_scanned_callback(mem_object, type_map) + except exceptions.InvalidAddressException: + continue + + @classmethod + def _process_scanned_callback( + cls, memory_object: objects.StructType, type_map: Dict[int, str] + ) -> Tuple[ + Union[str, interfaces.renderers.BaseAbsentValue], + int, + Union[str, interfaces.renderers.BaseAbsentValue], + ]: + symbol_table = memory_object.get_symbol_table_name() + type_name = memory_object.vol.type_name + + if isinstance(memory_object, callbacks._SHUTDOWN_PACKET) or ( + type_name == symbol_table + constants.BANG + "_SHUTDOWN_PACKET" + ): + callback_type = "IoRegisterShutdownNotification" + + try: + driver = memory_object.DeviceObject.DriverObject + index = driverirp.MAJOR_FUNCTIONS.index("IRP_MJ_SHUTDOWN") + callback_address = driver.MajorFunction[index] + details = driver.DriverName.String or renderers.UnparsableValue() + except exceptions.InvalidAddressException: + callback_address = memory_object.vol.offset + details = renderers.NotApplicableValue() + + elif type_name == symbol_table + constants.BANG + "_NOTIFICATION_PACKET": + callback_type = "IoRegisterFsRegistrationChange" + callback_address = memory_object.NotificationRoutine + details = renderers.NotApplicableValue() + + elif type_name == symbol_table + constants.BANG + "_NOTIFY_ENTRY_HEADER": + driver = ( + memory_object.DriverObject.dereference() + if memory_object.DriverObject.is_readable() + else None + ) + + if driver: + # Instantiate an object header for the driver name + header = driver.get_object_header() + try: + if header.get_object_type(type_map) == "Driver": + # Grab the object name + details = header.NameInfo.Name.String + else: + details = renderers.NotApplicableValue() + except exceptions.InvalidAddressException: + details = renderers.UnreadableValue() + else: + details = renderers.UnreadableValue() + + callback_type = ( + memory_object.EventCategory.description + if memory_object.EventCategory.is_valid_choice + else renderers.UnparsableValue() + ) + + callback_address = memory_object.CallbackRoutine + + elif type_name == symbol_table + constants.BANG + "_GENERIC_CALLBACK": + callback_type = "GenericKernelCallback" + callback_address = memory_object.Callback + details = renderers.NotApplicableValue() + + elif type_name == symbol_table + constants.BANG + "_DBGPRINT_CALLBACK": + callback_type = "DbgSetDebugPrintCallback" + callback_address = memory_object.Function + details = renderers.NotApplicableValue() + + else: + raise ValueError(f"Unexpected object type {type_name}") + + return ( + callback_type, + callback_address, + ( + details + if not isinstance(details, interfaces.renderers.BaseAbsentValue) + else details + ), + ) + @classmethod def list_notify_routines( cls, @@ -115,11 +383,14 @@ class Callbacks(interfaces.plugins.PluginInterface): else: count = 8 - fast_refs = ntkrnlmp.object( - object_type="array", - offset=symbol_offset, - subtype=ntkrnlmp.get_type("_EX_FAST_REF"), - count=count, + fast_refs = cast( + List[objects.Pointer], + ntkrnlmp.object( + object_type="array", + offset=symbol_offset, + subtype=ntkrnlmp.get_type("_EX_FAST_REF"), + count=count, + ), ) for fast_ref in fast_refs: @@ -159,11 +430,14 @@ class Callbacks(interfaces.plugins.PluginInterface): if callback_count == 0: return None - fast_refs = ntkrnlmp.object( - object_type="array", - offset=symbol_offset, - subtype=ntkrnlmp.get_type("_EX_FAST_REF"), - count=callback_count, + fast_refs = cast( + List[objects.Pointer], + ntkrnlmp.object( + object_type="array", + offset=symbol_offset, + subtype=ntkrnlmp.get_type("_EX_FAST_REF"), + count=callback_count, + ), ) for fast_ref in fast_refs: @@ -265,7 +539,13 @@ class Callbacks(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, callback_table_name: str, - ) -> Iterable[Tuple[str, int, str]]: + ) -> Iterable[ + Tuple[ + str, + int, + interfaces.renderers.BaseAbsentValue, + ] + ]: """Lists all kernel bugcheck reason callbacks. Args: @@ -323,7 +603,13 @@ class Callbacks(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, callback_table_name: str, - ) -> Iterable[Tuple[str, int, str]]: + ) -> Iterable[ + Tuple[ + str, + int, + Union[interfaces.objects.ObjectInterface, renderers.UnreadableValue], + ] + ]: """Lists all kernel bugcheck callbacks. Args: @@ -372,7 +658,7 @@ class Callbacks(interfaces.plugins.PluginInterface): def _generator(self): kernel = self.context.modules[self.config["kernel"]] - callback_table_name = self.create_callback_table( + callback_symbol_table = self.create_callback_symbol_table( self.context, kernel.symbol_table_name, self.config_path ) @@ -385,6 +671,7 @@ class Callbacks(interfaces.plugins.PluginInterface): self.list_bugcheck_callbacks, self.list_bugcheck_reason_callbacks, self.list_registry_callbacks, + self.scan, ) for callback_method in callback_methods: @@ -392,7 +679,7 @@ class Callbacks(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, - callback_table_name, + callback_symbol_table, ): if callback_detail is None: detail = renderers.NotApplicableValue() diff --git a/volatility3/framework/symbols/windows/extensions/callbacks.py b/volatility3/framework/symbols/windows/extensions/callbacks.py new file mode 100644 index 000000000..f894644db --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/callbacks.py @@ -0,0 +1,51 @@ +import logging + +from volatility3.framework import exceptions, objects +from volatility3.framework.symbols.windows.extensions import pool + +vollog = logging.getLogger(__name__) + + +class _SHUTDOWN_PACKET(objects.StructType, pool.ExecutiveObject): + """Class for _SHUTDOWN_PACKET objects found in IoSh pools. + + This class serves as a base class for all pooled shutdown callback packets. + + It exposes a function which sanity-checks structure members. + """ + + def is_valid(self) -> bool: + """ + Perform some checks. + """ + try: + if not ( + self.Entry.Flink.is_readable() + and self.Entry.Blink.is_readable() + and self.DeviceObject.is_readable() + ): + return False + + device = self.DeviceObject + if not device or not (device.DriverObject.DriverStart % 0x1000 == 0): + vollog.debug( + f"callback obj 0x{self.vol.offset:x} invalid due to invalid device object" + ) + return False + + except exceptions.InvalidAddressException: + vollog.debug( + f"callback obj 0x{self.vol.offset:x} invalid due to invalid address access" + ) + return False + + try: + header = device.get_object_header() + valid = header.NameInfo.Name == "Device" + return valid + except ValueError: + vollog.debug(f"Could not get NameInfo for object at 0x{self.vol.offset:x}") + return False + + +class_types_x86 = {"_SHUTDOWN_PACKET": _SHUTDOWN_PACKET} From 4d6f4bba0d5127f4a78074266091cf03b808b0f9 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 11 Mar 2024 21:48:58 -0500 Subject: [PATCH 081/263] Windows: Add version to driverirp.DriverIrp plugin This plugin class was missing a `_version` attribute, so I added one and set it to (1, 0, 0). --- volatility3/framework/plugins/windows/driverirp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index 433c61ca2..c3eb7c5c1 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -44,6 +44,7 @@ class DriverIrp(interfaces.plugins.PluginInterface): """List IRPs for drivers in a particular windows memory image.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls): From b36fd84ab57114c1250df57f3017937ba302d5ea Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 4 Jul 2024 13:07:17 +1000 Subject: [PATCH 082/263] Handle virtual_process_from_physical() invalid address exceptions to prevent psscan from breaking --- .../framework/plugins/windows/psscan.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 5634298a8..5ce470cd8 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -266,20 +266,28 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if proc.vol.layer_name == kernel.layer_name: vproc = proc else: - vproc = self.virtual_process_from_physical( - self.context, kernel.layer_name, kernel.symbol_table_name, proc + try: + vproc = self.virtual_process_from_physical( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + proc, + ) + except exceptions.PagedInvalidAddressException: + vproc = None + + file_output = "Error outputting file" + if vproc: + file_handle = pslist.PsList.process_dump( + self.context, + kernel.symbol_table_name, + pe_table_name, + vproc, + self.open, ) - file_handle = pslist.PsList.process_dump( - self.context, - kernel.symbol_table_name, - pe_table_name, - vproc, - self.open, - ) - file_output = "Error outputting file" - if file_handle: - file_output = file_handle.preferred_filename + if file_handle: + file_output = file_handle.preferred_filename if not self.config["physical"]: offset = proc.vol.offset From fcc9eab471ac050670d255bc5f1fba53862b6cf8 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 4 Jul 2024 13:09:12 +1000 Subject: [PATCH 083/263] Improve smear protection on windows processes validation --- .../symbols/windows/extensions/__init__.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index a8fa7b2ff..ae855d6c6 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -22,7 +22,7 @@ from volatility3.framework.interfaces.objects import ObjectInterface from volatility3.framework.layers import intel from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic -from volatility3.framework.symbols.windows.extensions import kdbg, pe, pool +from volatility3.framework.symbols.windows.extensions import pool vollog = logging.getLogger(__name__) @@ -611,11 +611,22 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): ctime = self.get_create_time() if not isinstance(ctime, datetime.datetime): + # A process must have a creation time return False if not (1998 < ctime.year < 2030): return False + etime = self.get_exit_time() + if isinstance(etime, datetime.datetime): + if not (1998 < etime.year < 2030): + return False + + # Exit time, if available, must be after the creation time + # At this point, we are sure both are datetimes, so let's compare them + if ctime > etime: + return False + # NT pids are divisible by 4 if self.UniqueProcessId % 4 != 0: return False @@ -633,7 +644,11 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): if dtb & ~0xFFF == 0: return False - ## TODO: we can also add the thread Flink and Blink tests if necessary + # Quick smear test on thread Flink and Blink + kernel = 0x80000000 # Yes, it's a quick test + list_head = self.ThreadListHead + if list_head.Flink < kernel or list_head.Blink < kernel: + return False except exceptions.InvalidAddressException: return False From 52e6812d390485403740af4ac526e07d39e86e10 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 4 Jul 2024 14:03:59 +1000 Subject: [PATCH 084/263] Fix hexdump text render. Set default to 16 bytes width --- volatility3/cli/text_renderer.py | 33 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index b0ba6baa7..ab9e44141 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -25,7 +25,7 @@ except ImportError: vollog.debug("Disassembly library capstone not found") -def hex_bytes_as_text(value: bytes) -> str: +def hex_bytes_as_text(value: bytes, width: int = 16) -> str: """Renders HexBytes as text. Args: @@ -36,19 +36,24 @@ def hex_bytes_as_text(value: bytes) -> str: """ if not isinstance(value, bytes): raise TypeError(f"hex_bytes_as_text takes bytes not: {type(value)}") - ascii = [] - hex = [] - count = 0 - output = "" - for byte in value: - hex.append(f"{byte:02x}") - ascii.append(chr(byte) if 0x20 < byte <= 0x7E else ".") - if (count % 8) == 7: - output += "\n" - output += " ".join(hex[count - 7 : count + 1]) - output += "\t" - output += "".join(ascii[count - 7 : count + 1]) - count += 1 + + printables = "" + output = "\n" + for count, byte in enumerate(value): + output += f"{byte:02x} " + char = chr(byte) + printables += char if 0x20 <= byte <= 0x7E else "." + if count % width == width - 1: + output += printables + if count < len(value) - 1: + output += "\n" + printables = "" + + # Handle leftovers when the lenght is not mutiple of width + if printables: + output += " " * (width - len(printables)) + output += printables + return output From 43fa84d2d1220d01826dd86c30138e3649d8806c Mon Sep 17 00:00:00 2001 From: Hannah Sarkey Date: Thu, 4 Jul 2024 21:33:55 -0400 Subject: [PATCH 085/263] Update dlllist.py to have more detailed --base description --- volatility3/framework/plugins/windows/dlllist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 4f50bdda5..d48a53663 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -61,7 +61,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), requirements.IntRequirement( name="base", - description="Specify a base address", + description="Specify a base virtual address in process memory", optional=True, ), requirements.BooleanRequirement( From 96a382ed525bb041cbe4fb0917e9af07160ea62a Mon Sep 17 00:00:00 2001 From: atcuno Date: Fri, 5 Jul 2024 09:33:34 -0500 Subject: [PATCH 086/263] Further type information --- .../plugins/windows/hollowprocesses.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index 3158667ac..69fa94f06 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import NamedTuple, Dict +from typing import NamedTuple, Dict, Generator from volatility3.framework import interfaces, exceptions, constants from volatility3.framework import renderers @@ -137,7 +137,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: return None - def _check_load_address(self, proc, _, __): + def _check_load_address(self, proc, _, __) -> Generator[str, None, None]: """ Detects when the image base in the PEB, which is writable by process malware, does not match the section base address - whose value lives in kernel memory. @@ -150,7 +150,9 @@ class HollowProcesses(interfaces.plugins.PluginInterface): image_base, proc.SectionBaseAddress ) - def _check_exe_protection(self, proc, vads, __): + def _check_exe_protection( + self, proc, vads: Dict[int, VadData], __ + ) -> Generator[str, None, None]: """ Legitimately mapped application executables and DLLs will have a VAD present and its initial protection will be @@ -172,7 +174,9 @@ class HollowProcesses(interfaces.plugins.PluginInterface): vads[base].protection, base, vads[base].path ) - def _check_dlls_protection(self, _, vads, dlls): + def _check_dlls_protection( + self, _, vads: Dict[int, VadData], dlls: Dict[int, DLLData] + ) -> Generator[str, None, None]: for dll_base in dlls: # could be malicious but triggers too many FPs from smear if dll_base not in vads: @@ -192,9 +196,6 @@ class HollowProcesses(interfaces.plugins.PluginInterface): ] for proc in procs: - proc_name = utility.array_to_string(proc.ImageFileName) - pid = proc.UniqueProcessId - # smear and/or terminated process dlls = self._get_dlls_map(proc) if len(dlls) < 3: @@ -204,6 +205,9 @@ class HollowProcesses(interfaces.plugins.PluginInterface): if len(vads) < 5: continue + proc_name = utility.array_to_string(proc.ImageFileName) + pid = proc.UniqueProcessId + for check in checks: for note in check(proc, vads, dlls): yield 0, ( From 4938195113613bd262f21d24886d674ae3c27d4d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 6 Jul 2024 17:04:51 +0100 Subject: [PATCH 087/263] Implement the fix that was reverted as part of #1162. --- volatility3/framework/layers/intel.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 75e561b33..b969b952b 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -180,7 +180,9 @@ class Intel(linear.LinearlyMappedLayer): position = self._initial_position entry = self._initial_entry - if self.minimum_address > offset > self.maximum_address: + if not ( + self.minimum_address <= (offset & self.address_mask) <= self.maximum_address + ): raise exceptions.PagedInvalidAddressException( self.name, offset, From bb4cfb6e00a9457081ebf4242c0f18996c4c1d08 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 7 Jul 2024 15:13:06 +0100 Subject: [PATCH 088/263] Initial draft of complete-vad vadyarascan --- .../framework/plugins/windows/vadyarascan.py | 52 ++++++++++++------- volatility3/framework/plugins/yarascan.py | 15 ++++-- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index d795818e9..81530ec67 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -2,10 +2,11 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import io import logging from typing import Iterable, List, Tuple -from volatility3.framework import interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan @@ -18,7 +19,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -32,11 +33,8 @@ class VadYaraScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) - ), requirements.PluginRequirement( - name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) + name="yarascan", plugin=yarascan.YaraScan, version=(1, 3, 0) ), requirements.ListRequirement( name="pid", @@ -59,6 +57,8 @@ class VadYaraScan(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + sanity_check = 0x1000 * 0x1000 * 0x1000 + for task in pslist.PsList.list_processes( context=self.context, layer_name=kernel.layer_name, @@ -67,18 +67,34 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] - for offset, rule_name, name, value in layer.scan( - context=self.context, - scanner=yarascan.YaraScanner(rules=rules), - sections=self.get_vad_maps(task), - ): - yield 0, ( - format_hints.Hex(offset), - task.UniqueProcessId, - rule_name, - name, - value, - ) + for start, end in self.get_vad_maps(task): + size = end - start + if size > sanity_check: + vollog.warn( + f"VAD at 0x{start:x} over sanity-check size, not scanning" + ) + continue + + for match in rules.match(data=layer.read(start, end - start, True)): + if yarascan.YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield 0, ( + format_hints.Hex(instance.offset + start), + task.UniqueProcessId, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield 0, ( + format_hints.Hex(offset + start), + task.UniqueProcessId, + match.rule, + name, + value, + ) @staticmethod def get_vad_maps( diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 11c708607..887b6027c 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -2,10 +2,11 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import io import logging from typing import Any, Dict, Iterable, List, Tuple -from volatility3.framework import interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import resources @@ -43,7 +44,7 @@ class YaraScanner(interfaces.layers.ScannerInterface): self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: for match in self._rules.match(data=data): - if self.st_object: + if YaraScan.yara_returns_instances(): for match_string in match.strings: for instance in match_string.instances: yield ( @@ -61,7 +62,7 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 2, 0) + _version = (1, 3, 0) # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string # or something that makes more sense @@ -119,6 +120,14 @@ class YaraScan(plugins.PluginInterface): ), ] + @classmethod + def yara_returns_instances(self) -> bool: + st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( + 4, + 3, + ) + return st_object + @classmethod def process_yara_options(cls, config: Dict[str, Any]): rules = None From ac15840fd817026d64959b51a63163e8d9bc5261 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 7 Jul 2024 15:23:48 +0100 Subject: [PATCH 089/263] Windows: Fix up yara plugin code scanning warnings --- volatility3/framework/plugins/windows/vadyarascan.py | 3 +-- volatility3/framework/plugins/yarascan.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 81530ec67..4b00fee57 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -2,11 +2,10 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import io import logging from typing import Iterable, List, Tuple -from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 887b6027c..496ec1844 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -2,11 +2,10 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import io import logging from typing import Any, Dict, Iterable, List, Tuple -from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import resources @@ -121,7 +120,7 @@ class YaraScan(plugins.PluginInterface): ] @classmethod - def yara_returns_instances(self) -> bool: + def yara_returns_instances(cls) -> bool: st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( 4, 3, From b35169d0bc3e802ced2d81534ac4452ac64f796b Mon Sep 17 00:00:00 2001 From: Brandon Barnacle Date: Thu, 20 Jun 2024 16:00:04 -0400 Subject: [PATCH 090/263] Windows: Bug fixes and additions to modules and modscan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This pull request adds several changes to the modules and modscan plugins that fix bugs, unify options and operations, and allow for filtering by name and base address in both plugins. Previously, modscan and modules had a substantial amount of partially duplicated code, which led to bugs, such as: Only one allowed filtering by --name Only one cycled through the session layers to find the one hosting the kernel module With the new inheritance and combined implementation, the correct and complete implementation applies to both. This PR also addresses the feedback in https://github.com/volatilityfoundation/volatility3/pull/1099. In this PR, --name and –base apply to the processing of modules and determine both which plugins are displayed in output as well as which are processed for extraction. This fixes the issue of using --filters, which only filters the final command line output, but leads to every module being extracted. --- .../framework/plugins/windows/modscan.py | 178 +++--------------- .../framework/plugins/windows/modules.py | 104 ++++++---- 2 files changed, 90 insertions(+), 192 deletions(-) diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index 99fadac07..3a786d2ca 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -9,16 +9,20 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import poolscanner, dlllist, pslist +from volatility3.plugins.windows import poolscanner, dlllist, pslist, modules vollog = logging.getLogger(__name__) -class ModScan(interfaces.plugins.PluginInterface): +class ModScan(modules.Modules): """Scans for modules present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._enumeration_method = self.scan_modules @classmethod def get_requirements(cls): @@ -31,6 +35,9 @@ class ModScan(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="poolscanner", component=poolscanner.PoolScanner, version=(1, 0, 0) ), + requirements.VersionRequirement( + name="modules", component=modules.Modules, version=(1, 1, 0) + ), requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), @@ -43,6 +50,17 @@ class ModScan(interfaces.plugins.PluginInterface): default=False, optional=True, ), + requirements.IntRequirement( + name="base", + description="Extract a single module with BASE address", + optional=True, + ), + requirements.StringRequirement( + name="name", + description="module name/sub string", + optional=True, + default=None, + ), ] @classmethod @@ -72,157 +90,3 @@ class ModScan(interfaces.plugins.PluginInterface): ): _constraint, mem_object, _header = result yield mem_object - - @classmethod - def get_session_layers( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - pids: List[int] = None, - ) -> Generator[str, None, None]: - """Build a cache of possible virtual layers, in priority starting with - the primary/kernel layer. Then keep one layer per session by cycling - through the process list. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - symbol_table: The name of the table containing the kernel symbols - pids: A list of process identifiers to include exclusively or None for no filter - - Returns: - A list of session layer names - """ - seen_ids: List[interfaces.objects.ObjectInterface] = [] - filter_func = pslist.PsList.create_pid_filter(pids or []) - - for proc in pslist.PsList.list_processes( - context=context, - layer_name=layer_name, - symbol_table=symbol_table, - filter_func=filter_func, - ): - proc_id = "Unknown" - try: - proc_id = proc.UniqueProcessId - proc_layer_name = proc.add_process_layer() - - # create the session space object in the process' own layer. - # not all processes have a valid session pointer. - session_space = context.object( - symbol_table + constants.BANG + "_MM_SESSION_SPACE", - layer_name=layer_name, - offset=proc.Session, - ) - - if session_space.SessionId in seen_ids: - continue - - except exceptions.InvalidAddressException: - vollog.log( - constants.LOGLEVEL_VVV, - "Process {} does not have a valid Session or a layer could not be constructed for it".format( - proc_id - ), - ) - continue - - # save the layer if we haven't seen the session yet - seen_ids.append(session_space.SessionId) - yield proc_layer_name - - @classmethod - def find_session_layer( - cls, - context: interfaces.context.ContextInterface, - session_layers: Iterable[str], - base_address: int, - ): - """Given a base address and a list of layer names, find a layer that - can access the specified address. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - symbol_table: The name of the table containing the kernel symbols - session_layers: A list of session layer names - base_address: The base address to identify the layers that can access it - - Returns: - Layer name or None if no layers that contain the base address can be found - """ - - for layer_name in session_layers: - if context.layers[layer_name].is_valid(base_address): - return layer_name - - return None - - def _generator(self): - kernel = self.context.modules[self.config["kernel"]] - - session_layers = list( - self.get_session_layers( - self.context, kernel.layer_name, kernel.symbol_table_name - ) - ) - pe_table_name = intermed.IntermediateSymbolTable.create( - self.context, self.config_path, "windows", "pe", class_types=pe.class_types - ) - - for mod in self.scan_modules( - self.context, kernel.layer_name, kernel.symbol_table_name - ): - try: - BaseDllName = mod.BaseDllName.get_string() - except exceptions.InvalidAddressException: - BaseDllName = "" - - try: - FullDllName = mod.FullDllName.get_string() - except exceptions.InvalidAddressException: - FullDllName = "" - - file_output = "Disabled" - if self.config["dump"]: - session_layer_name = self.find_session_layer( - self.context, session_layers, mod.DllBase - ) - file_output = f"Cannot find a viable session layer for {mod.DllBase:#x}" - if session_layer_name: - file_handle = dlllist.DllList.dump_pe( - self.context, - pe_table_name, - mod, - self.open, - layer_name=session_layer_name, - ) - file_output = "Error outputting file" - if file_handle: - file_output = file_handle.preferred_filename - - yield ( - 0, - ( - format_hints.Hex(mod.vol.offset), - format_hints.Hex(mod.DllBase), - format_hints.Hex(mod.SizeOfImage), - BaseDllName, - FullDllName, - file_output, - ), - ) - - def run(self): - return renderers.TreeGrid( - [ - ("Offset", format_hints.Hex), - ("Base", format_hints.Hex), - ("Size", format_hints.Hex), - ("Name", str), - ("Path", str), - ("File output", str), - ], - self._generator(), - ) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 7dabf6954..8aea3dd45 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -22,6 +22,10 @@ class Modules(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 1, 0) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._enumeration_method = self.list_modules + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ @@ -42,6 +46,11 @@ class Modules(interfaces.plugins.PluginInterface): default=False, optional=True, ), + requirements.IntRequirement( + name="base", + description="Extract a single module with BASE address", + optional=True, + ), requirements.StringRequirement( name="name", description="module name/sub string", @@ -50,49 +59,74 @@ class Modules(interfaces.plugins.PluginInterface): ), ] + def dump_module(self, session_layers, pe_table_name, mod): + session_layer_name = self.find_session_layer( + self.context, session_layers, mod.DllBase + ) + file_output = f"Cannot find a viable session layer for {mod.DllBase:#x}" + if session_layer_name: + file_handle = dlllist.DllList.dump_pe( + self.context, + pe_table_name, + mod, + self.open, + layer_name=session_layer_name, + ) + file_output = "Error outputting file" + if file_handle: + file_output = file_handle.preferred_filename + + return file_output + + def process_module(self, session_layers, pe_table_name, mod): + if self.config["base"] and self.config["base"] != mod.DllBase: + return None + + try: + BaseDllName = mod.BaseDllName.get_string() + except exceptions.InvalidAddressException: + BaseDllName = "" + + if self.config["name"] and self.config["name"] not in BaseDllName: + return None + + try: + FullDllName = mod.FullDllName.get_string() + except exceptions.InvalidAddressException: + FullDllName = "" + + file_output = "Disabled" + if self.config["dump"]: + file_output = self.dump_module(session_layers, pe_table_name, mod) + + return ( + format_hints.Hex(mod.vol.offset), + format_hints.Hex(mod.DllBase), + format_hints.Hex(mod.SizeOfImage), + BaseDllName, + FullDllName, + file_output, + ) + def _generator(self): kernel = self.context.modules[self.config["kernel"]] + pe_table_name = intermed.IntermediateSymbolTable.create( self.context, self.config_path, "windows", "pe", class_types=pe.class_types ) - for mod in self.list_modules( + session_layers = list( + self.get_session_layers( + self.context, kernel.layer_name, kernel.symbol_table_name + ) + ) + + for mod in self._enumeration_method( self.context, kernel.layer_name, kernel.symbol_table_name ): - try: - BaseDllName = mod.BaseDllName.get_string() - except exceptions.InvalidAddressException: - BaseDllName = "" - - try: - FullDllName = mod.FullDllName.get_string() - except exceptions.InvalidAddressException: - FullDllName = "" - - if self.config["name"] and self.config["name"] not in BaseDllName: - continue - - file_output = "Disabled" - if self.config["dump"]: - file_handle = dlllist.DllList.dump_pe( - self.context, pe_table_name, mod, self.open - ) - file_output = "Error outputting file" - if file_handle: - file_handle.close() - file_output = file_handle.preferred_filename - - yield ( - 0, - ( - format_hints.Hex(mod.vol.offset), - format_hints.Hex(mod.DllBase), - format_hints.Hex(mod.SizeOfImage), - BaseDllName, - FullDllName, - file_output, - ), - ) + record = self.process_module(session_layers, pe_table_name, mod) + if record: + yield (0, record) @classmethod def get_session_layers( From 8d60e9160f231460c20bde89488ceecc03814aec Mon Sep 17 00:00:00 2001 From: Brandon Barnacle Date: Mon, 8 Jul 2024 09:34:30 -0400 Subject: [PATCH 091/263] PR comment changes. --- .../framework/plugins/windows/modscan.py | 9 +- .../framework/plugins/windows/modules.py | 89 ++++++++++--------- 2 files changed, 48 insertions(+), 50 deletions(-) diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index 3a786d2ca..98546bc9c 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -2,13 +2,10 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import Iterable, List, Generator +from typing import Iterable -from volatility3.framework import renderers, interfaces, exceptions, constants +from volatility3.framework import interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import poolscanner, dlllist, pslist, modules vollog = logging.getLogger(__name__) @@ -36,7 +33,7 @@ class ModScan(modules.Modules): name="poolscanner", component=poolscanner.PoolScanner, version=(1, 0, 0) ), requirements.VersionRequirement( - name="modules", component=modules.Modules, version=(1, 1, 0) + name="modules", component=modules.Modules, version=(2, 0, 0) ), requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 8aea3dd45..79eea1cd7 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -4,9 +4,7 @@ import logging from typing import List, Iterable, Generator -from volatility3.framework import constants -from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers +from volatility3.framework import exceptions, interfaces, constants, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed @@ -20,7 +18,7 @@ class Modules(interfaces.plugins.PluginInterface): """Lists the loaded kernel modules.""" _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -78,55 +76,58 @@ class Modules(interfaces.plugins.PluginInterface): return file_output - def process_module(self, session_layers, pe_table_name, mod): - if self.config["base"] and self.config["base"] != mod.DllBase: - return None - - try: - BaseDllName = mod.BaseDllName.get_string() - except exceptions.InvalidAddressException: - BaseDllName = "" - - if self.config["name"] and self.config["name"] not in BaseDllName: - return None - - try: - FullDllName = mod.FullDllName.get_string() - except exceptions.InvalidAddressException: - FullDllName = "" - - file_output = "Disabled" - if self.config["dump"]: - file_output = self.dump_module(session_layers, pe_table_name, mod) - - return ( - format_hints.Hex(mod.vol.offset), - format_hints.Hex(mod.DllBase), - format_hints.Hex(mod.SizeOfImage), - BaseDllName, - FullDllName, - file_output, - ) - def _generator(self): kernel = self.context.modules[self.config["kernel"]] - pe_table_name = intermed.IntermediateSymbolTable.create( - self.context, self.config_path, "windows", "pe", class_types=pe.class_types - ) + pe_table_name = None + session_layers = None - session_layers = list( - self.get_session_layers( - self.context, kernel.layer_name, kernel.symbol_table_name + if self.config["dump"]: + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "windows", + "pe", + class_types=pe.class_types, + ) + + session_layers = list( + self.get_session_layers( + self.context, kernel.layer_name, kernel.symbol_table_name + ) ) - ) for mod in self._enumeration_method( self.context, kernel.layer_name, kernel.symbol_table_name ): - record = self.process_module(session_layers, pe_table_name, mod) - if record: - yield (0, record) + if self.config["base"] and self.config["base"] != mod.DllBase: + continue + + try: + BaseDllName = mod.BaseDllName.get_string() + except exceptions.InvalidAddressException: + BaseDllName = interfaces.renderers.BaseAbsentValue() + + if self.config["name"] and self.config["name"] not in BaseDllName: + continue + + try: + FullDllName = mod.FullDllName.get_string() + except exceptions.InvalidAddressException: + FullDllName = interfaces.renderers.BaseAbsentValue() + + file_output = "Disabled" + if self.config["dump"]: + file_output = self.dump_module(session_layers, pe_table_name, mod) + + yield 0, ( + format_hints.Hex(mod.vol.offset), + format_hints.Hex(mod.DllBase), + format_hints.Hex(mod.SizeOfImage), + BaseDllName, + FullDllName, + file_output, + ) @classmethod def get_session_layers( From c3cf319f7e44161ef6fd6537787de70daf71d126 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 12 Jul 2024 09:13:08 +0100 Subject: [PATCH 092/263] Windows: remove size from filescan output as it is not the file size but the object size --- volatility3/framework/plugins/windows/filescan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/filescan.py b/volatility3/framework/plugins/windows/filescan.py index 0f68f39d4..e21b5f518 100644 --- a/volatility3/framework/plugins/windows/filescan.py +++ b/volatility3/framework/plugins/windows/filescan.py @@ -13,7 +13,7 @@ from volatility3.plugins.windows import poolscanner class FileScan(interfaces.plugins.PluginInterface): """Scans for file objects present in a particular windows memory image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 0, 1) @classmethod def get_requirements(cls): @@ -67,10 +67,10 @@ class FileScan(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: continue - yield (0, (format_hints.Hex(fileobj.vol.offset), file_name, fileobj.Size)) + yield (0, (format_hints.Hex(fileobj.vol.offset), file_name)) def run(self): return renderers.TreeGrid( - [("Offset", format_hints.Hex), ("Name", str), ("Size", int)], + [("Offset", format_hints.Hex), ("Name", str)], self._generator(), ) From e5a530f08acc198939e3fa90616fb12062da34e7 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 12 Jul 2024 10:20:44 +0100 Subject: [PATCH 093/263] Windows: revert required framework version for filescan plugin --- volatility3/framework/plugins/windows/filescan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/filescan.py b/volatility3/framework/plugins/windows/filescan.py index e21b5f518..34c0c60d3 100644 --- a/volatility3/framework/plugins/windows/filescan.py +++ b/volatility3/framework/plugins/windows/filescan.py @@ -13,7 +13,7 @@ from volatility3.plugins.windows import poolscanner class FileScan(interfaces.plugins.PluginInterface): """Scans for file objects present in a particular windows memory image.""" - _required_framework_version = (2, 0, 1) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): From d4585be7cf4b6fbce85a6fa7a5b448b604036f86 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jul 2024 15:21:08 +0100 Subject: [PATCH 094/263] Linux: Update vmayarascan to scan complete VMA blocks --- .../framework/plugins/linux/vmayarascan.py | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index eda0d7dca..3edbe7784 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -33,9 +33,6 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) ), - requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) - ), requirements.ModuleRequirement( name="kernel", description="Linux kernel", @@ -69,19 +66,29 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): # get the proc_layer object from the context proc_layer = self.context.layers[proc_layer_name] - # scan the process layer with the yarascanner - for offset, rule_name, name, value in proc_layer.scan( - context=self.context, - scanner=yarascan.YaraScanner(rules=rules), - sections=self.get_vma_maps(task), - ): - yield 0, ( - format_hints.Hex(offset), - task.tgid, - rule_name, - name, - value, - ) + for start, end in self.get_vma_maps(task): + for match in rules.match( + data=proc_layer.read(start, end - start, True) + ): + if yarascan.YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield 0, ( + format_hints.Hex(instance.offset + start), + task.UniqueProcessId, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield 0, ( + format_hints.Hex(offset + start), + task.tgid, + match.rule, + name, + value, + ) @staticmethod def get_vma_maps( From 61e3e894eea28a8e0368c43f0fecb1b5031e0d2b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jul 2024 19:04:06 +0100 Subject: [PATCH 095/263] Core: Bump the version and minium python to support pip --- README.md | 2 +- pyproject.toml | 2 +- volatility3/framework/__init__.py | 2 +- volatility3/framework/constants/_version.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d303d2dff..886790df8 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.7.3 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/pyproject.toml b/pyproject.toml index 98c562a90..207f762cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, ] -requires-python = ">=3.7.0" +requires-python = ">=3.7.3" license = { text = "VSL" } dynamic = ["dependencies", "optional-dependencies", "version"] diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index feb97810b..74db773cf 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 7, 0) +required_python_version = (3, 7, 3) if ( sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 37c115431..21c339a6e 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 7 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From ac9236cb436d40a156f94b7aeab63afcc545663d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 15 Jul 2024 19:59:41 +1000 Subject: [PATCH 096/263] linux.netfilter.Netfilter: Add LinuxUtilities version requirement check on AbstractNetfilter --- .../framework/plugins/linux/netfilter.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 60a71f798..e0fbcbd70 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -11,6 +11,7 @@ from volatility3.framework import ( constants, interfaces, renderers, + exceptions, ) from volatility3.framework.renderers import format_hints from volatility3.framework.configuration import requirements @@ -80,6 +81,16 @@ class AbstractNetfilter(ABC): self.list_head_size = self.vmlinux.get_type("list_head").size modules = lsmod.Lsmod.list_modules(context, kernel_module_name) + + linuxutils_required_version = Netfilter._required_linuxutils_version + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( context, kernel_module_name, modules ) @@ -658,6 +669,8 @@ class Netfilter(interfaces.plugins.PluginInterface): _version = (1, 0, 0) + _required_linuxutils_version = (2, 1, 0) + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ @@ -670,7 +683,9 @@ class Netfilter(interfaces.plugins.PluginInterface): name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), requirements.VersionRequirement( - name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) + name="linuxutils", + component=linux.LinuxUtilities, + version=cls._required_linuxutils_version, ), ] From a6d8ba8bfaeeceebd564e35e84ae0e288f69871e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 15 Jul 2024 20:22:40 +1000 Subject: [PATCH 097/263] windows eprocess validation: Make the creation and exit year checks dynamic --- volatility3/framework/symbols/windows/extensions/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ae855d6c6..93c19599e 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -614,12 +614,13 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): # A process must have a creation time return False - if not (1998 < ctime.year < 2030): + current_year = datetime.datetime.now().year + if not (1998 < ctime.year < current_year + 10): return False etime = self.get_exit_time() if isinstance(etime, datetime.datetime): - if not (1998 < etime.year < 2030): + if not (1998 < etime.year < current_year + 10): return False # Exit time, if available, must be after the creation time From cb6929163b31726d811f7e825bcfad8690170b9c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 15 Jul 2024 18:14:50 +0100 Subject: [PATCH 098/263] Windows: Fix vadyarascan sanity check and bad documentation --- volatility3/framework/plugins/windows/vadyarascan.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 4b00fee57..219faf028 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -56,7 +56,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - sanity_check = 0x1000 * 0x1000 * 0x1000 + sanity_check = 0x2000 * 0x1000 * 0x1000 for task in pslist.PsList.list_processes( context=self.context, @@ -66,15 +66,14 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] - for start, end in self.get_vad_maps(task): - size = end - start + for start, size in self.get_vad_maps(task): if size > sanity_check: vollog.warn( f"VAD at 0x{start:x} over sanity-check size, not scanning" ) continue - for match in rules.match(data=layer.read(start, end - start, True)): + for match in rules.match(data=layer.read(start, size, True)): if yarascan.YaraScan.yara_returns_instances(): for match_string in match.strings: for instance in match_string.instances: @@ -106,7 +105,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): task: The EPROCESS object of which to traverse the vad tree Returns: - An iterable of tuples containing start and end addresses for each descriptor + An iterable of tuples containing start and size for each descriptor """ vad_root = task.get_vad_root() for vad in vad_root.traverse(): From 55fe4ba47aece0882a0b5c690710cba1fa438989 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 15 Jul 2024 15:20:10 -0500 Subject: [PATCH 099/263] #118 - MR feedback --- .../framework/plugins/windows/kpcrs.py | 106 ++++++++++++++++++ .../framework/plugins/windows/timers.py | 86 +++----------- .../symbols/windows/extensions/__init__.py | 6 +- 3 files changed, 125 insertions(+), 73 deletions(-) create mode 100644 volatility3/framework/plugins/windows/kpcrs.py diff --git a/volatility3/framework/plugins/windows/kpcrs.py b/volatility3/framework/plugins/windows/kpcrs.py new file mode 100644 index 000000000..558ea844c --- /dev/null +++ b/volatility3/framework/plugins/windows/kpcrs.py @@ -0,0 +1,106 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterator, List, Tuple + +from volatility3.framework import ( + renderers, + interfaces, + constants, +) +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints + +vollog = logging.getLogger(__name__) + + +class KPCRs(interfaces.plugins.PluginInterface): + """Print KPCR structure for each processor""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + ] + + @classmethod + def list_kpcrs( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + layer_name: str, + symbol_table: str, + ) -> interfaces.objects.ObjectInterface: + """Returns the KPCR structure for each processor + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + kernel_module_name: The name of the kernel module on which to operate + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + + Returns: + The _KPCR structure for each processor + """ + + kernel = context.modules[kernel_module_name] + cpu_count_offset = kernel.get_symbol("KeNumberProcessors").address + cpu_count = kernel.object( + object_type="unsigned int", layer_name=layer_name, offset=cpu_count_offset + ) + processor_block = kernel.object( + object_type="pointer", + layer_name=layer_name, + offset=kernel.get_symbol("KiProcessorBlock").address, + ) + processor_pointers = utility.array_of_pointers( + context=context, + array=processor_block, + count=cpu_count, + subtype=symbol_table + constants.BANG + "_KPRCB", + ) + for pointer in processor_pointers: + kprcb = pointer.dereference() + reloff = kernel.get_type("_KPCR").relative_child_offset("Prcb") + kpcr = context.object( + symbol_table + constants.BANG + "_KPCR", + offset=kprcb.vol.offset - reloff, + layer_name=layer_name, + ) + yield kpcr + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config["kernel"]] + layer_name = kernel.layer_name + symbol_table = kernel.symbol_table_name + + for kpcr in self.list_kpcrs( + self.context, self.config["kernel"], layer_name, symbol_table + ): + yield ( + 0, + ( + format_hints.Hex(kpcr.vol.offset), + format_hints.Hex(kpcr.CurrentPrcb), + ), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("PRCB Offset", format_hints.Hex), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index ec8bc17e9..d49c28784 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -7,17 +7,15 @@ import logging from typing import Iterator, List, Tuple, Iterable from volatility3.framework import ( - layers, renderers, interfaces, constants, symbols, ) from volatility3.framework.configuration import requirements -from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.windows import versions -from volatility3.plugins.windows import ssdt +from volatility3.plugins.windows import ssdt, kpcrs vollog = logging.getLogger(__name__) @@ -39,73 +37,16 @@ class Timers(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) ), + requirements.PluginRequirement( + name="kpcrs", plugin=kpcrs.KPCRs, version=(1, 0, 0) + ), ] - @classmethod - def get_kernel_module( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - ): - """Returns the kernel module based on the layer and symbol_table""" - virtual_layer = context.layers[layer_name] - if not isinstance(virtual_layer, layers.intel.Intel): - raise TypeError("Virtual Layer is not an intel layer") - - kvo = virtual_layer.config["kernel_virtual_offset"] - - ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) - return ntkrnlmp - - @classmethod - def get_kpcrs( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - ) -> interfaces.objects.ObjectInterface: - """Returns the KPCR structure for each processor - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - symbol_table: The name of an existing symbol table containing the kernel symbols - config_path: The configuration path within the context of the symbol table to create - - Returns: - The _KPCR structure for each processor - """ - - ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table) - cpu_count_offset = ntkrnlmp.get_symbol("KeNumberProcessors").address - cpu_count = ntkrnlmp.object( - object_type="unsigned int", layer_name=layer_name, offset=cpu_count_offset - ) - processor_block = ntkrnlmp.object( - object_type="pointer", - layer_name=layer_name, - offset=ntkrnlmp.get_symbol("KiProcessorBlock").address, - ) - processor_pointers = utility.array_of_pointers( - context=context, - array=processor_block, - count=cpu_count, - subtype=symbol_table + constants.BANG + "_KPRCB", - ) - for pointer in processor_pointers: - kprcb = pointer.dereference() - reloff = ntkrnlmp.get_type("_KPCR").relative_child_offset("Prcb") - kpcr = context.object( - symbol_table + constants.BANG + "_KPCR", - offset=kprcb.vol.offset - reloff, - layer_name=layer_name, - ) - yield kpcr - @classmethod def list_timers( cls, context: interfaces.context.ContextInterface, + kernel_module_name: str, layer_name: str, symbol_table: str, ) -> Iterable[Tuple[str, int, str]]: @@ -113,21 +54,24 @@ class Timers(interfaces.plugins.PluginInterface): Args: context: The context to retrieve required elements (layers, symbol tables) from + kernel_module_name: The name of the kernel module on which to operate layer_name: The name of the layer on which to operate symbol_table: The name of the table containing the kernel symbols Yields: A _KTIMER entry """ - ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table) + kernel = context.modules[kernel_module_name] if versions.is_windows_7( context=context, symbol_table=symbol_table ) or versions.is_windows_8_or_later(context=context, symbol_table=symbol_table): # Starting with Windows 7, there is no more KiTimerTableListHead. The list is # at _KPCR.PrcbData.TimerTable.TimerEntries # See http://pastebin.com/FiRsGW3f - for kpcr in cls.get_kpcrs(context, layer_name, symbol_table): + for kpcr in kpcrs.KPCRs.list_kpcrs( + context, kernel_module_name, layer_name, symbol_table + ): if hasattr(kpcr.Prcb.TimerTable, "TableState"): for timer_entries in kpcr.Prcb.TimerTable.TimerEntries: for timer_entry in timer_entries: @@ -160,10 +104,10 @@ class Timers(interfaces.plugins.PluginInterface): # is an array of 256 _LIST_ENTRY for _KTIMERs. array_size = 256 - timer_table_list_head = ntkrnlmp.object( + timer_table_list_head = kernel.object( object_type="array", - offset=ntkrnlmp.get_symbol("KiTimerTableListHead").address, - subtype=ntkrnlmp.get_type("_LIST_ENTRY"), + offset=kernel.get_symbol("KiTimerTableListHead").address, + subtype=kernel.get_type("_LIST_ENTRY"), count=array_size, ) for table in timer_table_list_head: @@ -185,7 +129,9 @@ class Timers(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name ) - for timer in self.list_timers(self.context, layer_name, symbol_table): + for timer in self.list_timers( + self.context, self.config["kernel"], layer_name, symbol_table + ): if not timer.valid_type(): continue try: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b9eabcda6..86ea2febb 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1011,12 +1011,12 @@ class KTIMER(objects.StructType): def get_raw_dpc(self): """Returns the encoded DPC since it may not look like a pointer after encoding""" symbol_table_name = self.get_symbol_table_name() - ulonglong_type = self._context.symbol_space.get_type( - symbol_table_name + constants.BANG + "unsigned long long" + pointer_type = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "pointer" ) return self._context.object( - object_type=ulonglong_type, + object_type=pointer_type, layer_name=self.vol.layer_name, offset=self.Dpc.vol.offset, ) From 7e3615335c44f79b2b39e9f669a33df6f3edf2fc Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 15 Jul 2024 15:40:22 -0500 Subject: [PATCH 100/263] #1175 - config_path change --- volatility3/framework/plugins/windows/unloadedmodules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py index 1fd4914ac..0d88e96ff 100644 --- a/volatility3/framework/plugins/windows/unloadedmodules.py +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -10,6 +10,7 @@ from volatility3.framework import constants from volatility3.framework import interfaces, symbols from volatility3.framework import renderers from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import configuration from volatility3.framework.renderers import format_hints, conversion from volatility3.framework.symbols import intermed from volatility3.plugins import timeliner @@ -60,7 +61,7 @@ class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInt return intermed.IntermediateSymbolTable.create( context, - config_path, + configuration.path_join(config_path, "unloadedmodules"), "windows", symbol_filename, native_types=native_types, From 99cf48597abbc1126ef06731ed7836c04614514d Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 15 Jul 2024 15:46:27 -0500 Subject: [PATCH 101/263] #118 - use canonicalize for offset --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 86ea2febb..d5c3d3f96 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1052,7 +1052,7 @@ class KTIMER(objects.StructType): low_byte = (wait_never) & 0xFF entry = utility.rol(self.get_raw_dpc() ^ wait_never, low_byte) - swap_xor = self.vol.offset | 0xFFFF000000000000 + swap_xor = self._context.layers[self.vol.native_layer_name].canonicalize(self.vol.offset) entry = utility.bswap_64(entry ^ swap_xor) dpc = entry ^ wait_always From cf6c7fb1f40c4bfa1c60cc050cfd346014a854ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 16 Jul 2024 09:38:49 +1000 Subject: [PATCH 102/263] linux.netfilter.Netfilter: Add lsmod version requirement check on AbstractNetfilter --- volatility3/framework/plugins/linux/netfilter.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index e0fbcbd70..d392a8371 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -80,7 +80,14 @@ class AbstractNetfilter(ABC): self.ptr_size = self.vmlinux.get_type("pointer").size self.list_head_size = self.vmlinux.get_type("list_head").size - modules = lsmod.Lsmod.list_modules(context, kernel_module_name) + lsmod_required_version = Netfilter._required_lsmod_version + lsmod_current_version = lsmod.Lsmod._version + if not requirements.VersionRequirement.matches_required( + lsmod_required_version, lsmod_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.lsmod.Lsmod version not suitable: required {lsmod_required_version} found {lsmod_current_version}" + ) linuxutils_required_version = Netfilter._required_linuxutils_version linuxutils_current_version = linux.LinuxUtilities._version @@ -91,6 +98,7 @@ class AbstractNetfilter(ABC): f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" ) + modules = lsmod.Lsmod.list_modules(context, kernel_module_name) self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( context, kernel_module_name, modules ) @@ -670,6 +678,7 @@ class Netfilter(interfaces.plugins.PluginInterface): _version = (1, 0, 0) _required_linuxutils_version = (2, 1, 0) + _required_lsmod_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -680,7 +689,7 @@ class Netfilter(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + name="lsmod", plugin=lsmod.Lsmod, version=cls._required_lsmod_version ), requirements.VersionRequirement( name="linuxutils", From 43e22d72bdaa2cbb6366b1911a6ff38ced83394f Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Tue, 16 Jul 2024 09:23:06 -0500 Subject: [PATCH 103/263] #118 - black formatting --- volatility3/framework/symbols/windows/extensions/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d5c3d3f96..b333755f7 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1052,7 +1052,9 @@ class KTIMER(objects.StructType): low_byte = (wait_never) & 0xFF entry = utility.rol(self.get_raw_dpc() ^ wait_never, low_byte) - swap_xor = self._context.layers[self.vol.native_layer_name].canonicalize(self.vol.offset) + swap_xor = self._context.layers[self.vol.native_layer_name].canonicalize( + self.vol.offset + ) entry = utility.bswap_64(entry ^ swap_xor) dpc = entry ^ wait_always From be5423f786827ee4056b2c638b2d1214a419c47c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jul 2024 17:08:15 +0100 Subject: [PATCH 104/263] Windows: Fix up broken imports in info plugin --- volatility3/framework/plugins/windows/info.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/info.py b/volatility3/framework/plugins/windows/info.py index 100a677c2..137d29c22 100644 --- a/volatility3/framework/plugins/windows/info.py +++ b/volatility3/framework/plugins/windows/info.py @@ -10,7 +10,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.renderers import TreeGrid from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows import extensions +from volatility3.framework.symbols.windows.extensions import kdbg, pe class Info(plugins.PluginInterface): @@ -94,16 +94,16 @@ class Info(plugins.PluginInterface): "windows", "kdbg", native_types=native_types, - class_types=extensions.kdbg.class_types, + class_types=kdbg.class_types, ) - kdbg = context.object( + kdbg_obj = context.object( kdbg_table_name + constants.BANG + "_KDDEBUGGER_DATA64", offset=ntkrnlmp.offset + kdbg_offset, layer_name=layer_name, ) - return kdbg + return kdbg_obj @classmethod def get_kuser_structure( @@ -173,7 +173,7 @@ class Info(plugins.PluginInterface): interfaces.configuration.path_join(config_path, "pe"), "windows", "pe", - class_types=extensions.pe.class_types, + class_types=pe.class_types, ) dos_header = context.object( From a5cf635fd8b81c56f14ee490bcf63e4e5aafd32d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jul 2024 17:31:44 +0100 Subject: [PATCH 105/263] Windows: Fix the vadyarascan sanity check --- volatility3/framework/plugins/windows/vadyarascan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 219faf028..dc318dd93 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -56,7 +56,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - sanity_check = 0x2000 * 0x1000 * 0x1000 + sanity_check = 1024 * 1024 * 1024 # 1 GB for task in pslist.PsList.list_processes( context=self.context, From a684e284ccca7675d4e3cd07c39679ecba703627 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 5 Jul 2024 15:45:46 -0500 Subject: [PATCH 106/263] Windows: Adds shimcache symbol files + extensions --- .../framework/symbols/windows/__init__.py | 1 + .../symbols/windows/extensions/__init__.py | 49 +- .../symbols/windows/extensions/shimcache.py | 278 ++++++++++ .../windows/shimcache/shimcache-2003-x64.json | 327 ++++++++++++ .../windows/shimcache/shimcache-2003-x86.json | 334 ++++++++++++ .../shimcache/shimcache-vista-x64.json | 334 ++++++++++++ .../shimcache/shimcache-vista-x86.json | 334 ++++++++++++ .../shimcache/shimcache-win10-x64.json | 371 ++++++++++++++ .../shimcache/shimcache-win10-x86.json | 371 ++++++++++++++ .../windows/shimcache/shimcache-win7-x64.json | 348 +++++++++++++ .../windows/shimcache/shimcache-win7-x86.json | 348 +++++++++++++ .../windows/shimcache/shimcache-win8-x64.json | 392 ++++++++++++++ .../windows/shimcache/shimcache-win8-x86.json | 386 ++++++++++++++ .../shimcache/shimcache-xp-sp2-x86.json | 485 ++++++++++++++++++ .../shimcache/shimcache-xp-sp3-x86.json | 485 ++++++++++++++++++ .../framework/symbols/windows/versions.py | 27 + 16 files changed, 4863 insertions(+), 7 deletions(-) create mode 100644 volatility3/framework/symbols/windows/extensions/shimcache.py create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-2003-x64.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-2003-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-vista-x64.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-vista-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win10-x64.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win10-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win7-x64.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win7-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win8-x64.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win8-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp2-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp3-x86.json diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index abf9f6da3..e43a2486b 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -17,6 +17,7 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("_KTHREAD", extensions.KTHREAD) self.set_type_class("_LIST_ENTRY", extensions.LIST_ENTRY) self.set_type_class("_EPROCESS", extensions.EPROCESS) + self.set_type_class("_ERESOURCE", extensions.ERESOURCE) self.set_type_class("_UNICODE_STRING", extensions.UNICODE_STRING) self.set_type_class("_EX_FAST_REF", extensions.EX_FAST_REF) self.set_type_class("_TOKEN", extensions.TOKEN) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 93c19599e..39b50e180 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -306,16 +306,19 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the private memory member") + @property + def Protection(self): + if self.has_member("u"): + return self.u.VadFlags.Protection + elif self.has_member("Core"): + return self.Core.u.VadFlags.Protection + else: + return None + def get_protection(self, protect_values, winnt_protections): """Get the VAD's protection constants as a string.""" - protect = None - - if self.has_member("u"): - protect = self.u.VadFlags.Protection - - elif self.has_member("Core"): - protect = self.Core.u.VadFlags.Protection + protect = self.Protection try: value = protect_values[protect] @@ -593,6 +596,38 @@ class UNICODE_STRING(objects.StructType): String = property(get_string) +class ERESOURCE(objects.StructType): + def is_valid(self) -> bool: + vollog.debug(f"Checking ERESOURCE Validity: {hex(self.vol.offset)}") + + if not self._context.layers[self.vol.layer_name].is_valid(self.vol.offset): + return False + + sym_table = self.get_symbol_table_name() + + waiters_valid = self.SharedWaiters == 0 or self._context.layers[ + self.vol.layer_name + ].is_valid( + self.SharedWaiters.vol.offset, + self._context.symbol_space.get_type( + sym_table + constants.BANG + "_KSEMAPHORE" + ).size, + ) + + try: + return ( + waiters_valid + and self.SystemResourcesList.Flink is not None + and self.SystemResourcesList.Blink is not None + and self.SystemResourcesList.Flink != self.SystemResourcesList.Blink + and self.SystemResourcesList.Flink.Blink == self.vol.offset + and self.SystemResourcesList.Blink.Flink == self.vol.offset + and self.NumberOfSharedWaiters == 0 + ) + except exceptions.InvalidAddressException: + return False + + class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): """A class for executive kernel processes objects.""" diff --git a/volatility3/framework/symbols/windows/extensions/shimcache.py b/volatility3/framework/symbols/windows/extensions/shimcache.py new file mode 100644 index 000000000..b84a7df6f --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/shimcache.py @@ -0,0 +1,278 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import struct +from datetime import datetime +from typing import Dict, Optional, Tuple, Union + +from volatility3.framework import constants, exceptions, interfaces, objects, renderers +from volatility3.framework.symbols.windows.extensions import conversion + +vollog = logging.getLogger(__name__) + + +class SHIM_CACHE_ENTRY(objects.StructType): + """Class for abstracting variations in the shimcache LRU list entry structure""" + + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + size: int, + members: Dict[str, Tuple[int, interfaces.objects.Template]], + ) -> None: + super().__init__(context, type_name, object_info, size, members) + self._exec_flag = None + self._file_path = None + self._file_size = None + self._last_modified = None + self._last_updated = None + + @property + def exec_flag(self) -> Union[bool, interfaces.renderers.BaseAbsentValue]: + """Checks if InsertFlags fields has been bitwise OR'd with a value of 2. + This behavior was observed when processes are created by CSRSS.""" + if self._exec_flag is not None: + return self._exec_flag + + if hasattr(self, "ListEntryDetail") and hasattr( + self.ListEntryDetail, "InsertFlags" + ): + self._exec_flag = self.ListEntryDetail.InsertFlags & 0x2 == 2 + + elif hasattr(self, "InsertFlags"): + self._exec_flag = self.InsertFlags & 0x2 == 2 + + elif hasattr(self, "ListEntryDetail") and hasattr( + self.ListEntryDetail, "BlobBuffer" + ): + blob_offset = self.ListEntryDetail.BlobBuffer + blob_size = self.ListEntryDetail.BlobSize + + if not self._context.layers[self.vol.native_layer_name].is_valid( + blob_offset, blob_size + ): + self._exec_flag = renderers.UnparsableValue() + + raw_flag = self._context.layers[self.vol.native_layer_name].read( + blob_offset, blob_size + ) + if not raw_flag: + self._exec_flag = renderers.UnparsableValue() + + try: + self._exec_flag = bool(struct.unpack(" Union[int, interfaces.renderers.BaseAbsentValue]: + if self._file_size is not None: + return self._file_size + try: + self._file_size = self.FileSize + if self._file_size < 0: + self._file_size = 0 + + except AttributeError: + self._file_size = renderers.NotApplicableValue() + except exceptions.InvalidAddressException: + self._file_size = renderers.UnreadableValue() + + return self._file_size + + @property + def last_modified(self) -> Union[datetime, interfaces.renderers.BaseAbsentValue]: + if self._last_modified is not None: + return self._last_modified + try: + self._last_modified = conversion.wintime_to_datetime( + self.ListEntryDetail.LastModified.QuadPart + ) + except AttributeError: + self._last_modified = conversion.wintime_to_datetime( + self.LastModified.QuadPart + ) + except exceptions.InvalidAddressException: + self._last_modified = renderers.UnreadableValue() + + return self._last_modified + + @property + def last_update(self) -> Union[datetime, interfaces.renderers.BaseAbsentValue]: + if self._last_updated is not None: + return self._last_updated + + try: + self._last_updated = conversion.wintime_to_datetime( + self.LastUpdate.QuadPart + ) + except AttributeError: + self._last_updated = renderers.NotApplicableValue() + + return self._last_updated + + @property + def file_path(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: + if self._file_path is not None: + return self._file_path + + if not hasattr(self.Path, "Buffer"): + return self.Path.cast( + "string", max_length=self.Path.vol.count, encoding="utf-16le" + ) + + try: + file_path_raw = ( + self._context.layers[self.vol.native_layer_name].read( + self.Path.Buffer, self.Path.Length + ) + or b"" + ) + self._file_path = file_path_raw.decode("utf-16", errors="replace") + except exceptions.InvalidAddressException: + self._file_path = renderers.UnreadableValue() + + return self._file_path + + def is_valid(self) -> bool: + """Shim cache validation is limited to ensuring that a subset of the + pointers in the LIST_ENTRY field are valid (similar to validation of + ERESOURCE)""" + + # shim entries on Windows XP do not have list entry attributes; in this case, + # perform a different set of validations + try: + if not hasattr(self, "ListEntry"): + return bool(self.last_modified and self.last_update and self.file_size) + + # on some platforms ListEntry.Blink is null, so this cannot be validated + if ( + self.ListEntry.Flink != 0 + and ( + self.ListEntry.Blink.dereference() + != self.ListEntry.Flink.dereference() + ) + and ( + self.ListEntry.Flink.Blink + == self.ListEntry.Flink.Blink.dereference().vol.offset + ) + ): + + return True + else: + return False + except exceptions.InvalidAddressException: + return False + + +class SHIM_CACHE_HANDLE(objects.StructType): + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + size: int, + members: Dict[str, Tuple[int, interfaces.objects.Template]], + ) -> None: + super().__init__(context, type_name, object_info, size, members) + + @property + def head(self) -> Optional[SHIM_CACHE_ENTRY]: + try: + if not self.eresource.is_valid(): + return None + except exceptions.InvalidAddressException: + return None + + rtl_avl_table = self._context.object( + self.get_symbol_table_name() + constants.BANG + "_RTL_AVL_TABLE", + self.vol.layer_name, + self.rtl_avl_table, + self.vol.native_layer_name, + ) + + if not self._context.layers[self.vol.layer_name].is_valid( + self.rtl_avl_table.vol.offset + ): + return None + + offset_head = rtl_avl_table.vol.offset + rtl_avl_table.vol.size + + head = self._context.object( + self.get_symbol_table_name() + constants.BANG + "SHIM_CACHE_ENTRY", + self.vol.layer_name, + offset_head, + ) + + if not head.is_valid(): + return None + + return head + + def is_valid(self, avl_section_start: int, avl_section_end: int) -> bool: + if self.vol.offset == 0: + return False + + vollog.debug(f"Checking SHIM_CACHE_HANDLE validity @ {hex(self.vol.offset)}") + + if not ( + self._context.layers[self.vol.layer_name].is_valid(self.vol.offset) + and self.eresource.is_valid() + and self.rtl_avl_table.is_valid(avl_section_start, avl_section_end) + and self.head + ): + return False + + return self.head.is_valid() + + +class RTL_AVL_TABLE(objects.StructType): + def is_valid(self, page_start: int, page_end: int) -> bool: + try: + if self.BalancedRoot.Parent != self.BalancedRoot.vol.offset: + vollog.debug( + f"RTL_AVL_TABLE @ {self.vol.offset} Invalid: Failed BalancedRoot parent equality check" + ) + return False + + elif self.AllocateRoutine < page_start or self.AllocateRoutine > page_end: + vollog.debug( + f"RTL_AVL_TABLE @ {self.vol.offset} Invalid: Failed AllocateRoutine range check" + ) + return False + + elif self.CompareRoutine < page_start or self.CompareRoutine > page_end: + vollog.debug( + f"RTL_AVL_TABLE @ {self.vol.offset} Invalid: Failed CompareRoutine range check" + ) + return False + + elif ( + (self.AllocateRoutine.vol.offset == self.CompareRoutine.vol.offset) + or (self.AllocateRoutine.vol.offset == self.FreeRoutine.vol.offset) + or (self.CompareRoutine.vol.offset == self.FreeRoutine.vol.offset) + ): + vollog.debug( + f"RTL_AVL_TABLE @ {self.vol.offset} Invalid: Failed (Compare|Allocate|Free)Routine uniqueness check" + ) + return False + + return True + except exceptions.InvalidAddressException: + return False + + +class_types = { + "SHIM_CACHE_HANDLE": SHIM_CACHE_HANDLE, + "SHIM_CACHE_ENTRY": SHIM_CACHE_ENTRY, + "_RTL_AVL_TABLE": RTL_AVL_TABLE, +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x64.json b/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x64.json new file mode 100644 index 000000000..d1540f12f --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x64.json @@ -0,0 +1,327 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 16 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 24 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 25 + } + }, + "kind": "struct", + "size": 32 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 40 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 44 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 48 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 56 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 64 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 72 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 80 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 88 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 104 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 16 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 16, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 32, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "FileSize": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 48 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x86.json new file mode 100644 index 000000000..e4739c4a0 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x86.json @@ -0,0 +1,334 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 8, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 16, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "FileSize": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "Padding": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 36 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x64.json b/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x64.json new file mode 100644 index 000000000..0c5183a4a --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x64.json @@ -0,0 +1,334 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 16 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 24 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 25 + } + }, + "kind": "struct", + "size": 32 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 40 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 44 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 48 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 56 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 64 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 72 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 80 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 88 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 104 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 16 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 16, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 32, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 44, + "type": { + "kind": "base", + "name": "unsigned int" + } + } + }, + "kind": "struct", + "size": 48 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x86.json new file mode 100644 index 000000000..91290b1ba --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x86.json @@ -0,0 +1,334 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 8, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 16, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 28, + "type": { + "kind": "base", + "name": "unsigned int" + } + } + }, + "kind": "struct", + "size": 36 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x64.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x64.json new file mode 100644 index 000000000..fe9593af3 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x64.json @@ -0,0 +1,371 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 16 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 24 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 25 + } + }, + "kind": "struct", + "size": 32 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 40 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 44 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 48 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 56 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 64 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 72 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 80 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 88 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 104 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 16 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "u1": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Path": { + "offset": 24, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "ListEntryDetail": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "SHIM_CACHE_ENTRY_DETAIL" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "SHIM_CACHE_ENTRY_DETAIL": { + "fields": { + "u1": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "LastModified": { + "offset": 8, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "BlobSize": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "u2": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "BlobBuffer": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + } + }, + "kind": "struct", + "size": 32 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x86.json new file mode 100644 index 000000000..26d493c37 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x86.json @@ -0,0 +1,371 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 4 + } + }, + "kind": "struct", + "size": 8 + }, + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "u1": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "Path": { + "offset": 12, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "ListEntryDetail": { + "offset": 20, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "SHIM_CACHE_ENTRY_DETAIL" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "SHIM_CACHE_ENTRY_DETAIL": { + "fields": { + "u1": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "InsertFlags": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "LastModified": { + "offset": 8, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "BlobSize": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "BlobBuffer": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 24 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x64.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x64.json new file mode 100644 index 000000000..eac5407ba --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x64.json @@ -0,0 +1,348 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 16 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 24 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 25 + } + }, + "kind": "struct", + "size": 32 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 40 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 44 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 48 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 56 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 64 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 72 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 80 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 88 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 104 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 16, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 32, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 44, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "BlobSize": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "BlobBuffer": { + "offset": 56, + "type": { + "kind": "base", + "name": "unsigned long long" + } + } + }, + "kind": "struct", + "size": 64 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x86.json new file mode 100644 index 000000000..423f7e255 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x86.json @@ -0,0 +1,348 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 4 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 8, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 16, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 28, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "BlobSize": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "BlobBuffer": { + "offset": 36, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 40 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x64.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x64.json new file mode 100644 index 000000000..a40c8d680 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x64.json @@ -0,0 +1,392 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 16 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 24 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 25 + } + }, + "kind": "struct", + "size": 32 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 40 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 44 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 48 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 56 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 64 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 72 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 80 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 88 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 104 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "u1": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Path": { + "offset": 24, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "u2": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "u3": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ListEntryDetail": { + "offset": 56, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "SHIM_CACHE_ENTRY_DETAIL" + } + } + } + }, + "kind": "struct", + "size": 64 + }, + "SHIM_CACHE_ENTRY_DETAIL": { + "fields": { + "LastModified": { + "offset": 0, + "type": { + "kind": "struct", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "BlobSize": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Padding": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "BlobBuffer": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long long" + } + } + }, + "kind": "struct", + "size": 40 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x86.json new file mode 100644 index 000000000..c3cee5feb --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x86.json @@ -0,0 +1,386 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 4 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "u1": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "u2": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "Path": { + "offset": 16, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "u3": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ListEntryDetail": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "SHIM_CACHE_ENTRY_DETAIL" + } + } + } + }, + "kind": "struct", + "size": 36 + }, + "SHIM_CACHE_ENTRY_DETAIL": { + "fields": { + "LastModified": { + "offset": 0, + "type": { + "kind": "struct", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "BlobSize": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "BlobBuffer": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 24 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} + diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp2-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp2-x86.json new file mode 100644 index 000000000..6114e6c85 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp2-x86.json @@ -0,0 +1,485 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 4 + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_HEADER": { + "fields": { + "Magic": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 0 + }, + "u1": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 4 + }, + "NumEntries": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 8 + }, + "u2": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 400 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "Path": { + "type": { + "count": 520, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "LastModified": { + "type": { + "kind": "union", + "name": "LARGE_INTEGER" + }, + "offset": 4 + }, + "FileSize": { + "type": { + "kind": "base", + "name": "long long" + }, + "offset": 8 + }, + "LastUpdate": { + "type": { + "kind": "union", + "name": "LARGE_INTEGER" + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 552 + }, + "_SEGMENT": { + "fields": { + "ControlArea": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_CONTROL_AREA" + } + } + }, + "TotalNumberOfPtes": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "NonExtendedPtes": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "WritableUserReferences": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "SizeOfSegment": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "SegmentPteTemplate": { + "offset": 24, + "type": { + "kind": "struct", + "name": "nt_symbols!_MMPTE" + } + }, + "NumberOfCommittedPages": { + "offset": 28, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "ExtendInfo": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_MMEXTEND_INFO" + } + } + }, + "SystemImageBase": { + "offset": 36, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + }, + "BasedAddress": { + "offset": 40, + "type": { + "kind": "base", + "name": "long" + } + }, + "u1": { + "offset": 44, + "type": { + "kind": "base", + "name": "long" + } + }, + "u2": { + "offset": 48, + "type": { + "kind": "base", + "name": "long" + } + }, + "PrototypePte": { + "offset": 52, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_MMPTE" + } + } + }, + "ThePtes": { + "offset": 60, + "type": { + "kind": "array", + "count": 1, + "subtype": { + "kind": "base", + "name": "nt_symbols!_MMPTE" + } + } + } + }, + "kind": "struct", + "size": 64 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp3-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp3-x86.json new file mode 100644 index 000000000..a9b86d93c --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp3-x86.json @@ -0,0 +1,485 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HEADER": { + "fields": { + "Magic": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 0 + }, + "u1": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 4 + }, + "NumEntries": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 8 + }, + "u2": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 400 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "Path": { + "type": { + "count": 520, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "LastModified": { + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + }, + "offset": 528 + }, + "FileSize": { + "type": { + "kind": "base", + "name": "long long" + }, + "offset": 536 + }, + "LastUpdate": { + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + }, + "offset": 544 + } + }, + "kind": "struct", + "size": 552 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 4 + } + }, + "kind": "struct", + "size": 8 + }, + "_SEGMENT": { + "fields": { + "ControlArea": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_CONTROL_AREA" + } + } + }, + "TotalNumberOfPtes": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "NonExtendedPtes": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "WritableUserReferences": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "SizeOfSegment": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "SegmentPteTemplate": { + "offset": 24, + "type": { + "kind": "struct", + "name": "nt_symbols!_MMPTE" + } + }, + "NumberOfCommittedPages": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "ExtendInfo": { + "offset": 36, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_MMEXTEND_INFO" + } + } + }, + "SystemImageBase": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + }, + "BasedAddress": { + "offset": 44, + "type": { + "kind": "base", + "name": "long" + } + }, + "u1": { + "offset": 48, + "type": { + "kind": "base", + "name": "long" + } + }, + "u2": { + "offset": 52, + "type": { + "kind": "base", + "name": "long" + } + }, + "PrototypePte": { + "offset": 56, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_MMPTE" + } + } + }, + "ThePtes": { + "offset": 64, + "type": { + "kind": "array", + "count": 1, + "subtype": { + "kind": "base", + "name": "nt_symbols!_MMPTE" + } + } + } + }, + "kind": "struct", + "size": 72 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index e1e74afc0..78e90a1d4 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -114,6 +114,24 @@ is_windows_xp = OsDistinguisher( ], ) +is_windows_xp_sp2 = OsDistinguisher( + version_check=lambda x: (5, 1) <= x < (5, 2), + fallback_checks=[ + ("KdCopyDataBlock", None, False), + ("_MMFREE_POOL_ENTRY", None, False), + ("_HANDLE_TABLE", "HandleCount", True), + ], +) + +is_windows_xp_sp3 = OsDistinguisher( + version_check=lambda x: (5, 1) <= x < (5, 2), + fallback_checks=[ + ("KdCopyDataBlock", None, False), + ("_MMFREE_POOL_ENTRY", None, True), + ("_HANDLE_TABLE", "HandleCount", True), + ], +) + is_xp_or_2003 = OsDistinguisher( version_check=lambda x: (5, 1) <= x < (6, 0), fallback_checks=[ @@ -122,6 +140,15 @@ is_xp_or_2003 = OsDistinguisher( ], ) +is_2003 = OsDistinguisher( + version_check=lambda x: (5, 2) <= x < (5, 3), + fallback_checks=[ + ("KdCopyDataBlock", None, False), + ("_HANDLE_TABLE", "HandleCount", True), + ("_MM_AVL_TABLE", None, True), + ], +) + is_win10_up_to_15063 = OsDistinguisher( version_check=lambda x: (10, 0) <= x < (10, 0, 15063), fallback_checks=[ From 4048d0b122eb9030a011012fd4f1ff59d1e0ca46 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 9 Jul 2024 11:43:29 -0500 Subject: [PATCH 107/263] Windows: Adds shimcachemem plugin --- .../framework/plugins/windows/shimcachemem.py | 610 ++++++++++++++++++ 1 file changed, 610 insertions(+) create mode 100644 volatility3/framework/plugins/windows/shimcachemem.py diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py new file mode 100644 index 000000000..6afaf4356 --- /dev/null +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -0,0 +1,610 @@ +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +import os +from datetime import datetime +from itertools import count +from typing import Iterator, List, Optional, Tuple + +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols +from volatility3.framework.configuration import requirements +from volatility3.framework.objects.utility import array_to_string +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows import versions +from volatility3.framework.symbols.windows.extensions import pe, shimcache +from volatility3.plugins import timeliner +from volatility3.plugins.windows import modules, pslist, vadinfo + +# from volatility3.plugins.windows import pslist, vadinfo, modules + +vollog = logging.getLogger(__name__) + + +class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """Reads Shimcache entries from the ahcache.sys AVL tree""" + + _required_framework_version = (2, 0, 0) + + # These checks must be completed from newest -> oldest OS version. + _win_version_file_map: List[Tuple[versions.OsDistinguisher, bool, str]] = [ + (versions.is_win10, True, "shimcache-win10-x64"), + (versions.is_win10, False, "shimcache-win10-x86"), + (versions.is_windows_8_or_later, True, "shimcache-win8-x64"), + (versions.is_windows_8_or_later, False, "shimcache-win8-x86"), + (versions.is_windows_7, True, "shimcache-win7-x64"), + (versions.is_windows_7, False, "shimcache-win7-x86"), + (versions.is_vista_or_later, True, "shimcache-vista-x64"), + (versions.is_vista_or_later, False, "shimcache-vista-x86"), + (versions.is_2003, False, "shimcache-2003-x86"), + (versions.is_2003, True, "shimcache-2003-x64"), + (versions.is_windows_xp_sp3, False, "shimcache-xp-sp3-x86"), + (versions.is_windows_xp_sp2, False, "shimcache-xp-sp2-x86"), + (versions.is_xp_or_2003, True, "shimcache-xp-2003-x64"), + (versions.is_xp_or_2003, False, "shimcache-xp-2003-x86"), + ] + + NT_KRNL_MODS = ["ntoskrnl.exe", "ntkrnlpa.exe", "ntkrnlmp.exe", "ntkrpamp.exe"] + + def generate_timeline( + self, + ) -> Iterator[Tuple[str, timeliner.TimeLinerType, datetime]]: + for _, (_, last_modified, last_update, _, _, file_path) in self._generator(): + if isinstance(last_update, datetime): + yield f"Shimcache: File {file_path} executed", timeliner.TimeLinerType.ACCESSED, last_update + if isinstance(last_modified, datetime): + yield f"Shimcache: File {file_path} modified", timeliner.TimeLinerType.MODIFIED, last_modified + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="modules", component=modules.Modules, version=(2, 0, 0) + ), + ] + + @staticmethod + def create_shimcache_table( + context: interfaces.context.ContextInterface, + symbol_table: str, + config_path: str, + ) -> str: + """Creates a shimcache symbol table + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of an existing symbol table containing the kernel symbols + config_path: The configuration path within the context of the symbol table to create + + Returns: + The name of the constructed shimcache table + """ + native_types = context.symbol_space[symbol_table].natives + is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) + table_mapping = {"nt_symbols": symbol_table} + + try: + symbol_filename = next( + filename + for version_check, for_64bit, filename in ShimcacheMem._win_version_file_map + if is_64bit == for_64bit + and version_check(context=context, symbol_table=symbol_table) + ) + except StopIteration: + raise NotImplementedError("This version of Windows is not supported!") + + vollog.debug(f"Using shimcache table {symbol_filename}") + + return intermed.IntermediateSymbolTable.create( + context, + config_path, + os.path.join("windows", "shimcache"), + symbol_filename, + class_types=shimcache.class_types, + native_types=native_types, + table_mapping=table_mapping, + ) + + @classmethod + def find_shimcache_win_xp( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + kernel_symbol_table: str, + shimcache_symbol_table: str, + ) -> Iterator[shimcache.SHIM_CACHE_ENTRY]: + """Attempts to find the shimcache in a Windows XP memory image + + :param context: The context to retrieve required elements (layers, symbol tables) from + :param layer_name: The name of the memory layer on which to operate. + :param kernel_symbol_table: The name of an existing symbol table containing the kernel symbols + :param shimcache_symbol_table: The name of a symbol table containing the hand-crafted shimcache symbols + """ + + SHIM_NUM_ENTRIES_OFFSET = 0x8 + SHIM_MAX_ENTRIES = 0x60 # 96 max entries in XP shim cache + SHIM_LRU_OFFSET = 0x10 + SHIM_HEADER_SIZE = 0x190 + SHIM_CACHE_ENTRY_SIZE = 0x228 + + seen = set() + + for process in pslist.PsList.list_processes( + context, layer_name, kernel_symbol_table + ): + pid = process.UniqueProcessId + vollog.debug("checking process %d" % pid) + for vad in vadinfo.VadInfo.list_vads( + process, lambda x: x.get_tag() == b"Vad " and x.Protection == 4 + ): + try: + proc_layer_name = process.add_process_layer() + proc_layer = context.layers[proc_layer_name] + except exceptions.InvalidAddressException: + continue + + try: + if proc_layer.read(vad.get_start(), 4) != b"\xEF\xBE\xAD\xDE": + if pid == 624: + vollog.debug("VAD magic bytes don't match DEADBEEF") + continue + except exceptions.InvalidAddressException: + continue + + num_entries = context.object( + shimcache_symbol_table + constants.BANG + "unsigned int", + proc_layer_name, + vad.get_start() + SHIM_NUM_ENTRIES_OFFSET, + ) + + if num_entries > SHIM_MAX_ENTRIES: + continue + + cache_idx_ptr = vad.get_start() + SHIM_LRU_OFFSET + + for _ in range(num_entries): + cache_idx_val = proc_layer.context.object( + shimcache_symbol_table + constants.BANG + "unsigned long", + proc_layer_name, + cache_idx_ptr, + ) + + cache_idx_ptr += 4 + + if cache_idx_val > SHIM_MAX_ENTRIES - 1: + continue + + shim_entry_offset = ( + vad.get_start() + + SHIM_HEADER_SIZE + + (SHIM_CACHE_ENTRY_SIZE * cache_idx_val) + ) + + if not proc_layer.is_valid(shim_entry_offset): + continue + + physical_addr = proc_layer.translate(shim_entry_offset) + + if physical_addr in seen: + continue + seen.add(physical_addr) + + shim_entry = proc_layer.context.object( + shimcache_symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", + proc_layer_name, + shim_entry_offset, + ) + if not proc_layer.is_valid(shim_entry.vol.offset): + continue + if not shim_entry.is_valid(): + continue + + yield shim_entry + + @classmethod + def find_shimcache_win_2k3_to_7( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + kernel_layer_name: str, + nt_symbol_table: str, + shimcache_symbol_table: str, + ) -> Iterator[shimcache.SHIM_CACHE_ENTRY]: + """Implements the algorithm to search for the shim cache on Windows 2000 + (x64) through Windows 7 / 2008 R2. The algorithm consists of the following: + + 1) Find the NT kernel module's .data and PAGE sections + 2) Iterate over every 4/8 bytes (depending on OS bitness) in the .data + section and test for the following: + a) offset represents a valid RTL_AVL_TABLE object + b) RTL_AVL_TABLE is preceeded by an ERESOURCE object + c) RTL_AVL_TABLE is followed by the beginning of the SHIM LRU list + + :param context: The context to retrieve required elements (layers, symbol tables) from + :param layer_name: The name of the memory layer on which to operate. + :param kernel_symbol_table: The name of an existing symbol table containing the kernel symbols + :param shimcache_symbol_table: The name of a symbol table containing the hand-crafted shimcache symbols + """ + + data_sec = cls.get_module_section_range( + context, + config_path, + kernel_layer_name, + nt_symbol_table, + cls.NT_KRNL_MODS, + ".data", + ) + mod_page = cls.get_module_section_range( + context, + config_path, + kernel_layer_name, + nt_symbol_table, + cls.NT_KRNL_MODS, + "PAGE", + ) + + # We require both in order to accurately handle AVL table + if not (data_sec and mod_page): + return None + + data_sec_offset, data_sec_size = data_sec + mod_page_offset, mod_page_size = mod_page + + addr_size = 8 if symbols.symbol_table_is_64bit(context, nt_symbol_table) else 4 + + shim_head = None + for offset in range( + data_sec_offset, data_sec_offset + data_sec_size, addr_size + ): + shim_head = cls.try_get_shim_head_at_offset( + context, + shimcache_symbol_table, + nt_symbol_table, + kernel_layer_name, + mod_page_offset, + mod_page_offset + mod_page_size, + offset, + ) + + if shim_head: + break + + if not shim_head: + return + + for shim_entry in shim_head.ListEntry.to_list( + shimcache_symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", "ListEntry" + ): + yield shim_entry + + @classmethod + def try_get_shim_head_at_offset( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + kernel_symbol_table: str, + layer_name: str, + mod_page_start: int, + mod_page_end: int, + offset: int, + ) -> Optional[shimcache.SHIM_CACHE_ENTRY]: + """Attempts to construct a SHIM_CACHE_HEAD within a layer of the given context, + using the provided offset within that layer, as well as the start and end offsets + of the kernel module's `PAGE` section start and end offsets. + + If a number of validity checks are passed, this method will return the `SHIM_CACHE_HEAD` + object. Otherwise, `None` is returned. + """ + # print("checking RTL_AVL_TABLE at offset %s" % hex(offset)) + rtl_avl_table = context.object( + symbol_table + constants.BANG + "_RTL_AVL_TABLE", layer_name, offset + ) + if not rtl_avl_table.is_valid(mod_page_start, mod_page_end): + return None + + vollog.debug(f"Candidate RTL_AVL_TABLE found at offset {hex(offset)}") + + ersrc_size = context.symbol_space.get_type( + kernel_symbol_table + constants.BANG + "_ERESOURCE" + ).size + ersrc_alignment = ( + 0x20 + if symbols.symbol_table_is_64bit(context, kernel_symbol_table) + else 0x10 + # 0x20 if context.symbol_space.get_type("pointer").size == 8 else 0x10 + ) + vollog.debug( + f"ERESOURCE size: {hex(ersrc_size)}, ERESOURCE alignment: {hex(ersrc_alignment)}" + ) + + eresource_rel_off = ersrc_size + ((offset - ersrc_size) % ersrc_alignment) + eresource_offset = offset - eresource_rel_off + + vollog.debug("Constructing ERESOURCE at %s" % hex(eresource_offset)) + eresource = context.object( + kernel_symbol_table + constants.BANG + "_ERESOURCE", + layer_name, + eresource_offset, + ) + if not eresource.is_valid(): + vollog.debug("ERESOURCE Invalid") + return None + + shim_head_offset = offset + rtl_avl_table.vol.size + + if not context.layers[layer_name].is_valid(shim_head_offset): + return None + + shim_head = context.object( + symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", + layer_name, + shim_head_offset, + ) + + if not shim_head.is_valid(): + vollog.debug("shim head invalid") + return None + else: + vollog.debug("returning shim head") + return shim_head + + @classmethod + def find_shimcache_win_8_or_later( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + kernel_layer_name: str, + nt_symbol_table: str, + shimcache_symbol_table: str, + ) -> Iterator[shimcache.SHIM_CACHE_ENTRY]: + """Attempts to locate and yield shimcache entries from a Windows 8 or later memory image. + + :param context: The context to retrieve required elements (layers, symbol tables) from + :param layer_name: The name of the memory layer on which to operate. + :param kernel_symbol_table: The name of an existing symbol table containing the kernel symbols + :param shimcache_symbol_table: The name of a symbol table containing the hand-crafted shimcache symbols + """ + + is_8_1_or_later = versions.is_windows_8_1_or_later( + context, nt_symbol_table + ) or versions.is_win10(context, nt_symbol_table) + + module_names = ["ahcache.sys"] if is_8_1_or_later else cls.NT_KRNL_MODS + vollog.debug(f"Searching for modules {module_names}") + + data_sec = cls.get_module_section_range( + context, + config_path, + kernel_layer_name, + nt_symbol_table, + module_names, + ".data", + ) + mod_page = cls.get_module_section_range( + context, + config_path, + kernel_layer_name, + nt_symbol_table, + module_names, + "PAGE", + ) + + if not (data_sec and mod_page): + return None + + mod_page_offset, mod_page_size = mod_page + data_sec_offset, data_sec_size = data_sec + + # iterate over ahcache kernel module's .data section in search of *two* SHIM handles + shim_heads = [] + + vollog.debug(f"PAGE offset: {hex(mod_page_offset)}") + vollog.debug(f".data offset: {hex(data_sec_offset)}") + + handle_type = context.symbol_space.get_type( + shimcache_symbol_table + constants.BANG + "SHIM_CACHE_HANDLE" + ) + for offset in range( + data_sec_offset, + data_sec_offset + data_sec_size, + 8 if symbols.symbol_table_is_64bit(context, nt_symbol_table) else 4, + ): + vollog.debug(f"Building shim handle pointer at {hex(offset)}") + shim_handle = context.object( + object_type=shimcache_symbol_table + constants.BANG + "pointer", + layer_name=kernel_layer_name, + subtype=handle_type, + offset=offset, + ) + + if shim_handle.is_valid(mod_page_offset, mod_page_offset + mod_page_size): + if shim_handle.head is not None: + vollog.debug( + f"Found valid shim handle @ {hex(shim_handle.vol.offset)}" + ) + shim_heads.append(shim_handle.head) + if len(shim_heads) == 2: + break + + if len(shim_heads) != 2: + vollog.debug("Failed to identify two valid SHIM_CACHE_HANDLE structures") + return + + # On Windows 8 x64, the frist cache contains the shim cache + # On Windows 8 x86, 8.1 x86/x64, and 10, the second cache contains the shim cache. + if ( + not symbols.symbol_table_is_64bit(context, nt_symbol_table) + and not is_8_1_or_later + ): + valid_head = shim_heads[1] + elif not is_8_1_or_later: + valid_head = shim_heads[0] + else: + valid_head = shim_heads[1] + + for shim_entry in valid_head.ListEntry.to_list( + shimcache_symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", "ListEntry" + ): + if shim_entry.is_valid(): + yield shim_entry + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + shimcache_table_name = self.create_shimcache_table( + self.context, kernel.symbol_table_name, self.config_path + ) + + c = count() + + if versions.is_windows_8_or_later(self._context, kernel.symbol_table_name): + vollog.info("Finding shimcache entries for Windows 8.0+") + entries = self.find_shimcache_win_8_or_later( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + shimcache_table_name, + ) + + elif ( + versions.is_2003(self.context, kernel.symbol_table_name) + or versions.is_vista_or_later(self.context, kernel.symbol_table_name) + or versions.is_windows_7(self.context, kernel.symbol_table_name) + ): + vollog.info("Finding shimcache entries for Windows 2k3/Vista/7") + entries = self.find_shimcache_win_2k3_to_7( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + shimcache_table_name, + ) + + elif versions.is_windows_xp_sp2( + self._context, kernel.symbol_table_name + ) or versions.is_windows_xp_sp3(self.context, kernel.symbol_table_name): + vollog.info("Finding shimcache entries for WinXP") + entries = self.find_shimcache_win_xp( + self._context, + kernel.layer_name, + kernel.symbol_table_name, + shimcache_table_name, + ) + else: + vollog.warn("Cannot parse shimcache entries for this version of Windows") + return + + for entry in entries: + try: + vollog.debug(f"SHIM_CACHE_ENTRY type: {entry.__class__}") + shim_entry = ( + entry.last_modified, + entry.last_update, + entry.exec_flag, + ( + format_hints.Hex(entry.file_size) + if isinstance(entry.file_size, int) + else entry.file_size + ), + entry.file_path, + ) + except exceptions.InvalidAddressException: + continue + + yield ( + 0, + (next(c), *shim_entry), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Order", int), + ("Last Modified", datetime), + ("Last Update", datetime), + ("Exec Flag", bool), + ("File Size", format_hints.Hex), + ("File Path", str), + ], + self._generator(), + ) + + @classmethod + def get_module_section_range( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + symbol_table: str, + module_list: List[str], + section_name: str, + ) -> Optional[Tuple[int, int]]: + """Locates the size and offset of the first found module section + specified by name from the list of modules. + + :param context: The context to operate on + :param layer_name: The memory layer to read from + :param module_list: A list of module names to search for the given section + :param section_name: The name of the section to search for. + + :return: The offset and size of the module, if found; Otherwise, returns `None` + """ + + try: + krnl_mod = next( + module + for module in modules.Modules.list_modules( + context, layer_name, symbol_table + ) + if module.BaseDllName.String in module_list + ) + except StopIteration: + return None + + pe_table_name = intermed.IntermediateSymbolTable.create( + context, + interfaces.configuration.path_join(config_path, "pe"), + "windows", + "pe", + class_types=pe.class_types, + ) + + # code taken from Win32KBase._section_chunks (win32_core.py) + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + layer_name, + offset=krnl_mod.DllBase, + ) + + if not dos_header: + return None + + nt_header = dos_header.get_nt_header() + + try: + section = next( + sec + for sec in nt_header.get_sections() + if section_name.lower() == array_to_string(sec.Name).lower() + ) + except StopIteration: + return None + + section_offset = krnl_mod.DllBase + section.VirtualAddress + section_size = section.Misc.VirtualSize + + return section_offset, section_size From 1508a414992ec958859ec5de84c3fbb51209d55f Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Thu, 18 Jul 2024 11:33:30 -0500 Subject: [PATCH 108/263] Move registry table init into the generator function --- .../framework/plugins/windows/registry/userassist.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index cf345c901..bd832b20c 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -286,6 +286,10 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac hive_offsets = [self.config.get("offset", None)] kernel = self.context.modules[self.config["kernel"]] + self._reg_table_name = intermed.IntermediateSymbolTable.create( + self.context, self._config_path, "windows", "registry" + ) + # get all the user hive offsets or use the one specified for hive in hivelist.HiveList.list_hives( context=self.context, @@ -337,10 +341,6 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac yield result def generate_timeline(self): - self._reg_table_name = intermed.IntermediateSymbolTable.create( - self.context, self._config_path, "windows", "registry" - ) - for row in self._generator(): _depth, row_data = row # check the name and the timestamp to not be empty @@ -351,10 +351,6 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac yield (description, timeliner.TimeLinerType.MODIFIED, row_data[10]) def run(self): - self._reg_table_name = intermed.IntermediateSymbolTable.create( - self.context, self._config_path, "windows", "registry" - ) - return renderers.TreeGrid( [ ("Hive Offset", renderers.format_hints.Hex), From 8da046530fd7ccd3f8a75fbe4e0b63d213207117 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 14:57:31 -0500 Subject: [PATCH 109/263] Address feedback --- .../framework/plugins/windows/svcdiff.py | 22 ++++--- .../framework/plugins/windows/svclist.py | 58 +++++++++++-------- .../framework/plugins/windows/svcscan.py | 43 ++++++++------ 3 files changed, 69 insertions(+), 54 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py index 809064946..c41a7b86c 100644 --- a/volatility3/framework/plugins/windows/svcdiff.py +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -20,8 +20,7 @@ from volatility3.framework.symbols.windows import versions vollog = logging.getLogger(__name__) - -class SvcDiff(svclist.SvcList, svcscan.SvcScan): +class SvcDiff(svcscan.SvcScan): """Compares services found through list walking versus scanning to find rootkits""" _required_framework_version = (2, 4, 0) @@ -39,22 +38,23 @@ class SvcDiff(svclist.SvcList, svcscan.SvcScan): name="svclist", component=svclist.SvcList, version=(1, 0, 0) ), requirements.VersionRequirement( - name="svcscan", component=svcscan.SvcScan, version=(2, 0, 0) + name="svcscan", component=svcscan.SvcScan, version=(3, 0, 0) ), ] def _generator(self): """ - Finds services by walking the services.exe list on supported Windows 10 versions + On Windows 10 version 15063+ 64bit Windows memory samples, walk the services list + and scan for services then report differences """ - kernel = self.context.modules[self.config["kernel"]] + kernel, service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() if not symbols.symbol_table_is_64bit( self.context, kernel.symbol_table_name ) or not versions.is_win10_15063_or_later( context=self.context, symbol_table=kernel.symbol_table_name ): - vollog.info( + vollog.warning( "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" ) return @@ -63,18 +63,16 @@ class SvcDiff(svclist.SvcList, svcscan.SvcScan): from_list = set() records = {} - service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - # collect unique service names from scanning - for service in self.service_scan( - service_table_name, service_binary_dll_map, filter_func + for service in svcscan.SvcScan.service_scan( + self.context, kernel, service_table_name, service_binary_dll_map, filter_func ): from_scan.add(service[6]) records[service[6]] = service # collect services from listing walking - for service in self.service_list( - service_table_name, service_binary_dll_map, filter_func + for service in svclist.SvcList.service_list( + self.context, kernel, service_table_name, service_binary_dll_map, filter_func ): from_list.add(service[6]) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index 1938dd182..832b3d129 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -4,7 +4,7 @@ import logging -from typing import List +from typing import List, Optional, Tuple from volatility3.framework import interfaces, exceptions, symbols from volatility3.framework.configuration import requirements @@ -20,16 +20,26 @@ class SvcList(svcscan.SvcScan): _version = (1, 0, 0) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._enumeration_method = self.service_list + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.PluginRequirement( - name="svcscan", plugin=svcscan.SvcScan, version=(2, 0, 0) + name="svcscan", plugin=svcscan.SvcScan, version=(3, 0, 0) + ), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], ), ] - def _get_exe_range(self, proc): + @classmethod + def _get_exe_range(cls, proc) -> Optional[Tuple[int, int]]: """ Returns a tuple of starting,ending address for the VAD containing services.exe @@ -45,21 +55,27 @@ class SvcList(svcscan.SvcScan): return None - def service_list(self, service_table_name, service_binary_dll_map, filter_func): - kernel = self.context.modules[self.config["kernel"]] - + @classmethod + def service_list( + cls, + context: interfaces.context.ContextInterface, + kernel, + service_table_name: str, + service_binary_dll_map, + filter_func, + ): if not symbols.symbol_table_is_64bit( - self.context, kernel.symbol_table_name + context, kernel.symbol_table_name ) or not versions.is_win10_15063_or_later( - context=self.context, symbol_table=kernel.symbol_table_name + context=context, symbol_table=kernel.symbol_table_name ): - vollog.info( + vollog.warning( "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" ) return for proc in pslist.PsList.list_processes( - context=self.context, + context=context, layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, filter_func=filter_func, @@ -74,9 +90,9 @@ class SvcList(svcscan.SvcScan): ) continue - layer = self.context.layers[layer_name] + layer = context.layers[layer_name] - exe_range = self._get_exe_range(proc) + exe_range = cls._get_exe_range(proc) if not exe_range: vollog.warning( "Could not find the application executable VAD for services.exe. Unable to proceed." @@ -84,19 +100,15 @@ class SvcList(svcscan.SvcScan): continue for offset in layer.scan( - context=self.context, + context=context, scanner=scanners.BytesScanner(needle=b"Sc27"), sections=exe_range, ): - for record in self.enumerate_vista_or_later_header( - service_table_name, service_binary_dll_map, layer_name, offset + for record in cls.enumerate_vista_or_later_header( + context, + service_table_name, + service_binary_dll_map, + layer_name, + offset, ): yield record - - def _generator(self): - service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - - for record in self.service_list( - service_table_name, service_binary_dll_map, filter_func - ): - yield (0, record) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 07274f03d..4368b83ce 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -39,7 +39,11 @@ class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (3, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._enumeration_method = self.service_scan @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -232,13 +236,14 @@ class SvcScan(interfaces.plugins.PluginInterface): for service_key in services } + @classmethod def enumerate_vista_or_later_header( - self, service_table_name, service_binary_dll_map, proc_layer_name, offset + cls, context, service_table_name, service_binary_dll_map, proc_layer_name, offset ): if offset % 8: return - service_header = self.context.object( + service_header =context.object( service_table_name + constants.BANG + "_SERVICE_HEADER", offset=offset, layer_name=proc_layer_name, @@ -257,17 +262,16 @@ class SvcScan(interfaces.plugins.PluginInterface): renderers.UnreadableValue(), renderers.UnreadableValue() ), ) - yield self.get_record_tuple(service_record, service_info) + yield cls.get_record_tuple(service_record, service_info) - def service_scan(self, service_table_name, service_binary_dll_map, filter_func): - kernel = self.context.modules[self.config["kernel"]] - - relative_tag_offset = self.context.symbol_space.get_type( + @classmethod + def service_scan(cls, context: interfaces.context.ContextInterface, kernel, service_table_name: str, service_binary_dll_map, filter_func): + relative_tag_offset = context.symbol_space.get_type( service_table_name + constants.BANG + "_SERVICE_RECORD" ).relative_child_offset("Tag") is_vista_or_later = versions.is_vista_or_later( - context=self.context, symbol_table=kernel.symbol_table_name + context=context, symbol_table=kernel.symbol_table_name ) if is_vista_or_later: @@ -278,7 +282,7 @@ class SvcScan(interfaces.plugins.PluginInterface): seen = [] for task in pslist.PsList.list_processes( - context=self.context, + context=context, layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, filter_func=filter_func, @@ -295,15 +299,15 @@ class SvcScan(interfaces.plugins.PluginInterface): ) continue - layer = self.context.layers[proc_layer_name] + layer = context.layers[proc_layer_name] for offset in layer.scan( - context=self.context, + context=context, scanner=scanners.BytesScanner(needle=service_tag), sections=vadyarascan.VadYaraScan.get_vad_maps(task), ): if not is_vista_or_later: - service_record = self.context.object( + service_record = context.object( service_table_name + constants.BANG + "_SERVICE_RECORD", offset=offset - relative_tag_offset, layer_name=proc_layer_name, @@ -318,9 +322,10 @@ class SvcScan(interfaces.plugins.PluginInterface): renderers.UnreadableValue(), renderers.UnreadableValue() ), ) - yield self.get_record_tuple(service_record, service_info) + yield cls.get_record_tuple(service_record, service_info) else: - for service_record in self.enumerate_vista_or_later_header( + for service_record in cls.enumerate_vista_or_later_header( + context, service_table_name, service_binary_dll_map, proc_layer_name, @@ -351,13 +356,13 @@ class SvcScan(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_name_filter(["services.exe"]) - return service_table_name, service_binary_dll_map, filter_func + return kernel, service_table_name, service_binary_dll_map, filter_func def _generator(self): - service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + kernel, service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - for record in self.service_scan( - service_table_name, service_binary_dll_map, filter_func + for record in self._enumeration_method( + self.context, kernel, service_table_name, service_binary_dll_map, filter_func ): yield (0, record) From a7af052509417c08eae702cd59fbfaa5306cc686 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 14:59:40 -0500 Subject: [PATCH 110/263] Black fixes --- .../framework/plugins/windows/svcdiff.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py index c41a7b86c..4325771db 100644 --- a/volatility3/framework/plugins/windows/svcdiff.py +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -20,6 +20,7 @@ from volatility3.framework.symbols.windows import versions vollog = logging.getLogger(__name__) + class SvcDiff(svcscan.SvcScan): """Compares services found through list walking versus scanning to find rootkits""" @@ -47,7 +48,9 @@ class SvcDiff(svcscan.SvcScan): On Windows 10 version 15063+ 64bit Windows memory samples, walk the services list and scan for services then report differences """ - kernel, service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + kernel, service_table_name, service_binary_dll_map, filter_func = ( + self.get_prereq_info() + ) if not symbols.symbol_table_is_64bit( self.context, kernel.symbol_table_name @@ -65,14 +68,22 @@ class SvcDiff(svcscan.SvcScan): # collect unique service names from scanning for service in svcscan.SvcScan.service_scan( - self.context, kernel, service_table_name, service_binary_dll_map, filter_func + self.context, + kernel, + service_table_name, + service_binary_dll_map, + filter_func, ): from_scan.add(service[6]) records[service[6]] = service # collect services from listing walking for service in svclist.SvcList.service_list( - self.context, kernel, service_table_name, service_binary_dll_map, filter_func + self.context, + kernel, + service_table_name, + service_binary_dll_map, + filter_func, ): from_list.add(service[6]) From 2a5e94a0946e4c74f92588df6ceafdd89acd0be4 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 15:01:10 -0500 Subject: [PATCH 111/263] Black fixes --- .../framework/plugins/windows/svcscan.py | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 4368b83ce..8b9706625 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -238,12 +238,17 @@ class SvcScan(interfaces.plugins.PluginInterface): @classmethod def enumerate_vista_or_later_header( - cls, context, service_table_name, service_binary_dll_map, proc_layer_name, offset + cls, + context, + service_table_name, + service_binary_dll_map, + proc_layer_name, + offset, ): if offset % 8: return - service_header =context.object( + service_header = context.object( service_table_name + constants.BANG + "_SERVICE_HEADER", offset=offset, layer_name=proc_layer_name, @@ -265,7 +270,14 @@ class SvcScan(interfaces.plugins.PluginInterface): yield cls.get_record_tuple(service_record, service_info) @classmethod - def service_scan(cls, context: interfaces.context.ContextInterface, kernel, service_table_name: str, service_binary_dll_map, filter_func): + def service_scan( + cls, + context: interfaces.context.ContextInterface, + kernel, + service_table_name: str, + service_binary_dll_map, + filter_func, + ): relative_tag_offset = context.symbol_space.get_type( service_table_name + constants.BANG + "_SERVICE_RECORD" ).relative_child_offset("Tag") @@ -359,10 +371,16 @@ class SvcScan(interfaces.plugins.PluginInterface): return kernel, service_table_name, service_binary_dll_map, filter_func def _generator(self): - kernel, service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + kernel, service_table_name, service_binary_dll_map, filter_func = ( + self.get_prereq_info() + ) for record in self._enumeration_method( - self.context, kernel, service_table_name, service_binary_dll_map, filter_func + self.context, + kernel, + service_table_name, + service_binary_dll_map, + filter_func, ): yield (0, record) From b0a89f210977663c8d106c50ac4602eb48b7b211 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 15:47:37 -0500 Subject: [PATCH 112/263] Address feedback --- volatility3/framework/plugins/windows/processghosting.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/processghosting.py b/volatility3/framework/plugins/windows/processghosting.py index b29ff04f0..dda0d7675 100644 --- a/volatility3/framework/plugins/windows/processghosting.py +++ b/volatility3/framework/plugins/windows/processghosting.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging +import contextlib from volatility3.framework import interfaces, exceptions from volatility3.framework import renderers @@ -57,14 +58,15 @@ class ProcessGhosting(interfaces.plugins.PluginInterface): else: file_object = 0 + if isinstance(delete_pending, int) and delete_pending not in [0, 1]: + vollog.debug(f"Invalid delete_pending value {delete_pending} found for {process_name} {proc.UniqueProcessId}") + # delete_pending besides 0 or 1 = smear if file_object == 0 or delete_pending == 1: path = renderers.UnreadableValue() if file_object: - try: + with contextlib.suppress(exceptions.InvalidAddressException): path = file_object.FileName.String - except exceptions.InvalidAddressException: - path = renderers.UnreadableValue() yield ( 0, From 920b3ec615b91ec4dcc4bdf0ef9d15715a5b1c8d Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 15:48:21 -0500 Subject: [PATCH 113/263] Black fix --- volatility3/framework/plugins/windows/processghosting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/processghosting.py b/volatility3/framework/plugins/windows/processghosting.py index dda0d7675..50f02c926 100644 --- a/volatility3/framework/plugins/windows/processghosting.py +++ b/volatility3/framework/plugins/windows/processghosting.py @@ -59,7 +59,9 @@ class ProcessGhosting(interfaces.plugins.PluginInterface): file_object = 0 if isinstance(delete_pending, int) and delete_pending not in [0, 1]: - vollog.debug(f"Invalid delete_pending value {delete_pending} found for {process_name} {proc.UniqueProcessId}") + vollog.debug( + f"Invalid delete_pending value {delete_pending} found for {process_name} {proc.UniqueProcessId}" + ) # delete_pending besides 0 or 1 = smear if file_object == 0 or delete_pending == 1: From 75ccf1bfab5ee6bc3d0f84baf427c11b8e61137c Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 17:54:33 -0500 Subject: [PATCH 114/263] Add dedicated plugin and API for extracting PE files from kernel and process memory --- .../framework/plugins/windows/dlllist.py | 76 +----- .../framework/plugins/windows/modscan.py | 11 +- .../framework/plugins/windows/modules.py | 12 +- .../framework/plugins/windows/pedump.py | 239 ++++++++++++++++++ 4 files changed, 258 insertions(+), 80 deletions(-) create mode 100644 volatility3/framework/plugins/windows/pedump.py diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index d48a53663..eef826ed5 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -1,10 +1,9 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import contextlib import datetime import logging -import ntpath import re from typing import List, Optional, Type @@ -14,7 +13,7 @@ from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins import timeliner -from volatility3.plugins.windows import info, pslist, psscan +from volatility3.plugins.windows import info, pslist, psscan, pedump vollog = logging.getLogger(__name__) @@ -23,7 +22,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the loaded modules in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (3, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -76,66 +75,10 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): default=False, optional=True, ), - ] - - @classmethod - def dump_pe( - cls, - context: interfaces.context.ContextInterface, - pe_table_name: str, - dll_entry: interfaces.objects.ObjectInterface, - open_method: Type[interfaces.plugins.FileHandlerInterface], - layer_name: str = None, - prefix: str = "", - ) -> Optional[interfaces.plugins.FileHandlerInterface]: - """Extracts the complete data for a process as a FileInterface - - Args: - context: the context to operate upon - pe_table_name: the name for the symbol table containing the PE format symbols - dll_entry: the object representing the module - layer_name: the layer that the DLL lives within - open_method: class for constructing output files - - Returns: - An open FileHandlerInterface object containing the complete data for the DLL or None in the case of failure - """ - try: - try: - name = dll_entry.FullDllName.get_string() - except exceptions.InvalidAddressException: - name = "UnreadableDLLName" - - if layer_name is None: - layer_name = dll_entry.vol.layer_name - - file_handle = open_method( - "{}{}.{:#x}.{:#x}.dmp".format( - prefix, - ntpath.basename(name), - dll_entry.vol.offset, - dll_entry.DllBase, - ) - ) - - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=dll_entry.DllBase, - layer_name=layer_name, - ) - - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - IOError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump dll at offset {dll_entry.DllBase}: {excp}") - return None - return file_handle + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), + ] def _generator(self, procs): pe_table_name = intermed.IntermediateSymbolTable.create( @@ -204,7 +147,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_output = "Disabled" if self.config["dump"]: - file_handle = self.dump_pe( + file_handle = pedump.PEDump.dump_ldr_entry( self.context, pe_table_name, entry, @@ -214,8 +157,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) file_output = "Error outputting file" if file_handle: - file_handle.close() - file_output = file_handle.preferred_filename + file_output = file_handle try: dllbase = format_hints.Hex(entry.DllBase) except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index 98546bc9c..fc45e6913 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -6,7 +6,7 @@ from typing import Iterable from volatility3.framework import interfaces from volatility3.framework.configuration import requirements -from volatility3.plugins.windows import poolscanner, dlllist, pslist, modules +from volatility3.plugins.windows import poolscanner, modules, pedump vollog = logging.getLogger(__name__) @@ -35,12 +35,6 @@ class ModScan(modules.Modules): requirements.VersionRequirement( name="modules", component=modules.Modules, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="pslist", component=pslist.PsList, version=(2, 0, 0) - ), - requirements.VersionRequirement( - name="dlllist", component=dlllist.DllList, version=(2, 0, 0) - ), requirements.BooleanRequirement( name="dump", description="Extract listed modules", @@ -58,6 +52,9 @@ class ModScan(modules.Modules): optional=True, default=None, ), + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), ] @classmethod diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 79eea1cd7..283d4dcb9 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -9,7 +9,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, dlllist +from volatility3.plugins.windows import pslist, pedump vollog = logging.getLogger(__name__) @@ -35,9 +35,6 @@ class Modules(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="dlllist", component=dlllist.DllList, version=(2, 0, 0) - ), requirements.BooleanRequirement( name="dump", description="Extract listed modules", @@ -55,6 +52,9 @@ class Modules(interfaces.plugins.PluginInterface): optional=True, default=None, ), + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), ] def dump_module(self, session_layers, pe_table_name, mod): @@ -63,7 +63,7 @@ class Modules(interfaces.plugins.PluginInterface): ) file_output = f"Cannot find a viable session layer for {mod.DllBase:#x}" if session_layer_name: - file_handle = dlllist.DllList.dump_pe( + file_handle = pedump.PEDump.dump_ldr_entry( self.context, pe_table_name, mod, @@ -72,7 +72,7 @@ class Modules(interfaces.plugins.PluginInterface): ) file_output = "Error outputting file" if file_handle: - file_output = file_handle.preferred_filename + file_output = file_handle return file_output diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py new file mode 100644 index 000000000..395d28ce5 --- /dev/null +++ b/volatility3/framework/plugins/windows/pedump.py @@ -0,0 +1,239 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +import ntpath +from typing import List, Type, Optional + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, modules + +vollog = logging.getLogger(__name__) + + +class PEDump(interfaces.plugins.PluginInterface): + """Allows extracting PE Files from a specific address in a specific address space""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), + requirements.IntRequirement( + name="base", + description="Base address to reconstruct a PE file", + optional=False, + ), + requirements.BooleanRequirement( + name="kernel_module", + description="Extract from kernel address space.", + default=False, + optional=True, + ), + ] + + @classmethod + def dump_pe( + cls, + context: interfaces.context.ContextInterface, + pe_table_name: str, + layer_name: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + file_name: str, + base: int + ) -> Optional[str]: + """ + Returns the filename of the dump file or None + """ + try: + file_handle = open_method(file_name) + + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) + + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + IOError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None + finally: + file_handle.close() + + return file_handle.preferred_filename + + @classmethod + def dump_ldr_entry( + cls, + context: interfaces.context.ContextInterface, + pe_table_name: str, + ldr_entry: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + layer_name: str = None, + prefix: str = "", + ) -> Optional[str]: + """Extracts the PE file referenced an LDR_DATA_TABLE_ENTRY (DLL, kernel module) instance + + Args: + context: the context to operate upon + pe_table_name: the name for the symbol table containing the PE format symbols + ldr_entry: the object representing the module + open_method: class for constructing output files + layer_name: the layer that the DLL lives within + prefix: optional string to prepend to filename + Returns: + The output file name or None in the case of failure + """ + try: + name = ldr_entry.FullDllName.get_string() + except exceptions.InvalidAddressException: + name = "UnreadableDLLName" + + if layer_name is None: + layer_name = ldr_entry.vol.layer_name + + file_name = "{}{}.{:#x}.{:#x}.dmp".format( + prefix, + ntpath.basename(name), + ldr_entry.vol.offset, + ldr_entry.DllBase, + ) + + return PEDump.dump_pe(context, pe_table_name, layer_name, open_method, file_name, ldr_entry.DllBase) + + @classmethod + def dump_pe_at_base( + cls, + context: interfaces.context.ContextInterface, + pe_table_name: str, + layer_name: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + proc_offset: int, + pid: int, + base: int, + ) -> Optional[str]: + file_name = "PE.{:#x}.{:d}.{:#x}.dmp".format( + proc_offset, + pid, + base, + ) + + return PEDump.dump_pe(context, pe_table_name, layer_name, open_method, file_name, base) + + @classmethod + def dump_kernel_pe_at_base(cls, context, kernel, pe_table_name, open_method, base): + session_layers = modules.Modules.get_session_layers( + context, kernel.layer_name, kernel.symbol_table_name + ) + + session_layer_name = modules.Modules.find_session_layer( + context, session_layers, base + ) + + if session_layer_name: + system_pid = 4 + + file_output = PEDump.dump_pe_at_base( + context, pe_table_name, session_layer_name, open_method, 0, system_pid, base + ) + + if file_output: + yield system_pid, "Kernel", file_output + else: + vollog.warning( + "Unable to find a session layer with the provided base address mapped in the kernel." + ) + + @classmethod + def dump_processes(cls, context, kernel, pe_table_name, open_method, filter_func, base): + """ + """ + + for proc in pslist.PsList.list_processes( + context=context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ): + pid = proc.UniqueProcessId + proc_name = proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ) + proc_layer_name = proc.add_process_layer() + + file_output = PEDump.dump_pe_at_base( + context, pe_table_name, proc_layer_name, open_method, proc.vol.offset, pid, base + ) + + if file_output: + yield pid, proc_name, file_output + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "windows", "pe", class_types=pe.class_types + ) + + if self.config["kernel_module"] and self.config["pid"]: + vollog.error("Only --kernel_module or --pid should be set. Not both") + return + + if not self.config["kernel_module"] and not self.config["pid"]: + vollog.error("--kernel_module or --pid must be set") + return + + if self.config["kernel_module"]: + pe_files = self.dump_kernel_pe_at_base(self.context, kernel, pe_table_name, self.open, self.config["base"]) + else: + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + pe_files = self.dump_processes(self.context, kernel, pe_table_name, self.open, filter_func, self.config["base"]) + + for pid, proc_name, file_output in pe_files: + yield ( + 0, + ( + pid, + proc_name, + file_output, + ), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("File output", str), + ], + self._generator(), + ) From 12f3beacfcb3ab16f8418cee2821ad74c3738045 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 17:56:24 -0500 Subject: [PATCH 115/263] Add dedicated plugin and API for extracting PE files from kernel and process memory --- .../framework/plugins/windows/dlllist.py | 2 +- .../framework/plugins/windows/pedump.py | 69 ++++++++++++++----- 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index eef826ed5..6c8c96dc3 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -78,7 +78,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="pedump", component=pedump.PEDump, version=(1, 0, 0) ), - ] + ] def _generator(self, procs): pe_table_name = intermed.IntermediateSymbolTable.create( diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 395d28ce5..0b4649abb 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -59,7 +59,7 @@ class PEDump(interfaces.plugins.PluginInterface): layer_name: str, open_method: Type[interfaces.plugins.FileHandlerInterface], file_name: str, - base: int + base: int, ) -> Optional[str]: """ Returns the filename of the dump file or None @@ -120,13 +120,20 @@ class PEDump(interfaces.plugins.PluginInterface): layer_name = ldr_entry.vol.layer_name file_name = "{}{}.{:#x}.{:#x}.dmp".format( - prefix, - ntpath.basename(name), - ldr_entry.vol.offset, - ldr_entry.DllBase, - ) + prefix, + ntpath.basename(name), + ldr_entry.vol.offset, + ldr_entry.DllBase, + ) - return PEDump.dump_pe(context, pe_table_name, layer_name, open_method, file_name, ldr_entry.DllBase) + return PEDump.dump_pe( + context, + pe_table_name, + layer_name, + open_method, + file_name, + ldr_entry.DllBase, + ) @classmethod def dump_pe_at_base( @@ -140,12 +147,14 @@ class PEDump(interfaces.plugins.PluginInterface): base: int, ) -> Optional[str]: file_name = "PE.{:#x}.{:d}.{:#x}.dmp".format( - proc_offset, - pid, - base, - ) + proc_offset, + pid, + base, + ) - return PEDump.dump_pe(context, pe_table_name, layer_name, open_method, file_name, base) + return PEDump.dump_pe( + context, pe_table_name, layer_name, open_method, file_name, base + ) @classmethod def dump_kernel_pe_at_base(cls, context, kernel, pe_table_name, open_method, base): @@ -161,7 +170,13 @@ class PEDump(interfaces.plugins.PluginInterface): system_pid = 4 file_output = PEDump.dump_pe_at_base( - context, pe_table_name, session_layer_name, open_method, 0, system_pid, base + context, + pe_table_name, + session_layer_name, + open_method, + 0, + system_pid, + base, ) if file_output: @@ -172,9 +187,10 @@ class PEDump(interfaces.plugins.PluginInterface): ) @classmethod - def dump_processes(cls, context, kernel, pe_table_name, open_method, filter_func, base): - """ - """ + def dump_processes( + cls, context, kernel, pe_table_name, open_method, filter_func, base + ): + """ """ for proc in pslist.PsList.list_processes( context=context, @@ -191,7 +207,13 @@ class PEDump(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() file_output = PEDump.dump_pe_at_base( - context, pe_table_name, proc_layer_name, open_method, proc.vol.offset, pid, base + context, + pe_table_name, + proc_layer_name, + open_method, + proc.vol.offset, + pid, + base, ) if file_output: @@ -213,10 +235,19 @@ class PEDump(interfaces.plugins.PluginInterface): return if self.config["kernel_module"]: - pe_files = self.dump_kernel_pe_at_base(self.context, kernel, pe_table_name, self.open, self.config["base"]) + pe_files = self.dump_kernel_pe_at_base( + self.context, kernel, pe_table_name, self.open, self.config["base"] + ) else: filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - pe_files = self.dump_processes(self.context, kernel, pe_table_name, self.open, filter_func, self.config["base"]) + pe_files = self.dump_processes( + self.context, + kernel, + pe_table_name, + self.open, + filter_func, + self.config["base"], + ) for pid, proc_name, file_output in pe_files: yield ( From bca4cdda744120a5ca868bdfd39119ebf0819201 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Fri, 19 Jul 2024 11:09:52 +0200 Subject: [PATCH 116/263] replace yara-python with yara-x --- doc/requirements.txt | 2 +- test/requirements-testing.txt | 2 +- volatility3/framework/plugins/yarascan.py | 74 ++++++----------------- 3 files changed, 22 insertions(+), 56 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index b715e59f5..c04f71219 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,6 +3,6 @@ sphinx>=4.0.0,<7 sphinx_autodoc_typehints>=1.4.0 sphinx-rtd-theme>=0.4.3 -yara-python +yara-x pycryptodome pefile diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index 7afe19b94..0955d1939 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -5,6 +5,6 @@ pefile>=2017.8.1 #foo # If certain packages are not necessary, place a comment (#) at the start of the line. # This is required for the yara plugins -yara-python>=3.8.0 +yara-x>=0.5.0 pytest>=7.0.0 diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 496ec1844..32a400f9e 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -14,14 +14,9 @@ from volatility3.framework.renderers import format_hints vollog = logging.getLogger(__name__) try: - import yara + import yara_x - if tuple([int(x) for x in yara.__version__.split(".")]) < (3, 8): - raise ImportError except ImportError: - vollog.info( - "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available" - ) raise @@ -34,27 +29,20 @@ class YaraScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules - self.st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( - 4, - 3, - ) def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: - for match in self._rules.match(data=data): - if YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: - yield ( - instance.offset + data_offset, - match.rule, - match_string.identifier, - instance.matched_data, - ) - else: - for offset, name, value in match.strings: - yield (offset + data_offset, match.rule, name, value) + results = self._rules.scan(data) + for match in results.matching_rules: + for match_string in match.patterns: + for instance in match_string.matches: + yield ( + instance.offset + data_offset, + f"{match.namespace}.{match.identifier}", + match_string.identifier, + data[instance.offset : instance.offset + instance.length], + ) class YaraScan(plugins.PluginInterface): @@ -63,9 +51,6 @@ class YaraScan(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 3, 0) - # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string - # or something that makes more sense - @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: """Returns the requirements needed to run yarascan directly, combining the TranslationLayerRequirement @@ -99,16 +84,13 @@ class YaraScan(plugins.PluginInterface): optional=True, ), requirements.StringRequirement( - name="yara_rules", description="Yara rules (as a string)", optional=True + name="yara_string", + description="Yara rules (as a string)", + optional=True, ), - requirements.URIRequirement( - name="yara_file", description="Yara rules (as a file)", optional=True - ), - # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code - # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later requirements.URIRequirement( name="yara_compiled_file", - description="Yara compiled rules (as a file)", + description="Yara-x compiled rules (as a file)", optional=True, ), requirements.IntRequirement( @@ -119,36 +101,20 @@ class YaraScan(plugins.PluginInterface): ), ] - @classmethod - def yara_returns_instances(cls) -> bool: - st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( - 4, - 3, - ) - return st_object - @classmethod def process_yara_options(cls, config: Dict[str, Any]): rules = None - if config.get("yara_rules", None) is not None: - rule = config["yara_rules"] + if config.get("yara_string") is not None: + rule = config["yara_string"] if rule[0] not in ["{", "/"]: rule = f'"{rule}"' if config.get("case", False): rule += " nocase" if config.get("wide", False): rule += " wide ascii" - rules = yara.compile( - sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} - ) - elif config.get("yara_source", None) is not None: - rules = yara.compile(source=config["yara_source"]) - elif config.get("yara_file", None) is not None: - rules = yara.compile( - file=resources.ResourceAccessor().open(config["yara_file"], "rb") - ) - elif config.get("yara_compiled_file", None) is not None: - rules = yara.load( + rules = yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") + elif config.get("yara_compiled_file") is not None: + rules = yara_x.Rules.deserialize_from( file=resources.ResourceAccessor().open( config["yara_compiled_file"], "rb" ) From 9454181b9892f5c4245435bb868f24aaee23ae01 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 21 Jul 2024 09:42:19 -0500 Subject: [PATCH 117/263] Address feedback --- volatility3/framework/plugins/windows/dlllist.py | 9 +++++---- volatility3/framework/plugins/windows/modules.py | 7 +++---- volatility3/framework/plugins/windows/pedump.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 6c8c96dc3..5a1b37fcf 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -147,7 +147,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_output = "Disabled" if self.config["dump"]: - file_handle = pedump.PEDump.dump_ldr_entry( + file_output = pedump.PEDump.dump_ldr_entry( self.context, pe_table_name, entry, @@ -155,9 +155,10 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): proc_layer_name, prefix=f"pid.{proc_id}.", ) - file_output = "Error outputting file" - if file_handle: - file_output = file_handle + + if not file_output: + file_output = "Error outputting file" + try: dllbase = format_hints.Hex(entry.DllBase) except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 283d4dcb9..ba45834d5 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -63,16 +63,15 @@ class Modules(interfaces.plugins.PluginInterface): ) file_output = f"Cannot find a viable session layer for {mod.DllBase:#x}" if session_layer_name: - file_handle = pedump.PEDump.dump_ldr_entry( + file_output = pedump.PEDump.dump_ldr_entry( self.context, pe_table_name, mod, self.open, layer_name=session_layer_name, ) - file_output = "Error outputting file" - if file_handle: - file_output = file_handle + if not file_output: + file_output = "Error outputting file" return file_output diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 0b4649abb..858d0615a 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -126,7 +126,7 @@ class PEDump(interfaces.plugins.PluginInterface): ldr_entry.DllBase, ) - return PEDump.dump_pe( + return cls.dump_pe( context, pe_table_name, layer_name, From 21396185d0faaab15a217cff2072aa7de2652b67 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 21 Jul 2024 11:10:38 -0500 Subject: [PATCH 118/263] Address feedback --- .../framework/plugins/windows/svcdiff.py | 37 ++++++++++----- .../framework/plugins/windows/svclist.py | 11 +++-- .../framework/plugins/windows/svcscan.py | 47 +++++++++++-------- 3 files changed, 58 insertions(+), 37 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py index 4325771db..84d06a695 100644 --- a/volatility3/framework/plugins/windows/svcdiff.py +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -13,7 +13,7 @@ import logging -from volatility3.framework import symbols +from volatility3.framework import symbols, interfaces from volatility3.framework.configuration import requirements from volatility3.plugins.windows import svclist, svcscan from volatility3.framework.symbols.windows import versions @@ -26,6 +26,10 @@ class SvcDiff(svcscan.SvcScan): _required_framework_version = (2, 4, 0) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._enumeration_method = self.service_diff + @classmethod def get_requirements(cls): # Since we're calling the plugin, make sure we have the plugin's requirements @@ -43,19 +47,24 @@ class SvcDiff(svcscan.SvcScan): ), ] - def _generator(self): + @classmethod + def service_diff( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + service_table_name: str, + service_binary_dll_map, + filter_func, + ): """ On Windows 10 version 15063+ 64bit Windows memory samples, walk the services list and scan for services then report differences """ - kernel, service_table_name, service_binary_dll_map, filter_func = ( - self.get_prereq_info() - ) - if not symbols.symbol_table_is_64bit( - self.context, kernel.symbol_table_name + context, symbol_table ) or not versions.is_win10_15063_or_later( - context=self.context, symbol_table=kernel.symbol_table_name + context=context, symbol_table=symbol_table ): vollog.warning( "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" @@ -68,8 +77,9 @@ class SvcDiff(svcscan.SvcScan): # collect unique service names from scanning for service in svcscan.SvcScan.service_scan( - self.context, - kernel, + context, + layer_name, + symbol_table, service_table_name, service_binary_dll_map, filter_func, @@ -79,8 +89,9 @@ class SvcDiff(svcscan.SvcScan): # collect services from listing walking for service in svclist.SvcList.service_list( - self.context, - kernel, + context, + layer_name, + symbol_table, service_table_name, service_binary_dll_map, filter_func, @@ -89,4 +100,4 @@ class SvcDiff(svcscan.SvcScan): # report services found from scanning but not list walking for hidden_service in from_scan - from_list: - yield (0, records[hidden_service]) + yield records[hidden_service] diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index 832b3d129..a59581063 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -59,15 +59,16 @@ class SvcList(svcscan.SvcScan): def service_list( cls, context: interfaces.context.ContextInterface, - kernel, + layer_name: str, + symbol_table: str, service_table_name: str, service_binary_dll_map, filter_func, ): if not symbols.symbol_table_is_64bit( - context, kernel.symbol_table_name + context, symbol_table ) or not versions.is_win10_15063_or_later( - context=context, symbol_table=kernel.symbol_table_name + context=context, symbol_table=symbol_table ): vollog.warning( "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" @@ -76,8 +77,8 @@ class SvcList(svcscan.SvcScan): for proc in pslist.PsList.list_processes( context=context, - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, + layer_name=layer_name, + symbol_table=symbol_table, filter_func=filter_func, ): try: diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 8b9706625..bf676acb7 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -148,14 +148,17 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types=native_types, ) - def _get_service_key(self, kernel) -> Optional[objects.StructType]: + @classmethod + def _get_service_key( + cls, context, config_path: str, layer_name: str, symbol_table: str + ) -> Optional[objects.StructType]: for hive in hivelist.HiveList.list_hives( - context=self.context, + context=context, base_config_path=interfaces.configuration.path_join( - self.config_path, "hivelist" + config_path, "hivelist" ), - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, + layer_name=layer_name, + symbol_table=symbol_table, filter_string="machine\\system", ): # Get ControlSet\Services. @@ -273,7 +276,8 @@ class SvcScan(interfaces.plugins.PluginInterface): def service_scan( cls, context: interfaces.context.ContextInterface, - kernel, + layer_name: str, + symbol_table: str, service_table_name: str, service_binary_dll_map, filter_func, @@ -283,7 +287,7 @@ class SvcScan(interfaces.plugins.PluginInterface): ).relative_child_offset("Tag") is_vista_or_later = versions.is_vista_or_later( - context=context, symbol_table=kernel.symbol_table_name + context=context, symbol_table=symbol_table ) if is_vista_or_later: @@ -295,8 +299,8 @@ class SvcScan(interfaces.plugins.PluginInterface): for task in pslist.PsList.list_processes( context=context, - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, + layer_name=layer_name, + symbol_table=symbol_table, filter_func=filter_func, ): proc_id = "Unknown" @@ -348,36 +352,41 @@ class SvcScan(interfaces.plugins.PluginInterface): seen.append(service_record) yield service_record - def get_prereq_info(self): + @classmethod + def get_prereq_info(cls, context, config_path, layer_name: str, symbol_table: str): """ Data structures and information needed to analyze service information """ - kernel = self.context.modules[self.config["kernel"]] - service_table_name = self.create_service_table( - self.context, kernel.symbol_table_name, self.config_path + service_table_name = cls.create_service_table( + context, symbol_table, config_path ) - services_key = self._get_service_key(kernel) + services_key = cls._get_service_key( + context, config_path, layer_name, symbol_table + ) service_binary_dll_map = ( - self._get_service_binary_map(services_key) + cls._get_service_binary_map(services_key) if services_key is not None else {} ) filter_func = pslist.PsList.create_name_filter(["services.exe"]) - return kernel, service_table_name, service_binary_dll_map, filter_func + return service_table_name, service_binary_dll_map, filter_func def _generator(self): - kernel, service_table_name, service_binary_dll_map, filter_func = ( - self.get_prereq_info() + kernel = self.context.modules[self.config["kernel"]] + + service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info( + self.context, self.config_path, kernel.layer_name, kernel.symbol_table_name ) for record in self._enumeration_method( self.context, - kernel, + kernel.layer_name, + kernel.symbol_table_name, service_table_name, service_binary_dll_map, filter_func, From 111873e173b1bc639754b851ce46ff5056f8edb8 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 21 Jul 2024 11:54:06 -0500 Subject: [PATCH 119/263] Fix class vs static method and leading underscores --- volatility3/framework/plugins/windows/svcscan.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index bf676acb7..52ed5e759 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -110,7 +110,7 @@ class SvcScan(interfaces.plugins.PluginInterface): ] @staticmethod - def create_service_table( + def _create_service_table( context: interfaces.context.ContextInterface, symbol_table: str, config_path: str, @@ -148,9 +148,9 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types=native_types, ) - @classmethod + @staticmethod def _get_service_key( - cls, context, config_path: str, layer_name: str, symbol_table: str + context, config_path: str, layer_name: str, symbol_table: str ) -> Optional[objects.StructType]: for hive in hivelist.HiveList.list_hives( context=context, @@ -358,7 +358,7 @@ class SvcScan(interfaces.plugins.PluginInterface): Data structures and information needed to analyze service information """ - service_table_name = cls.create_service_table( + service_table_name = cls._create_service_table( context, symbol_table, config_path ) From 5e96327cb0851fe75d1c7a4cb9ef27641bf119f7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 21 Jul 2024 22:58:43 +0100 Subject: [PATCH 120/263] Add in threads that only provides an implmentation method --- .../framework/plugins/windows/thrdscan.py | 32 +++++++++++------ .../framework/plugins/windows/threads.py | 34 +++++++++++++------ 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index bbf65cd6c..be7097347 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -3,7 +3,7 @@ ## import logging import datetime -from typing import Iterable +from typing import Callable, Iterable from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements @@ -21,6 +21,10 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) _required_framework_version = (2, 6, 0) _version = (1, 0, 0) + def __init__(self, *args, **kwargs): + self.implementation = self.scan_threads + super().__init__(*args, **kwargs) + @classmethod def get_requirements(cls): return [ @@ -38,8 +42,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) def scan_threads( cls, context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, + module_name: str, ) -> Iterable[interfaces.objects.ObjectInterface]: """Scans for threads using the poolscanner module and constraints. @@ -52,6 +55,10 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) A list of _ETHREAD objects found by scanning memory for the "Thre" / "Thr\\xE5" pool signatures """ + module = context.modules[module_name] + layer_name = module.layer_name + symbol_table = module.symbol_table_name + constraints = poolscanner.PoolScanner.builtin_constraints( symbol_table, [b"Thr\xe5", b"Thre"] ) @@ -76,7 +83,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ethread.get_exit_time() ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object except exceptions.InvalidAddressException: - vollog.debug("Thread invalid address {:#x}".format(thread.vol.offset)) + vollog.debug("Thread invalid address {:#x}".format(ethread.vol.offset)) return None return ( @@ -88,12 +95,10 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) thread_exit_time, ) - def _generator(self): - kernel = self.context.modules[self.config["kernel"]] + def _generator(self, filter_func: Callable): + kernel_name = self.config["kernel"] - for ethread in self.scan_threads( - self.context, kernel.layer_name, kernel.symbol_table_name - ): + for ethread in self.implementation(self.context, kernel_name): info = self.gather_thread_info(ethread) if info: yield (0, info) @@ -126,7 +131,14 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) row_dict["ExitTime"], ) + @classmethod + def filter_func(cls, config: interfaces.configuration.HierarchicalDict) -> Callable: + """Returns a function that can filter this plugin's implementation method based on the config""" + return lambda x: False + def run(self): + filt_func = self.filter_func(self.config) + return renderers.TreeGrid( [ ("Offset", format_hints.Hex), @@ -136,5 +148,5 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime), ], - self._generator(), + self._generator(filt_func), ) diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index 83d231abb..55720c6f3 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -3,7 +3,7 @@ # import logging -from typing import List, Generator +from typing import Callable, Iterable, List, Generator from volatility3.framework import interfaces, constants from volatility3.framework.configuration import requirements @@ -18,6 +18,9 @@ class Threads(thrdscan.ThrdScan): _required_framework_version = (2, 4, 0) _version = (1, 0, 0) + def __init__(self): + self.implementation = self.list_process_threads + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements @@ -60,18 +63,27 @@ class Threads(thrdscan.ThrdScan): seen.add(thread.vol.offset) yield thread - def _generator(self): - kernel = self.context.modules[self.config["kernel"]] + @classmethod + def filter_func(cls, config: interfaces.configuration.HierarchicalDict) -> Callable: + return pslist.PsList.create_pid_filter(config.get("pid", None)) - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + @classmethod + def list_process_threads( + cls, + context: interfaces.context.ContextInterface, + module_name: str, + filter_func: Callable, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Runs through all processes and lists threads for each process""" + module = context.modules[module_name] + layer_name = module.layer_name + symbol_table_name = module.symbol_table_name for proc in pslist.PsList.list_processes( - context=self.context, - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, filter_func=filter_func, ): - for thread in self.list_threads(kernel, proc): - info = self.gather_thread_info(thread) - if info: - yield (0, info) + for thread in cls.list_threads(module, proc): + yield thread From ce9832a2b3d524637a7bfaa49c7726ff234c121e Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Mon, 22 Jul 2024 13:58:33 +0200 Subject: [PATCH 121/263] both yara-python and yara-x support --- doc/requirements.txt | 1 + test/requirements-testing.txt | 1 + volatility3/framework/plugins/yarascan.py | 101 ++++++++++++++++++++-- 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index c04f71219..d3ba51224 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,6 +3,7 @@ sphinx>=4.0.0,<7 sphinx_autodoc_typehints>=1.4.0 sphinx-rtd-theme>=0.4.3 +yara-python yara-x pycryptodome pefile diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index 0955d1939..51c8f602c 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -5,6 +5,7 @@ pefile>=2017.8.1 #foo # If certain packages are not necessary, place a comment (#) at the start of the line. # This is required for the yara plugins +yara-python>=3.8.0 yara-x>=0.5.0 pytest>=7.0.0 diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 32a400f9e..03a45b8df 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -13,14 +13,73 @@ from volatility3.framework.renderers import format_hints vollog = logging.getLogger(__name__) +USE_YARA_X = False + try: import yara_x + USE_YARA_X = True + except ImportError: - raise + try: + import yara + + if tuple(int(x) for x in yara.__version__.split(".")) < (3, 8): + raise ImportError + + vollog.info("Using yara-python module") + + except ImportError: + vollog.info( + "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available" + ) + raise -class YaraScanner(interfaces.layers.ScannerInterface): +class YaraPythonScanner(interfaces.layers.ScannerInterface): + _version = (2, 0, 0) + + # yara.Rules isn't exposed, so we can't type this properly + def __init__(self, rules) -> None: + super().__init__() + if rules is None: + raise ValueError("No rules provided to YaraScanner") + self._rules = rules + self.st_object = not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) + + def __call__( + self, data: bytes, data_offset: int + ) -> Iterable[Tuple[int, str, str, bytes]]: + for match in self._rules.match(data=data): + if self.st_object: + for match_string in match.strings: + for instance in match_string.instances: + yield ( + instance.offset + data_offset, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield (offset + data_offset, match.rule, name, value) + + @staticmethod + def get_rule(rule): + return yara.compile( + sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} + ) + + @staticmethod + def from_compiled_file(filepath): + return yara.load(file=resources.ResourceAccessor().open(filepath, "rb")) + + @staticmethod + def from_file(filepath): + return yara.compile(file=resources.ResourceAccessor().open(filepath, "rb")) + + +class YaraXScanner(interfaces.layers.ScannerInterface): _version = (2, 0, 0) # yara.Rules isn't exposed, so we can't type this properly @@ -44,6 +103,25 @@ class YaraScanner(interfaces.layers.ScannerInterface): data[instance.offset : instance.offset + instance.length], ) + @staticmethod + def get_rule(rule): + return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") + + @staticmethod + def from_compiled_file(filepath): + return yara_x.Rules.deserialize_from( + file=resources.ResourceAccessor().open(filepath, "rb") + ) + + @staticmethod + def from_file(filepath): + return yara_x.compile( + resources.ResourceAccessor().open(filepath, "rb").read().decode() + ) + + +YaraScanner = YaraXScanner if USE_YARA_X else YaraPythonScanner + class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" @@ -88,9 +166,14 @@ class YaraScan(plugins.PluginInterface): description="Yara rules (as a string)", optional=True, ), + requirements.URIRequirement( + name="yara_file", + description="Yara rules (as a file)", + optional=True, + ), requirements.URIRequirement( name="yara_compiled_file", - description="Yara-x compiled rules (as a file)", + description="Yara compiled rules (as a file)", optional=True, ), requirements.IntRequirement( @@ -112,13 +195,15 @@ class YaraScan(plugins.PluginInterface): rule += " nocase" if config.get("wide", False): rule += " wide ascii" - rules = yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") + rules = YaraScanner.get_rule(rule) + elif config.get("yara_file") is not None: + vollog.debug(f"Plain file: {config["yara_file"]} - yara-x: {USE_YARA_X}") + rules = YaraScanner.from_file(config["yara_file"]) elif config.get("yara_compiled_file") is not None: - rules = yara_x.Rules.deserialize_from( - file=resources.ResourceAccessor().open( - config["yara_compiled_file"], "rb" - ) + vollog.debug( + f"Compiled file: {config["yara_compiled_file"]} - yara-x: {USE_YARA_X}" ) + rules = YaraScanner.from_compiled_file(config["yara_compiled_file"]) else: vollog.error("No yara rules, nor yara rules file were specified") return rules From af2d6206763a661ced9687f1c7b31b7888d3d0ce Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 23 Jul 2024 22:02:34 +0200 Subject: [PATCH 122/263] Improving lsof --- volatility3/framework/plugins/linux/lsof.py | 53 +++++++++++++++++-- .../framework/symbols/linux/__init__.py | 21 +++++++- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index d970ad8a9..f9aeafe14 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -3,7 +3,7 @@ # """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" -import logging +import logging, datetime from typing import List, Callable from volatility3.framework import renderers, interfaces, constants @@ -76,14 +76,59 @@ class Lsof(plugins.PluginInterface): ) for pid, task_comm, _task, fd_fields in fds_generator: - fd_num, _filp, full_path = fd_fields + ( + fd_num, + _filp, + full_path, + inode_num, + imode, + ctime, + mtime, + atime, + file_size, + ) = fd_fields - fields = (pid, task_comm, fd_num, full_path) + fields = ( + pid, + task_comm, + fd_num, + full_path, + inode_num, + imode, + ctime, + mtime, + atime, + file_size, + ) yield (0, fields) def run(self): pids = self.config.get("pid", None) symbol_table = self.config["kernel"] - tree_grid_args = [("PID", int), ("Process", str), ("FD", int), ("Path", str)] + tree_grid_args = [ + ("PID", int), + ("Process", str), + ("FD", int), + ("Path", str), + ("Inode", int), + ("Mode", str), + ("LastChange", datetime.datetime), + ("LastModify", datetime.datetime), + ("LastAccessed", datetime.datetime), + ("Size", int), + ] return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) + + def generate_timeline(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + for row in self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ): + _depth, row_data = row + description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[4]}"' + yield description, timeliner.TimeLinerType.CHANGED, row_data[5] + yield description, timeliner.TimeLinerType.MODIFIED, row_data[6] + yield description, timeliner.TimeLinerType.ACCESSED, row_data[7] diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index c4e2587f4..b9321f369 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,6 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import stat, datetime from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework @@ -265,8 +266,26 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): for fd_num, filp in enumerate(fds): if filp != 0: full_path = LinuxUtilities.path_for_file(context, task, filp) + dentry = filp.get_dentry() + if dentry != 0: + inode_object = dentry.d_inode + inode_num = inode_object.i_ino + file_size = inode_object.i_size # file size in bytes + imode = stat.filemode( + inode_object.i_mode + ) # file type & Permissions - yield fd_num, filp, full_path + # Timestamps + ctime = datetime.datetime.fromtimestamp( + inode_object.i_ctime.tv_sec + ) # last change time + mtime = datetime.datetime.fromtimestamp( + inode_object.i_mtime.tv_sec + ) # last modify time + atime = datetime.datetime.fromtimestamp( + inode_object.i_atime.tv_sec + ) # last access time + yield fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size @classmethod def mask_mods_list( From b8b146a4441f054bccf5697868c26250ea99d86d Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 24 Jul 2024 10:59:52 +0100 Subject: [PATCH 123/263] Windows: update handles plugin to use a default SAR value of 0x10 if decoding fails. Produce warnings when this happens. Ref issue #1147 --- volatility3/framework/plugins/windows/handles.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index a19e7a397..2edf28e86 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -25,7 +25,7 @@ class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -175,10 +175,11 @@ class Handles(interfaces.plugins.PluginInterface): virtual_layer_name, func_addr_to_read, num_bytes_to_read ) except exceptions.InvalidAddressException: - vollog.debug( - f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}" + vollog.warning( + f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of 0x10" ) - return None + self._sar_value = 0x10 + return self._sar_value md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) @@ -198,9 +199,10 @@ class Handles(interfaces.plugins.PluginInterface): break if self._sar_value is None: - vollog.debug( - f"Failed to to locate SAR value having parsed {instruction_count} instructions" + vollog.warning( + f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of 0x10" ) + self._sar_value = 0x10 return self._sar_value From 7d52f7992d187e4c18425ca71171dc27a9710903 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 24 Jul 2024 11:06:26 +0100 Subject: [PATCH 124/263] Windows: Make the default sar value used in handles plugin a variable so if it needs to be changed it gets updated in one place only --- volatility3/framework/plugins/windows/handles.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 2edf28e86..abe154fa1 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -142,6 +142,7 @@ class Handles(interfaces.plugins.PluginInterface): pointers in the _HANDLE_TABLE_ENTRY which allows us to find the associated _OBJECT_HEADER. """ + DEFAULT_SAR_VALUE = 0x10 # to be used only when decoding fails if self._sar_value is None: if not has_capstone: @@ -178,7 +179,7 @@ class Handles(interfaces.plugins.PluginInterface): vollog.warning( f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of 0x10" ) - self._sar_value = 0x10 + self._sar_value = DEFAULT_SAR_VALUE return self._sar_value md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) @@ -202,7 +203,7 @@ class Handles(interfaces.plugins.PluginInterface): vollog.warning( f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of 0x10" ) - self._sar_value = 0x10 + self._sar_value = DEFAULT_SAR_VALUE return self._sar_value From e1065f9e788322faded83520da29cd5cbcd5d5ed Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Jul 2024 20:32:36 +0100 Subject: [PATCH 125/263] Fix up a missing super which @atcuno spotted --- volatility3/framework/plugins/windows/threads.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index 55720c6f3..d57911650 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -20,6 +20,7 @@ class Threads(thrdscan.ThrdScan): def __init__(self): self.implementation = self.list_process_threads + super().__init__() @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From f4cbed856bcfb011707588d5563cd7144450ba1e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Jul 2024 20:47:05 +0100 Subject: [PATCH 126/263] Fix up missing filter parameter --- volatility3/framework/plugins/windows/thrdscan.py | 7 +++++-- volatility3/framework/plugins/windows/threads.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index be7097347..6e664e052 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -19,7 +19,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # version 2.6.0 adds support for scanning for 'Ethread' structures by pool tags _required_framework_version = (2, 6, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) def __init__(self, *args, **kwargs): self.implementation = self.scan_threads @@ -100,11 +100,14 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) for ethread in self.implementation(self.context, kernel_name): info = self.gather_thread_info(ethread) + if info: yield (0, info) def generate_timeline(self): - for row in self._generator(): + filt_func = self.filter_func(self.config) + + for row in self._generator(filt_func): _depth, row_data = row row_dict = {} ( diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index d57911650..ae70e717b 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -38,7 +38,7 @@ class Threads(thrdscan.ThrdScan): optional=True, ), requirements.PluginRequirement( - name="thrdscan", plugin=thrdscan.ThrdScan, version=(1, 0, 0) + name="thrdscan", plugin=thrdscan.ThrdScan, version=(1, 1, 0) ), ] From fceb79ba8b86e95919451ee29ca9fdfccd3ebc3e Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 08:34:04 +0200 Subject: [PATCH 127/263] use with context, bump release --- volatility3/framework/plugins/yarascan.py | 29 +++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 03a45b8df..7957aa0c5 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -27,11 +27,11 @@ except ImportError: if tuple(int(x) for x in yara.__version__.split(".")) < (3, 8): raise ImportError - vollog.info("Using yara-python module") + vollog.debug("Using yara-python module") except ImportError: vollog.info( - "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available" + "Neither yara-x nor yara-python (>3.8.0) module not found, plugin (and dependent plugins) not available" ) raise @@ -51,7 +51,7 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: for match in self._rules.match(data=data): - if self.st_object: + if YaraScan.yara_returns_instances(): for match_string in match.strings: for instance in match_string.instances: yield ( @@ -72,11 +72,13 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): @staticmethod def from_compiled_file(filepath): - return yara.load(file=resources.ResourceAccessor().open(filepath, "rb")) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara.load(file=fp) @staticmethod def from_file(filepath): - return yara.compile(file=resources.ResourceAccessor().open(filepath, "rb")) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara.compile(file=fp) class YaraXScanner(interfaces.layers.ScannerInterface): @@ -109,15 +111,13 @@ class YaraXScanner(interfaces.layers.ScannerInterface): @staticmethod def from_compiled_file(filepath): - return yara_x.Rules.deserialize_from( - file=resources.ResourceAccessor().open(filepath, "rb") - ) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara_x.Rules.deserialize_from(file=fp) @staticmethod def from_file(filepath): - return yara_x.compile( - resources.ResourceAccessor().open(filepath, "rb").read().decode() - ) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara_x.compile(fp.read().decode()) YaraScanner = YaraXScanner if USE_YARA_X else YaraPythonScanner @@ -127,7 +127,7 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 3, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -184,6 +184,11 @@ class YaraScan(plugins.PluginInterface): ), ] + @classmethod + def yara_returns_instances(cls) -> bool: + st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < (4, 3) + return st_object + @classmethod def process_yara_options(cls, config: Dict[str, Any]): rules = None From 1616a898ae5ddfe7369c0ea90306546abd4d929e Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 08:37:27 +0200 Subject: [PATCH 128/263] restore comment --- volatility3/framework/plugins/yarascan.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 7957aa0c5..3bf894d06 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -171,6 +171,8 @@ class YaraScan(plugins.PluginInterface): description="Yara rules (as a file)", optional=True, ), + # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code + # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later requirements.URIRequirement( name="yara_compiled_file", description="Yara compiled rules (as a file)", From c6727ffb701d31067e6617d49904614bedb63ac9 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 08:43:21 +0200 Subject: [PATCH 129/263] fix f-string for previous python release --- volatility3/framework/plugins/yarascan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 3bf894d06..8d2f242d9 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -204,11 +204,11 @@ class YaraScan(plugins.PluginInterface): rule += " wide ascii" rules = YaraScanner.get_rule(rule) elif config.get("yara_file") is not None: - vollog.debug(f"Plain file: {config["yara_file"]} - yara-x: {USE_YARA_X}") + vollog.debug(f"Plain file: {config['yara_file']} - yara-x: {USE_YARA_X}") rules = YaraScanner.from_file(config["yara_file"]) elif config.get("yara_compiled_file") is not None: vollog.debug( - f"Compiled file: {config["yara_compiled_file"]} - yara-x: {USE_YARA_X}" + f"Compiled file: {config['yara_compiled_file']} - yara-x: {USE_YARA_X}" ) rules = YaraScanner.from_compiled_file(config["yara_compiled_file"]) else: From dc8dc9b078d42e41698ff93f2e65c5ee99aa2172 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 14:37:18 +0200 Subject: [PATCH 130/263] fix yara depending plugins --- .../framework/plugins/linux/vmayarascan.py | 4 +- .../framework/plugins/windows/mftscan.py | 6 +-- .../framework/plugins/windows/vadyarascan.py | 49 +++++++++++++------ volatility3/framework/plugins/yarascan.py | 24 ++++----- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index eda0d7dca..6d055707c 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -31,10 +31,10 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) ), requirements.ModuleRequirement( name="kernel", diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 1b1a1b45e..9e6585345 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -29,7 +29,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) ), ] @@ -38,7 +38,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\\*|BAAD/"} + {"yara_string": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File @@ -197,7 +197,7 @@ class ADS(interfaces.plugins.PluginInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\\*|BAAD/"} + {"yara_string": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index dc318dd93..4a84a1285 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -33,7 +33,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="yarascan", plugin=yarascan.YaraScan, version=(1, 3, 0) + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), requirements.ListRequirement( name="pid", @@ -73,26 +73,43 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ) continue - for match in rules.match(data=layer.read(start, size, True)): - if yarascan.YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: + if not yarascan.YaraScan._yara_x: + for match in rules.match(data=layer.read(start, size, True)): + if yarascan.YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield 0, ( + format_hints.Hex(instance.offset + start), + task.UniqueProcessId, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield 0, ( + format_hints.Hex(offset + start), + task.UniqueProcessId, + match.rule, + name, + value, + ) + else: + data = layer.read(start, size, True) + results = rules.scan(data) + for match in results.matching_rules: + for match_string in match.patterns: + for instance in match_string.matches: yield 0, ( format_hints.Hex(instance.offset + start), task.UniqueProcessId, - match.rule, + f"{match.namespace}.{match.identifier}", match_string.identifier, - instance.matched_data, + data[ + instance.offset : instance.offset + + instance.length + ], ) - else: - for offset, name, value in match.strings: - yield 0, ( - format_hints.Hex(offset + start), - task.UniqueProcessId, - match.rule, - name, - value, - ) @staticmethod def get_vad_maps( diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 8d2f242d9..6a4dd9251 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -36,8 +36,8 @@ except ImportError: raise -class YaraPythonScanner(interfaces.layers.ScannerInterface): - _version = (2, 0, 0) +class BaseYaraScanner(interfaces.layers.ScannerInterface): + _version = (2, 1, 0) # yara.Rules isn't exposed, so we can't type this properly def __init__(self, rules) -> None: @@ -45,6 +45,11 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules + + +class YaraPythonScanner(BaseYaraScanner): + def __init__(self, rules) -> None: + super().__init__(rules) self.st_object = not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) def __call__( @@ -81,16 +86,7 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): return yara.compile(file=fp) -class YaraXScanner(interfaces.layers.ScannerInterface): - _version = (2, 0, 0) - - # yara.Rules isn't exposed, so we can't type this properly - def __init__(self, rules) -> None: - super().__init__() - if rules is None: - raise ValueError("No rules provided to YaraScanner") - self._rules = rules - +class YaraXScanner(BaseYaraScanner): def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: @@ -128,6 +124,7 @@ class YaraScan(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (2, 0, 0) + _yara_x = USE_YARA_X @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -188,8 +185,7 @@ class YaraScan(plugins.PluginInterface): @classmethod def yara_returns_instances(cls) -> bool: - st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < (4, 3) - return st_object + return not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) @classmethod def process_yara_options(cls, config: Dict[str, Any]): From 6f3f645dbcfb94b797f2e244e56e43dd8e4cba0a Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 25 Jul 2024 17:59:10 +0100 Subject: [PATCH 131/263] Windows: update handles plugin sar warnings to use DEFAULT_SAR_VALUE var --- volatility3/framework/plugins/windows/handles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index abe154fa1..6010b4c72 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -177,7 +177,7 @@ class Handles(interfaces.plugins.PluginInterface): ) except exceptions.InvalidAddressException: vollog.warning( - f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of 0x10" + f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of {hex(DEFAULT_SAR_VALUE)}}" ) self._sar_value = DEFAULT_SAR_VALUE return self._sar_value @@ -201,7 +201,7 @@ class Handles(interfaces.plugins.PluginInterface): if self._sar_value is None: vollog.warning( - f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of 0x10" + f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of {hex(DEFAULT_SAR_VALUE)}" ) self._sar_value = DEFAULT_SAR_VALUE From 799afe6e51558dc4fe3828276f28b888e866e708 Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 25 Jul 2024 18:02:30 +0100 Subject: [PATCH 132/263] Windows: fix type in handles plugin --- volatility3/framework/plugins/windows/handles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 6010b4c72..3e5a2fd82 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -177,7 +177,7 @@ class Handles(interfaces.plugins.PluginInterface): ) except exceptions.InvalidAddressException: vollog.warning( - f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of {hex(DEFAULT_SAR_VALUE)}}" + f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of {hex(DEFAULT_SAR_VALUE)}" ) self._sar_value = DEFAULT_SAR_VALUE return self._sar_value From 7a03e9deabc6f743eb8e201fe75dcdd810339635 Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 25 Jul 2024 18:21:12 +0100 Subject: [PATCH 133/263] Windows: Add a _version to the filescan plugin --- volatility3/framework/plugins/windows/filescan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/filescan.py b/volatility3/framework/plugins/windows/filescan.py index 34c0c60d3..82566361d 100644 --- a/volatility3/framework/plugins/windows/filescan.py +++ b/volatility3/framework/plugins/windows/filescan.py @@ -14,6 +14,7 @@ class FileScan(interfaces.plugins.PluginInterface): """Scans for file objects present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): From 650dd06245918f1b14d8477eff85a743c9e42c4f Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 27 Jul 2024 16:03:59 +0200 Subject: [PATCH 134/263] Modifications following the review --- volatility3/framework/plugins/linux/lsof.py | 74 +++++++++++-------- .../framework/symbols/linux/__init__.py | 51 +++++++------ 2 files changed, 70 insertions(+), 55 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index f9aeafe14..3bbc855f9 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # """A module containing a collection of plugins that produce data typically @@ -12,16 +12,17 @@ from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.framework.symbols import linux from volatility3.plugins.linux import pslist +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class Lsof(plugins.PluginInterface): +class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -46,7 +47,7 @@ class Lsof(plugins.PluginInterface): ] @classmethod - def list_fds( + def list_fds_and_inodes( cls, context: interfaces.context.ContextInterface, symbol_table: str, @@ -67,27 +68,38 @@ class Lsof(plugins.PluginInterface): ) for fd_fields in fd_generator: - yield pid, task_comm, task, fd_fields + fd_num, filp, full_path = fd_fields + inode_metadata = linux.LinuxUtilities.get_inode_metadata(context, filp) + try: + inode_num, file_size, imode, ctime, mtime, atime = next( + inode_metadata + ) + except Exception as e: + vollog.warning( + f"Can't get inode metadata for file descriptor {fd_num}: {e}" + ) + continue + yield pid, task_comm, task, fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size def _generator(self, pids, symbol_table): filter_func = pslist.PsList.create_pid_filter(pids) - fds_generator = self.list_fds( + fds_generator = self.list_fds_and_inodes( self.context, symbol_table, filter_func=filter_func ) - - for pid, task_comm, _task, fd_fields in fds_generator: - ( - fd_num, - _filp, - full_path, - inode_num, - imode, - ctime, - mtime, - atime, - file_size, - ) = fd_fields - + for ( + pid, + task_comm, + task, + fd_num, + filp, + full_path, + inode_num, + imode, + ctime, + mtime, + atime, + file_size, + ) in fds_generator: fields = ( pid, task_comm, @@ -113,22 +125,20 @@ class Lsof(plugins.PluginInterface): ("Path", str), ("Inode", int), ("Mode", str), - ("LastChange", datetime.datetime), - ("LastModify", datetime.datetime), - ("LastAccessed", datetime.datetime), + ("Changed", datetime.datetime), + ("Modified", datetime.datetime), + ("Accessed", datetime.datetime), ("Size", int), ] return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) def generate_timeline(self): + pids = self.config.get("pid", None) + symbol_table = self.config["kernel"] filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - for row in self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ): + for row in self._generator(pids, symbol_table): _depth, row_data = row - description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[4]}"' - yield description, timeliner.TimeLinerType.CHANGED, row_data[5] - yield description, timeliner.TimeLinerType.MODIFIED, row_data[6] - yield description, timeliner.TimeLinerType.ACCESSED, row_data[7] + description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' + yield description, timeliner.TimeLinerType.CHANGED, row_data[6] + yield description, timeliner.TimeLinerType.MODIFIED, row_data[7] + yield description, timeliner.TimeLinerType.ACCESSED, row_data[8] diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index b9321f369..2b97bc5f4 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,9 +1,8 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import stat, datetime from typing import Iterator, List, Tuple, Optional, Union - +import logging, datetime, stat from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility @@ -62,7 +61,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 0) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -266,26 +265,32 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): for fd_num, filp in enumerate(fds): if filp != 0: full_path = LinuxUtilities.path_for_file(context, task, filp) - dentry = filp.get_dentry() - if dentry != 0: - inode_object = dentry.d_inode - inode_num = inode_object.i_ino - file_size = inode_object.i_size # file size in bytes - imode = stat.filemode( - inode_object.i_mode - ) # file type & Permissions - # Timestamps - ctime = datetime.datetime.fromtimestamp( - inode_object.i_ctime.tv_sec - ) # last change time - mtime = datetime.datetime.fromtimestamp( - inode_object.i_mtime.tv_sec - ) # last modify time - atime = datetime.datetime.fromtimestamp( - inode_object.i_atime.tv_sec - ) # last access time - yield fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size + yield fd_num, filp, full_path + + @classmethod + def get_inode_metadata(cls, context: interfaces.context.ContextInterface, filp): + """ + A helper function that gets the inodes metadata from a file descriptor + """ + dentry = filp.get_dentry() + if dentry != 0: + inode_object = dentry.d_inode + inode_num = inode_object.i_ino + file_size = inode_object.i_size # file size in bytes + imode = stat.filemode(inode_object.i_mode) # file type & Permissions + + # Timestamps + ctime = datetime.datetime.fromtimestamp( + inode_object.i_ctime.tv_sec + ) # last change time + mtime = datetime.datetime.fromtimestamp( + inode_object.i_mtime.tv_sec + ) # last modify time + atime = datetime.datetime.fromtimestamp( + inode_object.i_atime.tv_sec + ) # last access time + yield inode_num, file_size, imode, ctime, mtime, atime @classmethod def mask_mods_list( From 7024588076adf95c2d6667c851cddf87e8c68555 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 27 Jul 2024 16:09:52 +0200 Subject: [PATCH 135/263] Code clean --- volatility3/framework/plugins/linux/lsof.py | 1 - volatility3/framework/symbols/linux/__init__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 3bbc855f9..98a215ecf 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -135,7 +135,6 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): def generate_timeline(self): pids = self.config.get("pid", None) symbol_table = self.config["kernel"] - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for row in self._generator(pids, symbol_table): _depth, row_data = row description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 2b97bc5f4..1b3f75e98 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # from typing import Iterator, List, Tuple, Optional, Union -import logging, datetime, stat +import datetime, stat from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility From 815252c9ba31913d22e8836183828059217c4e9d Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 27 Jul 2024 16:35:22 +0200 Subject: [PATCH 136/263] Adding watchdogs --- volatility3/framework/plugins/linux/lsof.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 98a215ecf..c1de48c1a 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -78,7 +78,13 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.warning( f"Can't get inode metadata for file descriptor {fd_num}: {e}" ) - continue + # Yield NotAvailableValue for each field in case of an exception + inode_num = renderers.NotAvailableValue() + file_size = renderers.NotAvailableValue() + imode = renderers.NotAvailableValue() + ctime = renderers.NotAvailableValue() + mtime = renderers.NotAvailableValue() + atime = renderers.NotAvailableValue() yield pid, task_comm, task, fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size def _generator(self, pids, symbol_table): From f44ceb321d0c3b10249c1c697b3fb0c40f38ba39 Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Sat, 27 Jul 2024 20:11:39 -0500 Subject: [PATCH 137/263] Added psxview --- .../framework/plugins/windows/psxview.py | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 volatility3/framework/plugins/windows/psxview.py diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py new file mode 100644 index 000000000..3bf6c27f5 --- /dev/null +++ b/volatility3/framework/plugins/windows/psxview.py @@ -0,0 +1,188 @@ +import datetime, logging + +from volatility3.framework import constants, exceptions +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, TreeGrid +from volatility3.plugins.windows import handles, info, pslist, psscan, sessions, thrdscan + +vollog = logging.getLogger(__name__) + +class PsXView(plugins.PluginInterface): + """Lists all processes found via 6 of the methods described in \"The Art of Memory Forensics,\" which may help + identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this + plugin's output in a terminal.""" + # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality + # which the original plugin to do it. + + # I don't think it's worth including the sessions method either because both the original psxview plugin + # and Volatility3's sessions plugin begin with the list of processes found by PsList. + # The original psxview plugin's session code essentially just filters the pslist for processes + # whose session ID is not None. I've matched this in my code, but again, it doesn't seem worth including. + + # Lastly, I've omitted the pspcid method because I could not for the life of me get it to work. I saved the + # code I do have from it, and will happily share it if anyone else wants to add it. + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [requirements.ModuleRequirement(name="kernel", description="Windows kernel", architectures=["Intel32", "Intel64"]), + requirements.VersionRequirement(name="info", component=info.Info, version=(1, 0, 0)), + requirements.VersionRequirement(name="pslist", component=pslist.PsList, version=(2, 0, 0)), + requirements.VersionRequirement(name="psscan", component=psscan.PsScan, version=(1, 0, 0)), + requirements.VersionRequirement(name="thrdscan", component=thrdscan.ThrdScan, version=(1, 0, 0)), + requirements.VersionRequirement(name="handles", component=handles.Handles, version=(1, 0, 0)), + requirements.VersionRequirement(name="sessions", component=sessions.Sessions, version=(0, 0, 0)), + requirements.BooleanRequirement(name="identify-expected", description="In the plugin's output, replace false with \ + normal where false is the expected result for a Windows machine running normally. \ + Keep in mind that this plugin uses simple checks to identify \"normal\" behavior, \ + so you may want to double-check the legitimacy of these processes yourself.", optional=True), + requirements.BooleanRequirement(name="physical-offsets", description="List processes with phyiscall offsets instead of virtual offsets.", optional=True)] + + def proc_name_to_string(self, proc): + return proc.ImageFileName.cast("string", max_length=proc.ImageFileName.vol.count, errors="replace") + + def is_ascii(self, str): + return str.split('.')[0].isalnum() + + def filter_garbage_procs(self, proc_list): + return [p for p in proc_list if p.is_valid() and self.is_ascii(self.proc_name_to_string(p))] + + def translate_offset(self, offset): + if self.config["physical-offsets"]: + return offset + + kernel = self.context.modules[self.config["kernel"]] + layer_name = kernel.layer_name + + try: + offset = list(self.context.layers[layer_name].mapping(offset=offset, length=0))[0][2] + except: + # already have physical address + pass + + return offset + + def proc_list_to_dict(self, tasks): + return {self.translate_offset(proc.vol.offset):proc for proc in tasks} + + def check_pslist(self, tasks): + res = self.filter_garbage_procs(tasks) + return self.proc_list_to_dict(tasks) + + def check_psscan(self, layer_name, symbol_table): + res = psscan.PsScan.scan_processes(context=self.context, layer_name=layer_name, symbol_table=symbol_table) + res = self.filter_garbage_procs(res) + + return self.proc_list_to_dict(res) + + def check_thrdscan(self): + ret = [] + + for ethread in thrdscan.ThrdScan.scan_threads(self.context, module_name='kernel'): + process = None + try: + process = ethread.owning_process() + if not process.is_valid(): + continue + + ret.append(process) + except AttributeError: + vollog.log(constants.LOGLEVEL_VVV, "Unable to find the owning process of ethread") + + return self.proc_list_to_dict(ret) + + def check_csrss_handles(self, tasks, layer_name, symbol_table): + ret = [] + + for p in tasks: + name = self.proc_name_to_string(p) + if name == 'csrss.exe': + try: + if p.has_member("ObjectTable"): + handles_plugin = handles.Handles(context=self.context, config_path=self.config_path) + hndls = list(handles_plugin.handles(p.ObjectTable)) + for h in hndls: + if (h.get_object_type(handles_plugin.get_type_map(self.context, layer_name, symbol_table)) == "Process"): + ret.append(h.Body.cast("_EPROCESS")) + + except exceptions.InvalidAddressException: + vollog.log(constants.LOGLEVEL_VVV, "Cannot access eprocess object table") + + ret = self.filter_garbage_procs(ret) + return self.proc_list_to_dict(ret) + + def check_session(self, pslist_procs): + procs = [p for p in pslist_procs if p.get_session_id() != None] + + return self.proc_list_to_dict(procs) + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + layer_name = kernel.layer_name + symbol_table = kernel.symbol_table_name + + kdbg_list_processes = list(pslist.PsList.list_processes(context=self.context, layer_name=layer_name, symbol_table=symbol_table)) + + processes = {} + + processes['pslist'] = self.check_pslist(kdbg_list_processes) + processes['psscan'] = self.check_psscan(layer_name, symbol_table) + processes['thrdscan'] = self.check_thrdscan() + processes['csrss'] = self.check_csrss_handles(kdbg_list_processes, layer_name, symbol_table) + processes['sessions'] = self.check_session(kdbg_list_processes) + + seen_offsets = set() + for source in processes: + for offset in processes[source]: + if offset not in seen_offsets: + seen_offsets.add(offset) + proc = processes[source][offset] + + pid = proc.UniqueProcessId + name = self.proc_name_to_string(proc) + + exit_time = proc.get_exit_time() + if (type(exit_time) != datetime.datetime): + exit_time = "" + else: + exit_time = str(exit_time) + + in_sources = {src:str(offset in processes[src]) for src in processes} + + if self.config["identify-expected"]: + f = "False" + n = "Normal" + + if in_sources["pslist"] == f: + if exit_time != "": + in_sources["pslist"] = n + + if in_sources["thrdscan"] == f: + if exit_time != "": + in_sources["thrdscan"] = n + + if in_sources["csrss"] == f: + if name.lower() in ["system", "smss.exe", "csrss.exe"]: + in_sources["csrss"] = n + elif exit_time != "": + in_sources["csrss"] = n + + if in_sources["sessions"] == f: + if name.lower() in ["system", "smss.exe"]: + in_sources["sessions"] = n + + yield (0, (format_hints.Hex(offset), name, pid, in_sources["pslist"], + in_sources["psscan"], in_sources["thrdscan"], in_sources["csrss"], + in_sources["sessions"], exit_time)) + + + def run(self): + offset_type = "(Physical)" if self.config["physical-offsets"] else "(Virtual)" + offset_str = "Offset" + offset_type + + return TreeGrid([(offset_str, format_hints.Hex), ("Name", str), ("PID", int), ("pslist", str), ("psscan", str), + ("thrdscan", str), ("csrss", str), ("sessions", str), ("Exit Time", str) ], self._generator()) \ No newline at end of file From 824b0599f20d2a1f0e7f16ca08f9b498898e4c52 Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Sat, 27 Jul 2024 20:35:08 -0500 Subject: [PATCH 138/263] formatted --- .../framework/plugins/windows/psxview.py | 203 +++++++++++++----- 1 file changed, 147 insertions(+), 56 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 3bf6c27f5..1fde5b12d 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -4,19 +4,28 @@ from volatility3.framework import constants, exceptions from volatility3.framework.interfaces import plugins from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints, TreeGrid -from volatility3.plugins.windows import handles, info, pslist, psscan, sessions, thrdscan +from volatility3.plugins.windows import ( + handles, + info, + pslist, + psscan, + sessions, + thrdscan, +) vollog = logging.getLogger(__name__) + class PsXView(plugins.PluginInterface): - """Lists all processes found via 6 of the methods described in \"The Art of Memory Forensics,\" which may help + """Lists all processes found via 6 of the methods described in \"The Art of Memory Forensics,\" which may help identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this plugin's output in a terminal.""" - # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality - # which the original plugin to do it. - # I don't think it's worth including the sessions method either because both the original psxview plugin - # and Volatility3's sessions plugin begin with the list of processes found by PsList. + # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality + # which the original plugin to do it. + + # I don't think it's worth including the sessions method either because both the original psxview plugin + # and Volatility3's sessions plugin begin with the list of processes found by PsList. # The original psxview plugin's session code essentially just filters the pslist for processes # whose session ID is not None. I've matched this in my code, but again, it doesn't seem worth including. @@ -28,52 +37,88 @@ class PsXView(plugins.PluginInterface): @classmethod def get_requirements(cls): - return [requirements.ModuleRequirement(name="kernel", description="Windows kernel", architectures=["Intel32", "Intel64"]), - requirements.VersionRequirement(name="info", component=info.Info, version=(1, 0, 0)), - requirements.VersionRequirement(name="pslist", component=pslist.PsList, version=(2, 0, 0)), - requirements.VersionRequirement(name="psscan", component=psscan.PsScan, version=(1, 0, 0)), - requirements.VersionRequirement(name="thrdscan", component=thrdscan.ThrdScan, version=(1, 0, 0)), - requirements.VersionRequirement(name="handles", component=handles.Handles, version=(1, 0, 0)), - requirements.VersionRequirement(name="sessions", component=sessions.Sessions, version=(0, 0, 0)), - requirements.BooleanRequirement(name="identify-expected", description="In the plugin's output, replace false with \ + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="info", component=info.Info, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="psscan", component=psscan.PsScan, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="thrdscan", component=thrdscan.ThrdScan, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="handles", component=handles.Handles, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="sessions", component=sessions.Sessions, version=(0, 0, 0) + ), + requirements.BooleanRequirement( + name="identify-expected", + description='In the plugin\'s output, replace false with \ normal where false is the expected result for a Windows machine running normally. \ - Keep in mind that this plugin uses simple checks to identify \"normal\" behavior, \ - so you may want to double-check the legitimacy of these processes yourself.", optional=True), - requirements.BooleanRequirement(name="physical-offsets", description="List processes with phyiscall offsets instead of virtual offsets.", optional=True)] - + Keep in mind that this plugin uses simple checks to identify "normal" behavior, \ + so you may want to double-check the legitimacy of these processes yourself.', + optional=True, + ), + requirements.BooleanRequirement( + name="physical-offsets", + description="List processes with phyiscall offsets instead of virtual offsets.", + optional=True, + ), + ] + def proc_name_to_string(self, proc): - return proc.ImageFileName.cast("string", max_length=proc.ImageFileName.vol.count, errors="replace") + return proc.ImageFileName.cast( + "string", max_length=proc.ImageFileName.vol.count, errors="replace" + ) def is_ascii(self, str): - return str.split('.')[0].isalnum() - + return str.split(".")[0].isalnum() + def filter_garbage_procs(self, proc_list): - return [p for p in proc_list if p.is_valid() and self.is_ascii(self.proc_name_to_string(p))] - + return [ + p + for p in proc_list + if p.is_valid() and self.is_ascii(self.proc_name_to_string(p)) + ] + def translate_offset(self, offset): if self.config["physical-offsets"]: return offset - + kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name try: - offset = list(self.context.layers[layer_name].mapping(offset=offset, length=0))[0][2] + offset = list( + self.context.layers[layer_name].mapping(offset=offset, length=0) + )[0][2] except: # already have physical address pass return offset - + def proc_list_to_dict(self, tasks): - return {self.translate_offset(proc.vol.offset):proc for proc in tasks} - + return {self.translate_offset(proc.vol.offset): proc for proc in tasks} + def check_pslist(self, tasks): res = self.filter_garbage_procs(tasks) return self.proc_list_to_dict(tasks) - + def check_psscan(self, layer_name, symbol_table): - res = psscan.PsScan.scan_processes(context=self.context, layer_name=layer_name, symbol_table=symbol_table) + res = psscan.PsScan.scan_processes( + context=self.context, layer_name=layer_name, symbol_table=symbol_table + ) res = self.filter_garbage_procs(res) return self.proc_list_to_dict(res) @@ -81,7 +126,9 @@ class PsXView(plugins.PluginInterface): def check_thrdscan(self): ret = [] - for ethread in thrdscan.ThrdScan.scan_threads(self.context, module_name='kernel'): + for ethread in thrdscan.ThrdScan.scan_threads( + self.context, module_name="kernel" + ): process = None try: process = ethread.owning_process() @@ -90,50 +137,70 @@ class PsXView(plugins.PluginInterface): ret.append(process) except AttributeError: - vollog.log(constants.LOGLEVEL_VVV, "Unable to find the owning process of ethread") + vollog.log( + constants.LOGLEVEL_VVV, + "Unable to find the owning process of ethread", + ) return self.proc_list_to_dict(ret) - + def check_csrss_handles(self, tasks, layer_name, symbol_table): ret = [] for p in tasks: name = self.proc_name_to_string(p) - if name == 'csrss.exe': + if name == "csrss.exe": try: if p.has_member("ObjectTable"): - handles_plugin = handles.Handles(context=self.context, config_path=self.config_path) + handles_plugin = handles.Handles( + context=self.context, config_path=self.config_path + ) hndls = list(handles_plugin.handles(p.ObjectTable)) for h in hndls: - if (h.get_object_type(handles_plugin.get_type_map(self.context, layer_name, symbol_table)) == "Process"): + if ( + h.get_object_type( + handles_plugin.get_type_map( + self.context, layer_name, symbol_table + ) + ) + == "Process" + ): ret.append(h.Body.cast("_EPROCESS")) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, "Cannot access eprocess object table") + vollog.log( + constants.LOGLEVEL_VVV, "Cannot access eprocess object table" + ) ret = self.filter_garbage_procs(ret) return self.proc_list_to_dict(ret) def check_session(self, pslist_procs): procs = [p for p in pslist_procs if p.get_session_id() != None] - + return self.proc_list_to_dict(procs) def _generator(self): kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name - symbol_table = kernel.symbol_table_name + symbol_table = kernel.symbol_table_name + + kdbg_list_processes = list( + pslist.PsList.list_processes( + context=self.context, layer_name=layer_name, symbol_table=symbol_table + ) + ) - kdbg_list_processes = list(pslist.PsList.list_processes(context=self.context, layer_name=layer_name, symbol_table=symbol_table)) - processes = {} - processes['pslist'] = self.check_pslist(kdbg_list_processes) - processes['psscan'] = self.check_psscan(layer_name, symbol_table) - processes['thrdscan'] = self.check_thrdscan() - processes['csrss'] = self.check_csrss_handles(kdbg_list_processes, layer_name, symbol_table) - processes['sessions'] = self.check_session(kdbg_list_processes) + processes["pslist"] = self.check_pslist(kdbg_list_processes) + processes["psscan"] = self.check_psscan(layer_name, symbol_table) + processes["thrdscan"] = self.check_thrdscan() + processes["csrss"] = self.check_csrss_handles( + kdbg_list_processes, layer_name, symbol_table + ) + processes["sessions"] = self.check_session(kdbg_list_processes) seen_offsets = set() for source in processes: @@ -146,12 +213,14 @@ class PsXView(plugins.PluginInterface): name = self.proc_name_to_string(proc) exit_time = proc.get_exit_time() - if (type(exit_time) != datetime.datetime): + if type(exit_time) != datetime.datetime: exit_time = "" else: exit_time = str(exit_time) - in_sources = {src:str(offset in processes[src]) for src in processes} + in_sources = { + src: str(offset in processes[src]) for src in processes + } if self.config["identify-expected"]: f = "False" @@ -173,16 +242,38 @@ class PsXView(plugins.PluginInterface): if in_sources["sessions"] == f: if name.lower() in ["system", "smss.exe"]: - in_sources["sessions"] = n + in_sources["sessions"] = n + + yield ( + 0, + ( + format_hints.Hex(offset), + name, + pid, + in_sources["pslist"], + in_sources["psscan"], + in_sources["thrdscan"], + in_sources["csrss"], + in_sources["sessions"], + exit_time, + ), + ) - yield (0, (format_hints.Hex(offset), name, pid, in_sources["pslist"], - in_sources["psscan"], in_sources["thrdscan"], in_sources["csrss"], - in_sources["sessions"], exit_time)) - - def run(self): offset_type = "(Physical)" if self.config["physical-offsets"] else "(Virtual)" offset_str = "Offset" + offset_type - return TreeGrid([(offset_str, format_hints.Hex), ("Name", str), ("PID", int), ("pslist", str), ("psscan", str), - ("thrdscan", str), ("csrss", str), ("sessions", str), ("Exit Time", str) ], self._generator()) \ No newline at end of file + return TreeGrid( + [ + (offset_str, format_hints.Hex), + ("Name", str), + ("PID", int), + ("pslist", str), + ("psscan", str), + ("thrdscan", str), + ("csrss", str), + ("sessions", str), + ("Exit Time", str), + ], + self._generator(), + ) From 44f26c928eddaaf6743ac07b22331ad082e86bc1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jul 2024 12:35:02 +0100 Subject: [PATCH 139/263] Add in shtab autocompletion --- volatility3/cli/__init__.py | 22 +++++++++++++++++++--- volatility3/cli/volargparse.py | 6 ++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 6b17edac0..883b54ac4 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -22,6 +22,13 @@ import traceback from typing import Any, Dict, List, Tuple, Type, Union from urllib import parse, request +try: + import shtab + + HAS_SHTAB = True +except ImportError: + HAS_SHTAB = False + from volatility3.cli import text_filter import volatility3.plugins import volatility3.symbols @@ -106,6 +113,9 @@ class CommandLine: ] ) + # Argument for doing autocompletion + print_completion_arg = "--print-completion" + # Load up system defaults delayed_logs, default_config = self.load_system_defaults("vol.json") @@ -246,10 +256,12 @@ class CommandLine: # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != "--help" and arg != "-h"] - partial_args, _ = parser.parse_known_args(known_args) - + partial_args, unknown_args = parser.parse_known_args(known_args) banner_output = sys.stdout - if renderers[partial_args.renderer].structured_output: + if ( + renderers[partial_args.renderer].structured_output + or print_completion_arg in unknown_args + ): banner_output = sys.stderr banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") @@ -351,6 +363,10 @@ class CommandLine: # Hand the plugin requirements over to the CLI (us) and let it construct the config tree # Run the argparser + if HAS_SHTAB: + # The autocompletion line must be after the partial_arg handling, so that it doesn't trip it + # before all the plugins have been added + shtab.add_argument_to(parser, [print_completion_arg]) args = parser.parse_args() if args.plugin is None: parser.error("Please select a plugin to run") diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index 3048a0885..3e7eb6751 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -21,8 +21,6 @@ class HelpfulSubparserAction(argparse._SubParsersAction): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - # We don't want the action self-check to kick in, so we remove the choices list, the check happens in __call__ - self.choices = None def __call__( self, @@ -100,3 +98,7 @@ class HelpfulArgParser(argparse.ArgumentParser): # return the number of arguments matched return len(match.group(1)) + + def _check_value(self, action, value): + if not isinstance(action, HelpfulSubparserAction): + return super()._check_value(action, value) From e89e77637776b14c61516d2c47a7148a2f13860a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jul 2024 12:41:31 +0100 Subject: [PATCH 140/263] Try out argcomplete as well --- vol.py | 1 + volatility3/cli/__init__.py | 21 ++++++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/vol.py b/vol.py index ff420cad5..c49d5985d 100755 --- a/vol.py +++ b/vol.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# PYTHON_ARGCOMPLETE_OK # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 883b54ac4..1209d7cdc 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -23,11 +23,11 @@ from typing import Any, Dict, List, Tuple, Type, Union from urllib import parse, request try: - import shtab + import argcomplete - HAS_SHTAB = True + HAS_ARGCOMPLETE = True except ImportError: - HAS_SHTAB = False + HAS_ARGCOMPLETE = False from volatility3.cli import text_filter import volatility3.plugins @@ -113,9 +113,6 @@ class CommandLine: ] ) - # Argument for doing autocompletion - print_completion_arg = "--print-completion" - # Load up system defaults delayed_logs, default_config = self.load_system_defaults("vol.json") @@ -256,12 +253,10 @@ class CommandLine: # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != "--help" and arg != "-h"] - partial_args, unknown_args = parser.parse_known_args(known_args) + partial_args, _ = parser.parse_known_args(known_args) + banner_output = sys.stdout - if ( - renderers[partial_args.renderer].structured_output - or print_completion_arg in unknown_args - ): + if renderers[partial_args.renderer].structured_output: banner_output = sys.stderr banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") @@ -363,10 +358,10 @@ class CommandLine: # Hand the plugin requirements over to the CLI (us) and let it construct the config tree # Run the argparser - if HAS_SHTAB: + if HAS_ARGCOMPLETE: # The autocompletion line must be after the partial_arg handling, so that it doesn't trip it # before all the plugins have been added - shtab.add_argument_to(parser, [print_completion_arg]) + argcomplete.autocomplete(parser) args = parser.parse_args() if args.plugin is None: parser.error("Please select a plugin to run") From 0d8fb76b3a48599ac9beb8eea5d79e46fe7b877e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jul 2024 19:57:18 +0100 Subject: [PATCH 141/263] Renderers: Allow BaseAbsentValues in value results Fixes #1216 --- volatility3/framework/renderers/format_hints.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index 6120b77c9..194e38099 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -10,6 +10,8 @@ Text renderers should attempt to honour all hints provided in this module where """ from typing import Type, Union +from volatility3.framework import interfaces + class Bin(int): """A class to indicate that the integer value should be represented as a @@ -66,3 +68,17 @@ class MultiTypeData(bytes): and self.split_nulls == other.split_nulls and self.show_hex == other.show_hex ) + + +BinOrAbsent = lambda x: ( + Bin(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x +) +HexOrAbsent = lambda x: ( + Hex(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x +) +HexBytesOrAbsent = lambda x: ( + HexBytes(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x +) +MultiTypeDataOrAbsent = lambda x: ( + MultiTypeData(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x +) From b659a060bd6caf37eb0e51560986cc58aab067de Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jul 2024 21:57:48 +0100 Subject: [PATCH 142/263] Renderers: Ensure the version is bumped so plugins can require the format_hints properly --- volatility3/framework/constants/_version.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 21c339a6e..f219fb0af 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,11 +1,9 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 7 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_MINOR = 8 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" -# TODO: At version 2.0.0, remove the symbol_shift feature - PACKAGE_VERSION = ( ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + VERSION_SUFFIX From 76414d3246717ba9dde2a3cdd94a8c1037a7374b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 29 Jul 2024 14:13:54 +1000 Subject: [PATCH 143/263] Renderers conversion fix: Create aware datetimes to represent times in UTC. Fix Python 3.12 datetime.utcfromtimestamp() deprecation. See warning note on https://docs.python.org/3/library/datetime.html#datetime.datetime.utcfromtimestamp: Because naive datetime objects are treated by many datetime methods as local times, it is preferred to use aware datetimes to represent times in UTC. As such, the recommended way to create an object representing a specific timestamp in UTC is by calling datetime.fromtimestamp(timestamp, tz=timezone.utc). Additionaly, datetime.utcfromtimestamp() is deprecated since 3.12 --- volatility3/framework/renderers/conversion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index bb18fcc8a..c833cc2cf 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -19,7 +19,7 @@ def wintime_to_datetime( return renderers.NotApplicableValue() unix_time = unix_time - 11644473600 try: - return datetime.datetime.utcfromtimestamp(unix_time) + return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc) # Windows sometimes throws OSErrors rather than ValueErrors when it can't convert a value except (ValueError, OSError): return renderers.UnparsableValue() @@ -34,7 +34,7 @@ def unixtime_to_datetime( if unixtime > 0: with contextlib.suppress(ValueError): - ret = datetime.datetime.utcfromtimestamp(unixtime) + ret = datetime.datetime.fromtimestamp(unixtime, datetime.timezone.utc) return ret From ece15c914f27e6fbb651e8b3f3ae7e525aa50138 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 29 Jul 2024 14:18:10 +1000 Subject: [PATCH 144/263] Renderers conversion exceptions fix: Since version 3.3 utcfromtimestamp() and fromtimestamp() Python datimetime module raises OverflowError instead of ValueError. As of today, Volatility3 requires Python 3.7.3 so we should only include OverflowError --- volatility3/framework/renderers/conversion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index c833cc2cf..c94201681 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -20,8 +20,8 @@ def wintime_to_datetime( unix_time = unix_time - 11644473600 try: return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc) - # Windows sometimes throws OSErrors rather than ValueErrors when it can't convert a value - except (ValueError, OSError): + # Windows sometimes throws OSErrors rather than OverflowError when it can't convert a value + except (OverflowError, OSError): return renderers.UnparsableValue() @@ -33,7 +33,7 @@ def unixtime_to_datetime( ) if unixtime > 0: - with contextlib.suppress(ValueError): + with contextlib.suppress(OverflowError): ret = datetime.datetime.fromtimestamp(unixtime, datetime.timezone.utc) return ret From b343734bae0d3090a5477997d27885d237096e50 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 29 Jul 2024 15:16:34 +1000 Subject: [PATCH 145/263] Renderers conversion exceptions fix: Even though the documentation states that OverflowError should be raised starting from version 3.3, it has been observed that ValueError is still being triggered. Also, in Linux, we noticed that OSError is also being raised in some cases. --- volatility3/framework/renderers/conversion.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index c94201681..864794860 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -20,8 +20,10 @@ def wintime_to_datetime( unix_time = unix_time - 11644473600 try: return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc) - # Windows sometimes throws OSErrors rather than OverflowError when it can't convert a value - except (OverflowError, OSError): + # Windows sometimes throws OSErrors rather than ValueError/OverflowError when it can't convert a value + # Since Python 3.3, this should raise OverflowError instead of ValueError. However, it was observed + # that even in Python 3.7.17, ValueError is still being raised. + except (ValueError, OverflowError, OSError): return renderers.UnparsableValue() @@ -33,7 +35,9 @@ def unixtime_to_datetime( ) if unixtime > 0: - with contextlib.suppress(OverflowError): + # Since Python 3.3, this should raise OverflowError instead of ValueError. However, it was observed + # that even in Python 3.7.17, ValueError is still being raised. OSError is also raised on Linux + with contextlib.suppress(ValueError, OverflowError, OSError): ret = datetime.datetime.fromtimestamp(unixtime, datetime.timezone.utc) return ret From ba7ec0059960fd6470254e8b3cd4ae15e89ea174 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 29 Jul 2024 17:55:56 -0500 Subject: [PATCH 146/263] Windows: Fixes bad structure member in callbacks This fixes a bug in the x64 callbacks symbols. The `NotificationRoutine` is currently an `unsigned int` instead of a void pointer. This prevents the correct mapping of the notification routine to the kernel module that contains it. --- volatility3/framework/symbols/windows/callbacks-x64.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/callbacks-x64.json b/volatility3/framework/symbols/windows/callbacks-x64.json index 3f891b94f..705f2361d 100644 --- a/volatility3/framework/symbols/windows/callbacks-x64.json +++ b/volatility3/framework/symbols/windows/callbacks-x64.json @@ -105,8 +105,11 @@ }, "NotificationRoutine": { "type": { - "kind": "base", - "name": "unsigned int" + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } }, "offset": 24 } From d97fd777f37d58fff19df48334546c2e709f0d27 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 29 Jul 2024 18:36:43 -0500 Subject: [PATCH 147/263] Windows: Bumps netstat module version requirement This is a bump of the version number for the netstat plugin's `modules` requirement - it didn't get updated after #1173 was merged. --- volatility3/framework/plugins/windows/netstat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 24eb02018..0908767fc 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -35,7 +35,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): name="netscan", component=netscan.NetScan, version=(1, 0, 0) ), requirements.VersionRequirement( - name="modules", component=modules.Modules, version=(1, 0, 0) + name="modules", component=modules.Modules, version=(2, 0, 0) ), requirements.VersionRequirement( name="pdbutil", component=pdbutil.PDBUtility, version=(1, 0, 0) From 59fec176aa5377e7324c6996770d27a46fdebd9a Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 29 Jul 2024 18:49:48 -0500 Subject: [PATCH 148/263] Fix bugs in thrdscan and threads. Add orphan kernel threads plugin --- .../plugins/windows/orphan_kernel_threads.py | 95 +++++++++++++++++++ .../framework/plugins/windows/thrdscan.py | 2 +- .../framework/plugins/windows/threads.py | 12 +-- 3 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 volatility3/framework/plugins/windows/orphan_kernel_threads.py diff --git a/volatility3/framework/plugins/windows/orphan_kernel_threads.py b/volatility3/framework/plugins/windows/orphan_kernel_threads.py new file mode 100644 index 000000000..013ba98ff --- /dev/null +++ b/volatility3/framework/plugins/windows/orphan_kernel_threads.py @@ -0,0 +1,95 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List, Generator + +from volatility3.framework import interfaces, symbols +from volatility3.framework.configuration import requirements +from volatility3.plugins.windows import thrdscan, ssdt + +vollog = logging.getLogger(__name__) + + +class Threads(thrdscan.ThrdScan): + """Lists process threads""" + + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.implementation = self.list_orphan_kernel_threads + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="thrdscan", plugin=thrdscan.ThrdScan, version=(1, 1, 0) + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), + ] + + @classmethod + def list_orphan_kernel_threads( + cls, + context: interfaces.context.ContextInterface, + module_name: str, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Yields thread objects of kernel threads that do not map to a module + + Args: + kernel + + Returns: + A generator of thread objects of orphaned threads + """ + module = context.modules[module_name] + layer_name = module.layer_name + symbol_table = module.symbol_table_name + + collection = ssdt.SSDT.build_module_collection( + context, layer_name, symbol_table + ) + + # used to filter out smeared pointers + if symbols.symbol_table_is_64bit(context, symbol_table): + kernel_start = 0xFFFFF80000000000 + else: + kernel_start = 0x80000000 + + for thread in thrdscan.ThrdScan.scan_threads(context, module_name): + # we don't want smeared or terminated threads + try: + proc = thread.owning_process() + except AttributeError: + continue + + # we only care about kernel threads, 4 = System + # previous methods for determining if a thread was a kernel thread + # such as bit fields and flags are not stable in Win10+ + # so we check if the thread is from the kernel itself or one its child + # kernel processes (MemCompression, Regsitry, ...) + if proc.UniqueProcessId != 4 and proc.InheritedFromUniqueProcessId != 4: + continue + + if thread.StartAddress < kernel_start: + continue + + module_symbols = list( + collection.get_module_symbols_by_absolute_location(thread.StartAddress) + ) + + # alert on threads that do not map to a module + if not module_symbols: + yield thread + diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 6e664e052..ad885e8e9 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -22,8 +22,8 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) _version = (1, 1, 0) def __init__(self, *args, **kwargs): - self.implementation = self.scan_threads super().__init__(*args, **kwargs) + self.implementation = self.scan_threads @classmethod def get_requirements(cls): diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index ae70e717b..98a3169a5 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -18,9 +18,9 @@ class Threads(thrdscan.ThrdScan): _required_framework_version = (2, 4, 0) _version = (1, 0, 0) - def __init__(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.implementation = self.list_process_threads - super().__init__() @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -50,7 +50,6 @@ class Threads(thrdscan.ThrdScan): Args: proc: _EPROCESS object from which to list the VADs - filter_func: Function to take a virtual address descriptor value and return True if it should be filtered out Returns: A list of threads based on the process and filtered based on the filter function @@ -64,22 +63,19 @@ class Threads(thrdscan.ThrdScan): seen.add(thread.vol.offset) yield thread - @classmethod - def filter_func(cls, config: interfaces.configuration.HierarchicalDict) -> Callable: - return pslist.PsList.create_pid_filter(config.get("pid", None)) - @classmethod def list_process_threads( cls, context: interfaces.context.ContextInterface, module_name: str, - filter_func: Callable, ) -> Iterable[interfaces.objects.ObjectInterface]: """Runs through all processes and lists threads for each process""" module = context.modules[module_name] layer_name = module.layer_name symbol_table_name = module.symbol_table_name + filter_func = pslist.PsList.create_pid_filter(context.config.get("pid", None)) + for proc in pslist.PsList.list_processes( context=context, layer_name=layer_name, From 896b40bd22028f8e68d954a9f9080d564c6982e2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 29 Jul 2024 18:50:55 -0500 Subject: [PATCH 149/263] Fix bugs in thrdscan and threads. Add orphan kernel threads plugin --- volatility3/framework/plugins/windows/orphan_kernel_threads.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/orphan_kernel_threads.py b/volatility3/framework/plugins/windows/orphan_kernel_threads.py index 013ba98ff..1cdb1dcdf 100644 --- a/volatility3/framework/plugins/windows/orphan_kernel_threads.py +++ b/volatility3/framework/plugins/windows/orphan_kernel_threads.py @@ -92,4 +92,3 @@ class Threads(thrdscan.ThrdScan): # alert on threads that do not map to a module if not module_symbols: yield thread - From efc48d5831d00fa7db45e853d380165731736841 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 30 Jul 2024 17:18:14 +1000 Subject: [PATCH 150/263] Make timeliner able to sort aware datetimes. Otherwise, it will raise an exception when comparing the plugin output data with this naive datetime --- volatility3/framework/plugins/timeliner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 26a100f53..f657a2918 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -105,7 +105,9 @@ class Timeliner(interfaces.plugins.PluginInterface): data = item[1] def sortable(timestamp): - max_date = datetime.datetime(day=1, month=12, year=datetime.MAXYEAR) + max_date = datetime.datetime( + day=1, month=12, year=datetime.MAXYEAR, tzinfo=datetime.timezone.utc + ) if isinstance(timestamp, interfaces.renderers.BaseAbsentValue): return max_date return timestamp From 56b618f5c3345984c10af7e2868254e2d2bb7ea8 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 30 Jul 2024 13:00:38 -0500 Subject: [PATCH 151/263] Windows: Updates netscan with new symbol file `netscan` was missing coverage for Windows 10 Build 20348, causing owners and create times for `_TCP_ENDPOINTS` to be missing. This adds a symbol file and the necessary version check in the netscan plugin. Testing confirms that this returns the correct creation time and owner process. --- .../framework/plugins/windows/netscan.py | 1 + .../netscan/netscan-win10-20348-x64.json | 582 ++++++++++++++++++ 2 files changed, 583 insertions(+) create mode 100644 volatility3/framework/symbols/windows/netscan/netscan-win10-20348-x64.json diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 62ead3ab7..868bd8bcd 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -218,6 +218,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): (10, 0, 18362, 0): "netscan-win10-18362-x64", (10, 0, 18363, 0): "netscan-win10-18363-x64", (10, 0, 19041, 0): "netscan-win10-19041-x64", + (10, 0, 20348, 0): "netscan-win10-20348-x64", } # we do not need to check for tcpip's specific FileVersion in every case diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win10-20348-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-20348-x64.json new file mode 100644 index 000000000..bd574b2b7 --- /dev/null +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-20348-x64.json @@ -0,0 +1,582 @@ +{ + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned be short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "big" + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "symbols": {}, + "user_types": { + "_UDP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 88, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } + }, + "LocalAddr": { + "offset": 168, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS_WIN10_UDP" + } + } + }, + "InetAF": { + "offset": 32, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Port": { + "offset": 160, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 168 + }, + "_TCP_LISTENER": { + "fields": { + "Owner": { + "offset": 48, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 64, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 96, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + + } + }, + "InetAF": { + "offset": 40, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Next": { + "offset": 120, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } + }, + "Port": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 128 + }, + "_TCP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 752, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + }, + "CreateTime": { + "offset": 776, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "AddrInfo": { + "offset": 24, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ADDRINFO" + } + } + }, + "ListEntry": { + "offset": 40, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "InetAF": { + "offset": 16, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + } + }, + "LocalPort": { + "offset": 112, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "State": { + "offset": 108, + "type": { + "kind": "enum", + "name": "TCPStateEnum" + } + } + }, + "kind": "struct", + "size": 632 + }, + "_LOCAL_ADDRESS": { + "fields": { + "pData": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + } + }, + "kind": "struct", + "size": 20 + }, + "_LOCAL_ADDRESS_WIN10_UDP": { + "fields": { + "pData": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 + }, + "_ADDRINFO": { + "fields": { + "Local": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "Remote": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 + }, + "_IN_ADDR": { + "fields": { + "addr4": { + "offset": 0, + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + }, + "addr6": { + "offset": 0, + "type": { + "count": 16, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + } + }, + "kind": "struct", + "size": 6 + }, + "_INETAF": { + "fields": { + "AddressFamily": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 26 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 6144 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 224, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 208, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 192 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 + } + }, + "enums": { + "TCPStateEnum": { + "base": "long", + "constants": { + "CLOSED": 0, + "LISTENING": 1, + "SYN_SENT": 2, + "SYN_RCVD": 3, + "ESTABLISHED": 4, + "FIN_WAIT1": 5, + "FIN_WAIT2": 6, + "CLOSE_WAIT": 7, + "CLOSING": 8, + "LAST_ACK": 9, + "TIME_WAIT": 12, + "DELETE_TCB": 13 + }, + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona-by-hand", + "datetime": "2024-07-30T13:00:00" + }, + "format": "6.0.0" + } +} From a84a3706c5acee2c93df650a167ab058fe657053 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 30 Jul 2024 14:00:41 -0500 Subject: [PATCH 152/263] Remove errant filter on ldrmodule checks --- volatility3/framework/plugins/windows/ldrmodules.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index ffd84ea4a..a888f22e1 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -47,14 +47,6 @@ class LdrModules(interfaces.plugins.PluginInterface): self.context, self.config_path, "windows", "pe", class_types=pe.class_types ) - def filter_function(x: interfaces.objects.ObjectInterface) -> bool: - try: - return not (x.get_private_memory() == 0 and x.ControlArea) - except AttributeError: - return False - - filter_func = filter_function - for proc in procs: proc_layer_name = proc.add_process_layer() @@ -69,7 +61,7 @@ class LdrModules(interfaces.plugins.PluginInterface): # Build dictionary of mapped files, where the VAD start address is the key and value is the file name of the mapped file mapped_files = {} - for vad in vadinfo.VadInfo.list_vads(proc, filter_func=filter_func): + for vad in vadinfo.VadInfo.list_vads(proc): dos_header = self.context.object( pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", offset=vad.get_start(), From 73bc10c2834d9a8fa2ab04c098a4735542226170 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jul 2024 21:46:57 +0100 Subject: [PATCH 153/263] Wire the argcomplete into volshell too --- volatility3/cli/volshell/__init__.py | 12 ++++++++++++ volshell.py | 1 + 2 files changed, 13 insertions(+) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 035ed9b2e..2bf1958e2 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -21,6 +21,14 @@ from volatility3.framework import ( plugins, ) +try: + import argcomplete + + HAS_ARGCOMPLETE = True +except ImportError: + HAS_ARGCOMPLETE = False + + # Make sure we log everything rootlog = logging.getLogger() @@ -276,6 +284,10 @@ class VolShell(cli.CommandLine): # Hand the plugin requirements over to the CLI (us) and let it construct the config tree # Run the argparser + if HAS_ARGCOMPLETE: + # The autocompletion line must be after the partial_arg handling, so that it doesn't trip it + # before all the plugins have been added + argcomplete.autocomplete(parser) args = parser.parse_args() vollog.log( diff --git a/volshell.py b/volshell.py index 71d35a47c..65b11885e 100755 --- a/volshell.py +++ b/volshell.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# PYTHON_ARGCOMPLETE_OK # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 From d576f8cb48d064ce3dc87936df642481aa1f3186 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jul 2024 21:49:07 +0100 Subject: [PATCH 154/263] Fix CodeQL error --- volatility3/cli/volargparse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index 3e7eb6751..2bd53077b 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -99,6 +99,7 @@ class HelpfulArgParser(argparse.ArgumentParser): # return the number of arguments matched return len(match.group(1)) - def _check_value(self, action, value): + def _check_value(self, action: argparse.Action, value: Any) -> None: if not isinstance(action, HelpfulSubparserAction): return super()._check_value(action, value) + return None From fd6e4bec5cab84035491f629d234522ac6d5a8af Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Wed, 31 Jul 2024 17:53:29 -0500 Subject: [PATCH 155/263] Updated with feedback from the PR --- .../framework/plugins/windows/psxview.py | 172 ++++++++---------- 1 file changed, 74 insertions(+), 98 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 1fde5b12d..dc5e77ec3 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -1,4 +1,4 @@ -import datetime, logging +import datetime, logging, string from volatility3.framework import constants, exceptions from volatility3.framework.interfaces import plugins @@ -22,7 +22,7 @@ class PsXView(plugins.PluginInterface): plugin's output in a terminal.""" # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality - # which the original plugin to do it. + # which the original plugin used to do it. # I don't think it's worth including the sessions method either because both the original psxview plugin # and Volatility3's sessions plugin begin with the list of processes found by PsList. @@ -35,6 +35,10 @@ class PsXView(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) + valid_proc_name_chars = set( + string.ascii_lowercase + string.ascii_uppercase + "." + " " + ) + @classmethod def get_requirements(cls): return [ @@ -58,17 +62,6 @@ class PsXView(plugins.PluginInterface): requirements.VersionRequirement( name="handles", component=handles.Handles, version=(1, 0, 0) ), - requirements.VersionRequirement( - name="sessions", component=sessions.Sessions, version=(0, 0, 0) - ), - requirements.BooleanRequirement( - name="identify-expected", - description='In the plugin\'s output, replace false with \ - normal where false is the expected result for a Windows machine running normally. \ - Keep in mind that this plugin uses simple checks to identify "normal" behavior, \ - so you may want to double-check the legitimacy of these processes yourself.', - optional=True, - ), requirements.BooleanRequirement( name="physical-offsets", description="List processes with phyiscall offsets instead of virtual offsets.", @@ -76,54 +69,56 @@ class PsXView(plugins.PluginInterface): ), ] - def proc_name_to_string(self, proc): + def _proc_name_to_string(self, proc): return proc.ImageFileName.cast( "string", max_length=proc.ImageFileName.vol.count, errors="replace" ) - def is_ascii(self, str): - return str.split(".")[0].isalnum() + def _is_valid_proc_name(self, str): + for c in str: + if not c in self.valid_proc_name_chars: + return False + return True - def filter_garbage_procs(self, proc_list): + def _filter_garbage_procs(self, proc_list): return [ p for p in proc_list - if p.is_valid() and self.is_ascii(self.proc_name_to_string(p)) + if p.is_valid() and self._is_valid_proc_name(self._proc_name_to_string(p)) ] - def translate_offset(self, offset): - if self.config["physical-offsets"]: + def _translate_offset(self, offset): + if not self.config["physical-offsets"]: return offset kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name try: - offset = list( + _, _, offset, _, _ = list( self.context.layers[layer_name].mapping(offset=offset, length=0) - )[0][2] + )[0] except: # already have physical address pass return offset - def proc_list_to_dict(self, tasks): - return {self.translate_offset(proc.vol.offset): proc for proc in tasks} + def _proc_list_to_dict(self, tasks): + tasks = self._filter_garbage_procs(tasks) + return {self._translate_offset(proc.vol.offset): proc for proc in tasks} - def check_pslist(self, tasks): - res = self.filter_garbage_procs(tasks) - return self.proc_list_to_dict(tasks) + def _check_pslist(self, tasks): + return self._proc_list_to_dict(tasks) - def check_psscan(self, layer_name, symbol_table): + def _check_psscan(self, layer_name, symbol_table): res = psscan.PsScan.scan_processes( context=self.context, layer_name=layer_name, symbol_table=symbol_table ) - res = self.filter_garbage_procs(res) - return self.proc_list_to_dict(res) + return self._proc_list_to_dict(res) - def check_thrdscan(self): + def _check_thrdscan(self): ret = [] for ethread in thrdscan.ThrdScan.scan_threads( @@ -142,13 +137,13 @@ class PsXView(plugins.PluginInterface): "Unable to find the owning process of ethread", ) - return self.proc_list_to_dict(ret) + return self._proc_list_to_dict(ret) - def check_csrss_handles(self, tasks, layer_name, symbol_table): + def _check_csrss_handles(self, tasks, layer_name, symbol_table): ret = [] for p in tasks: - name = self.proc_name_to_string(p) + name = self._proc_name_to_string(p) if name == "csrss.exe": try: if p.has_member("ObjectTable"): @@ -172,13 +167,7 @@ class PsXView(plugins.PluginInterface): constants.LOGLEVEL_VVV, "Cannot access eprocess object table" ) - ret = self.filter_garbage_procs(ret) - return self.proc_list_to_dict(ret) - - def check_session(self, pslist_procs): - procs = [p for p in pslist_procs if p.get_session_id() != None] - - return self.proc_list_to_dict(procs) + return self._proc_list_to_dict(ret) def _generator(self): kernel = self.context.modules[self.config["kernel"]] @@ -192,72 +181,60 @@ class PsXView(plugins.PluginInterface): ) ) + # get processes from each source processes = {} - processes["pslist"] = self.check_pslist(kdbg_list_processes) - processes["psscan"] = self.check_psscan(layer_name, symbol_table) - processes["thrdscan"] = self.check_thrdscan() - processes["csrss"] = self.check_csrss_handles( + processes["pslist"] = self._check_pslist(kdbg_list_processes) + processes["psscan"] = self._check_psscan(layer_name, symbol_table) + processes["thrdscan"] = self._check_thrdscan() + processes["csrss"] = self._check_csrss_handles( kdbg_list_processes, layer_name, symbol_table ) - processes["sessions"] = self.check_session(kdbg_list_processes) - seen_offsets = set() - for source in processes: - for offset in processes[source]: - if offset not in seen_offsets: - seen_offsets.add(offset) - proc = processes[source][offset] + # print results - pid = proc.UniqueProcessId - name = self.proc_name_to_string(proc) + # list of lists of offsets + todo_offsets = [list(processes[source].keys()) for source in processes] - exit_time = proc.get_exit_time() - if type(exit_time) != datetime.datetime: - exit_time = "" - else: - exit_time = str(exit_time) + # flatten to one list + todo_offsets = sum(todo_offsets, []) - in_sources = { - src: str(offset in processes[src]) for src in processes - } + # remove duplicates + todo_offsets = set(todo_offsets) - if self.config["identify-expected"]: - f = "False" - n = "Normal" + for offset in todo_offsets: + proc = None - if in_sources["pslist"] == f: - if exit_time != "": - in_sources["pslist"] = n + in_sources = {src: False for src in processes} - if in_sources["thrdscan"] == f: - if exit_time != "": - in_sources["thrdscan"] = n + for source in processes: + if offset in processes[source]: + in_sources[source] = True + if not proc: + proc = processes[source][offset] - if in_sources["csrss"] == f: - if name.lower() in ["system", "smss.exe", "csrss.exe"]: - in_sources["csrss"] = n - elif exit_time != "": - in_sources["csrss"] = n + pid = proc.UniqueProcessId + name = self._proc_name_to_string(proc) - if in_sources["sessions"] == f: - if name.lower() in ["system", "smss.exe"]: - in_sources["sessions"] = n + exit_time = proc.get_exit_time() + if type(exit_time) != datetime.datetime: + exit_time = "" + else: + exit_time = str(exit_time) - yield ( - 0, - ( - format_hints.Hex(offset), - name, - pid, - in_sources["pslist"], - in_sources["psscan"], - in_sources["thrdscan"], - in_sources["csrss"], - in_sources["sessions"], - exit_time, - ), - ) + yield ( + 0, + ( + format_hints.Hex(offset), + name, + pid, + in_sources["pslist"], + in_sources["psscan"], + in_sources["thrdscan"], + in_sources["csrss"], + exit_time, + ), + ) def run(self): offset_type = "(Physical)" if self.config["physical-offsets"] else "(Virtual)" @@ -268,11 +245,10 @@ class PsXView(plugins.PluginInterface): (offset_str, format_hints.Hex), ("Name", str), ("PID", int), - ("pslist", str), - ("psscan", str), - ("thrdscan", str), - ("csrss", str), - ("sessions", str), + ("pslist", bool), + ("psscan", bool), + ("thrdscan", bool), + ("csrss", bool), ("Exit Time", str), ], self._generator(), From 9e8864521c3fb06e80536a8d0f249ac19fd9a9c0 Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:02:52 -0500 Subject: [PATCH 156/263] Added debug log for failed address translation --- volatility3/framework/plugins/windows/psxview.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index dc5e77ec3..918eb44ba 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -98,9 +98,8 @@ class PsXView(plugins.PluginInterface): _, _, offset, _, _ = list( self.context.layers[layer_name].mapping(offset=offset, length=0) )[0] - except: - # already have physical address - pass + except exceptions.PagedInvalidAddressException: + vollog.debug(f"Page fault: unable to translate {offset:0x}") return offset From 90b8b5340f9bc886e9b45ecadd80b7d86b8384f3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 1 Aug 2024 11:46:10 +1000 Subject: [PATCH 157/263] Linux: Add inode, timespec, and timespec64 object extensions to support different kernel versions, ensuring we will get aware datetimes when using them. --- .../framework/symbols/linux/__init__.py | 6 + .../symbols/linux/extensions/__init__.py | 132 ++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index c4e2587f4..03353135d 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -29,12 +29,18 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) + self.set_type_class("inode", extensions.inode) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) + # kernels >= 4.18 + self.optional_set_type_class("timespec64", extensions.timespec64) + # kernels < 4.18. Reuses timespec64 obj extension, since both has the same members + self.optional_set_type_class("timespec", extensions.timespec64) + # Mount self.set_type_class("vfsmount", extensions.vfsmount) # Might not exist in older kernels or the current symbols diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e7c6b66d7..be31e298c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -4,10 +4,13 @@ import collections.abc import logging +import stat +from datetime import datetime import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List from volatility3.framework import constants, exceptions, objects, interfaces, symbols +from volatility3.framework import renderers from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1761,3 +1764,132 @@ class kernel_cap_t(kernel_cap_struct): ) return cap_value & self.get_kernel_cap_full() + + +class timespec64(objects.StructType): + def to_datetime(self) -> datetime: + """Returns the respective aware datetime""" + + dt = renderers.conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) + return dt + + +class inode(objects.StructType): + def is_valid(self) -> bool: + # i_count is a 'signed' counter (atomic_t). Smear, or essentially a wrong inode + # pointer, will easily cause an integer overflow here. + return self.i_ino > 0 and self.i_count.counter >= 0 + + def is_dir(self) -> bool: + """Returns True if the inode is a directory""" + return stat.S_ISDIR(self.i_mode) != 0 + + def is_reg(self) -> bool: + """Returns True if the inode is a regular file""" + return stat.S_ISREG(self.i_mode) != 0 + + def is_link(self) -> bool: + """Returns True if the inode is a symlink""" + return stat.S_ISLNK(self.i_mode) != 0 + + def is_fifo(self) -> bool: + """Returns True if the inode is a FIFO""" + return stat.S_ISFIFO(self.i_mode) != 0 + + def is_sock(self) -> bool: + """Returns True if the inode is a socket""" + return stat.S_ISSOCK(self.i_mode) != 0 + + def is_block(self) -> bool: + """Returns True if the inode is a block device""" + return stat.S_ISBLK(self.i_mode) != 0 + + def is_char(self) -> bool: + """Returns True if the inode is a char device""" + return stat.S_ISCHR(self.i_mode) != 0 + + def is_sticky(self) -> bool: + """Returns True if the sticky bit is set""" + return (self.i_mode & stat.S_ISVTX) != 0 + + def get_inode_type(self) -> str: + """Returns inode type name + + Returns: + The inode type name + """ + if self.is_dir(): + return "DIR" + elif self.is_reg(): + return "REG" + elif self.is_link(): + return "LNK" + elif self.is_fifo(): + return "FIFO" + elif self.is_sock(): + return "SOCK" + elif self.is_char(): + return "CHR" + elif self.is_block(): + return "BLK" + else: + return renderers.UnparsableValue() + + def get_inode_number(self) -> int: + """Returns the inode number""" + return int(self.i_ino) + + def ___time_member_to_datetime(self, member) -> datetime: + if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): + # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 + # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 + return renderers.conversion.unixtime_to_datetime( + self.member(f"{member}_sec") + self.has_member(f"{member}_nsec") / 1e9 + ) + elif self.has_member(f"__{member}"): + # 6.6 <= kernels < 6.11 it's a timespec64 + # Ref Linux commit 13bc24457850583a2e7203ded05b7209ab4bc5ef / 12cd44023651666bd44baa36a5c999698890debb + return self.member(f"__{member}").to_datetime() + elif self.has_member(member): + # In kernels < 6.6 it's a timespec64 or timespec + return self.member(member).to_datetime() + else: + raise exceptions.VolatilityException( + "Unsupported kernel inode type implementation" + ) + + def get_access_time(self) -> datetime: + """Returns the inode's last access time + This is updated when inode contents are read + + Returns: + A datetime with the inode's last access time + """ + return self.___time_member_to_datetime("i_atime") + + def get_modification_time(self) -> datetime: + """Returns the inode's last modification time + This is updated when the inode contents change + + Returns: + A datetime with the inode's last data modification time + """ + + return self.___time_member_to_datetime("i_mtime") + + def get_change_time(self) -> datetime: + """Returns the inode's last change time + This is updated when the inode metadata changes + + Returns: + A datetime with the inode's last change time + """ + return self.___time_member_to_datetime("i_ctime") + + def get_file_mode(self) -> str: + """Returns the inode's file mode as string of the form '-rwxrwxrwx'. + + Returns: + The inode's file mode string + """ + return stat.filemode(self.i_mode) From 3017a7d00c3a9cfca3f94b9e7dfb8ba08d48fd0e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 1 Aug 2024 11:46:10 +1000 Subject: [PATCH 158/263] Linux: Add inode, timespec, and timespec64 object extensions to support different kernel versions, ensuring we will get aware datetimes when using them. --- .../framework/symbols/linux/__init__.py | 6 + .../symbols/linux/extensions/__init__.py | 132 ++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index c4e2587f4..03353135d 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -29,12 +29,18 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) + self.set_type_class("inode", extensions.inode) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) + # kernels >= 4.18 + self.optional_set_type_class("timespec64", extensions.timespec64) + # kernels < 4.18. Reuses timespec64 obj extension, since both has the same members + self.optional_set_type_class("timespec", extensions.timespec64) + # Mount self.set_type_class("vfsmount", extensions.vfsmount) # Might not exist in older kernels or the current symbols diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e7c6b66d7..be31e298c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -4,10 +4,13 @@ import collections.abc import logging +import stat +from datetime import datetime import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List from volatility3.framework import constants, exceptions, objects, interfaces, symbols +from volatility3.framework import renderers from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1761,3 +1764,132 @@ class kernel_cap_t(kernel_cap_struct): ) return cap_value & self.get_kernel_cap_full() + + +class timespec64(objects.StructType): + def to_datetime(self) -> datetime: + """Returns the respective aware datetime""" + + dt = renderers.conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) + return dt + + +class inode(objects.StructType): + def is_valid(self) -> bool: + # i_count is a 'signed' counter (atomic_t). Smear, or essentially a wrong inode + # pointer, will easily cause an integer overflow here. + return self.i_ino > 0 and self.i_count.counter >= 0 + + def is_dir(self) -> bool: + """Returns True if the inode is a directory""" + return stat.S_ISDIR(self.i_mode) != 0 + + def is_reg(self) -> bool: + """Returns True if the inode is a regular file""" + return stat.S_ISREG(self.i_mode) != 0 + + def is_link(self) -> bool: + """Returns True if the inode is a symlink""" + return stat.S_ISLNK(self.i_mode) != 0 + + def is_fifo(self) -> bool: + """Returns True if the inode is a FIFO""" + return stat.S_ISFIFO(self.i_mode) != 0 + + def is_sock(self) -> bool: + """Returns True if the inode is a socket""" + return stat.S_ISSOCK(self.i_mode) != 0 + + def is_block(self) -> bool: + """Returns True if the inode is a block device""" + return stat.S_ISBLK(self.i_mode) != 0 + + def is_char(self) -> bool: + """Returns True if the inode is a char device""" + return stat.S_ISCHR(self.i_mode) != 0 + + def is_sticky(self) -> bool: + """Returns True if the sticky bit is set""" + return (self.i_mode & stat.S_ISVTX) != 0 + + def get_inode_type(self) -> str: + """Returns inode type name + + Returns: + The inode type name + """ + if self.is_dir(): + return "DIR" + elif self.is_reg(): + return "REG" + elif self.is_link(): + return "LNK" + elif self.is_fifo(): + return "FIFO" + elif self.is_sock(): + return "SOCK" + elif self.is_char(): + return "CHR" + elif self.is_block(): + return "BLK" + else: + return renderers.UnparsableValue() + + def get_inode_number(self) -> int: + """Returns the inode number""" + return int(self.i_ino) + + def ___time_member_to_datetime(self, member) -> datetime: + if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): + # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 + # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 + return renderers.conversion.unixtime_to_datetime( + self.member(f"{member}_sec") + self.has_member(f"{member}_nsec") / 1e9 + ) + elif self.has_member(f"__{member}"): + # 6.6 <= kernels < 6.11 it's a timespec64 + # Ref Linux commit 13bc24457850583a2e7203ded05b7209ab4bc5ef / 12cd44023651666bd44baa36a5c999698890debb + return self.member(f"__{member}").to_datetime() + elif self.has_member(member): + # In kernels < 6.6 it's a timespec64 or timespec + return self.member(member).to_datetime() + else: + raise exceptions.VolatilityException( + "Unsupported kernel inode type implementation" + ) + + def get_access_time(self) -> datetime: + """Returns the inode's last access time + This is updated when inode contents are read + + Returns: + A datetime with the inode's last access time + """ + return self.___time_member_to_datetime("i_atime") + + def get_modification_time(self) -> datetime: + """Returns the inode's last modification time + This is updated when the inode contents change + + Returns: + A datetime with the inode's last data modification time + """ + + return self.___time_member_to_datetime("i_mtime") + + def get_change_time(self) -> datetime: + """Returns the inode's last change time + This is updated when the inode metadata changes + + Returns: + A datetime with the inode's last change time + """ + return self.___time_member_to_datetime("i_ctime") + + def get_file_mode(self) -> str: + """Returns the inode's file mode as string of the form '-rwxrwxrwx'. + + Returns: + The inode's file mode string + """ + return stat.filemode(self.i_mode) From e6308a6035156cab5abb6f0cdf537fb75e5881e8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 1 Aug 2024 21:04:24 +0100 Subject: [PATCH 159/263] Make suggested changes by gcmoreira --- volatility3/cli/volargparse.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index 2bd53077b..5ce2646ed 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -100,6 +100,11 @@ class HelpfulArgParser(argparse.ArgumentParser): return len(match.group(1)) def _check_value(self, action: argparse.Action, value: Any) -> None: + """This is called to ensure a value is correct/valid + This fails when we want to accept partial values for the plugin name, + so we disable the check (which will throw ArgumentErrors for failed checks) + but only for our plugin subparser, so all other arguments are checked correctly + """ if not isinstance(action, HelpfulSubparserAction): - return super()._check_value(action, value) + super()._check_value(action, value) return None From 0bda1543854d8f92e7fb2bf4c5eff3afdf117819 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 1 Aug 2024 21:08:18 +0100 Subject: [PATCH 160/263] Clarify the documentation a little --- volatility3/cli/volargparse.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index 5ce2646ed..fd61ddce0 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -101,9 +101,16 @@ class HelpfulArgParser(argparse.ArgumentParser): def _check_value(self, action: argparse.Action, value: Any) -> None: """This is called to ensure a value is correct/valid - This fails when we want to accept partial values for the plugin name, - so we disable the check (which will throw ArgumentErrors for failed checks) - but only for our plugin subparser, so all other arguments are checked correctly + + In normal operation, it would check that a value provided is valid and return None + If it was not valid, it would throw an ArgumentError + + When people provide a partial plugin name, we want to look for a matching plugin name + which happens in the HelpfulSubparserAction's __call_method + + To get there without tripping the check_value failure, we have to prevent the exception + being thrown when the value is a HelpfulSubparserAction. This therefore affects no other + checks for normal parameters. """ if not isinstance(action, HelpfulSubparserAction): super()._check_value(action, value) From d19013c85261ea48dd774dcda66dcb8d1b36782d Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:19:24 -0500 Subject: [PATCH 161/263] fixed typo, updated plugin docstring, and updated comment --- volatility3/framework/plugins/windows/psxview.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 918eb44ba..cab27f3e3 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -17,17 +17,14 @@ vollog = logging.getLogger(__name__) class PsXView(plugins.PluginInterface): - """Lists all processes found via 6 of the methods described in \"The Art of Memory Forensics,\" which may help + """Lists all processes found via four of the methods described in \"The Art of Memory Forensics,\" which may help identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this plugin's output in a terminal.""" # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality # which the original plugin used to do it. - # I don't think it's worth including the sessions method either because both the original psxview plugin - # and Volatility3's sessions plugin begin with the list of processes found by PsList. - # The original psxview plugin's session code essentially just filters the pslist for processes - # whose session ID is not None. I've matched this in my code, but again, it doesn't seem worth including. + # The sessions method is omitted because it begins with the list of processes found by Pslist anyway. # Lastly, I've omitted the pspcid method because I could not for the life of me get it to work. I saved the # code I do have from it, and will happily share it if anyone else wants to add it. @@ -64,7 +61,7 @@ class PsXView(plugins.PluginInterface): ), requirements.BooleanRequirement( name="physical-offsets", - description="List processes with phyiscall offsets instead of virtual offsets.", + description="List processes with physical offsets instead of virtual offsets.", optional=True, ), ] From 98c0094da5cfb8a99c9e211004952c9104c619cd Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Thu, 1 Aug 2024 18:51:11 -0500 Subject: [PATCH 162/263] Updated unpacked variable names --- volatility3/framework/plugins/windows/psxview.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index cab27f3e3..71919c410 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -92,7 +92,7 @@ class PsXView(plugins.PluginInterface): layer_name = kernel.layer_name try: - _, _, offset, _, _ = list( + _original_offset, _original_length, offset, _length, _layer_name = list( self.context.layers[layer_name].mapping(offset=offset, length=0) )[0] except exceptions.PagedInvalidAddressException: @@ -190,15 +190,15 @@ class PsXView(plugins.PluginInterface): # print results # list of lists of offsets - todo_offsets = [list(processes[source].keys()) for source in processes] + offsets = [list(processes[source].keys()) for source in processes] # flatten to one list - todo_offsets = sum(todo_offsets, []) + offsets = sum(offsets, []) # remove duplicates - todo_offsets = set(todo_offsets) + offsets = set(offsets) - for offset in todo_offsets: + for offset in offsets: proc = None in_sources = {src: False for src in processes} From 1dcaf9c0b15dfd00f2b35846488f19f3982ae5b2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 13:50:51 +1000 Subject: [PATCH 163/263] PR review fixes: Rename method name from private to internal --- .../framework/symbols/linux/extensions/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index be31e298c..599fedb6f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1839,7 +1839,7 @@ class inode(objects.StructType): """Returns the inode number""" return int(self.i_ino) - def ___time_member_to_datetime(self, member) -> datetime: + def _time_member_to_datetime(self, member) -> datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 @@ -1865,7 +1865,7 @@ class inode(objects.StructType): Returns: A datetime with the inode's last access time """ - return self.___time_member_to_datetime("i_atime") + return self._time_member_to_datetime("i_atime") def get_modification_time(self) -> datetime: """Returns the inode's last modification time @@ -1875,7 +1875,7 @@ class inode(objects.StructType): A datetime with the inode's last data modification time """ - return self.___time_member_to_datetime("i_mtime") + return self._time_member_to_datetime("i_mtime") def get_change_time(self) -> datetime: """Returns the inode's last change time @@ -1884,7 +1884,7 @@ class inode(objects.StructType): Returns: A datetime with the inode's last change time """ - return self.___time_member_to_datetime("i_ctime") + return self._time_member_to_datetime("i_ctime") def get_file_mode(self) -> str: """Returns the inode's file mode as string of the form '-rwxrwxrwx'. From 79529fb8153b3a58b4a1d1c6192cb92cf107800f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 13:50:51 +1000 Subject: [PATCH 164/263] PR review fixes: Rename method name from private to internal --- .../framework/symbols/linux/extensions/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index be31e298c..599fedb6f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1839,7 +1839,7 @@ class inode(objects.StructType): """Returns the inode number""" return int(self.i_ino) - def ___time_member_to_datetime(self, member) -> datetime: + def _time_member_to_datetime(self, member) -> datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 @@ -1865,7 +1865,7 @@ class inode(objects.StructType): Returns: A datetime with the inode's last access time """ - return self.___time_member_to_datetime("i_atime") + return self._time_member_to_datetime("i_atime") def get_modification_time(self) -> datetime: """Returns the inode's last modification time @@ -1875,7 +1875,7 @@ class inode(objects.StructType): A datetime with the inode's last data modification time """ - return self.___time_member_to_datetime("i_mtime") + return self._time_member_to_datetime("i_mtime") def get_change_time(self) -> datetime: """Returns the inode's last change time @@ -1884,7 +1884,7 @@ class inode(objects.StructType): Returns: A datetime with the inode's last change time """ - return self.___time_member_to_datetime("i_ctime") + return self._time_member_to_datetime("i_ctime") def get_file_mode(self) -> str: """Returns the inode's file mode as string of the form '-rwxrwxrwx'. From 930d29046ad8c1cd47d6167e84118d78a3e549a6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 14:15:01 +1000 Subject: [PATCH 165/263] PR review fixes: Avoid using renderers in core functions. --- .../framework/symbols/linux/extensions/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 599fedb6f..1b5e1d286 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,10 +7,10 @@ import logging import stat from datetime import datetime import socket as socket_module -from typing import Generator, Iterable, Iterator, Optional, Tuple, List +from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union from volatility3.framework import constants, exceptions, objects, interfaces, symbols -from volatility3.framework import renderers +from volatility3.framework.renderers import conversion from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1770,7 +1770,7 @@ class timespec64(objects.StructType): def to_datetime(self) -> datetime: """Returns the respective aware datetime""" - dt = renderers.conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) + dt = conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) return dt @@ -1812,7 +1812,7 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 - def get_inode_type(self) -> str: + def get_inode_type(self) -> Union[str, None]: """Returns inode type name Returns: @@ -1833,7 +1833,7 @@ class inode(objects.StructType): elif self.is_block(): return "BLK" else: - return renderers.UnparsableValue() + return None def get_inode_number(self) -> int: """Returns the inode number""" @@ -1843,7 +1843,7 @@ class inode(objects.StructType): if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 - return renderers.conversion.unixtime_to_datetime( + return conversion.unixtime_to_datetime( self.member(f"{member}_sec") + self.has_member(f"{member}_nsec") / 1e9 ) elif self.has_member(f"__{member}"): From efba3a1b7336d5b31f7ac5ee1d8e99d95bcd74f6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 14:17:57 +1000 Subject: [PATCH 166/263] PR review fixes: Convert inode's is_* functions to properties --- .../symbols/linux/extensions/__init__.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1b5e1d286..00f6730eb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1780,34 +1780,42 @@ class inode(objects.StructType): # pointer, will easily cause an integer overflow here. return self.i_ino > 0 and self.i_count.counter >= 0 + @property def is_dir(self) -> bool: """Returns True if the inode is a directory""" return stat.S_ISDIR(self.i_mode) != 0 + @property def is_reg(self) -> bool: """Returns True if the inode is a regular file""" return stat.S_ISREG(self.i_mode) != 0 + @property def is_link(self) -> bool: """Returns True if the inode is a symlink""" return stat.S_ISLNK(self.i_mode) != 0 + @property def is_fifo(self) -> bool: """Returns True if the inode is a FIFO""" return stat.S_ISFIFO(self.i_mode) != 0 + @property def is_sock(self) -> bool: """Returns True if the inode is a socket""" return stat.S_ISSOCK(self.i_mode) != 0 + @property def is_block(self) -> bool: """Returns True if the inode is a block device""" return stat.S_ISBLK(self.i_mode) != 0 + @property def is_char(self) -> bool: """Returns True if the inode is a char device""" return stat.S_ISCHR(self.i_mode) != 0 + @property def is_sticky(self) -> bool: """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 @@ -1818,19 +1826,19 @@ class inode(objects.StructType): Returns: The inode type name """ - if self.is_dir(): + if self.is_dir: return "DIR" - elif self.is_reg(): + elif self.is_reg: return "REG" - elif self.is_link(): + elif self.is_link: return "LNK" - elif self.is_fifo(): + elif self.is_fifo: return "FIFO" - elif self.is_sock(): + elif self.is_sock: return "SOCK" - elif self.is_char(): + elif self.is_char: return "CHR" - elif self.is_block(): + elif self.is_block: return "BLK" else: return None From b8d68b9a5ee0c643b68eaa9f33bd051279374eb3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 14:15:01 +1000 Subject: [PATCH 167/263] PR review fixes: Avoid using renderers in core functions. --- .../framework/symbols/linux/extensions/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 599fedb6f..1b5e1d286 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,10 +7,10 @@ import logging import stat from datetime import datetime import socket as socket_module -from typing import Generator, Iterable, Iterator, Optional, Tuple, List +from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union from volatility3.framework import constants, exceptions, objects, interfaces, symbols -from volatility3.framework import renderers +from volatility3.framework.renderers import conversion from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1770,7 +1770,7 @@ class timespec64(objects.StructType): def to_datetime(self) -> datetime: """Returns the respective aware datetime""" - dt = renderers.conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) + dt = conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) return dt @@ -1812,7 +1812,7 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 - def get_inode_type(self) -> str: + def get_inode_type(self) -> Union[str, None]: """Returns inode type name Returns: @@ -1833,7 +1833,7 @@ class inode(objects.StructType): elif self.is_block(): return "BLK" else: - return renderers.UnparsableValue() + return None def get_inode_number(self) -> int: """Returns the inode number""" @@ -1843,7 +1843,7 @@ class inode(objects.StructType): if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 - return renderers.conversion.unixtime_to_datetime( + return conversion.unixtime_to_datetime( self.member(f"{member}_sec") + self.has_member(f"{member}_nsec") / 1e9 ) elif self.has_member(f"__{member}"): From 933a41fa3a60f3f02185b530a1a710b6dcae895c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 14:17:57 +1000 Subject: [PATCH 168/263] PR review fixes: Convert inode's is_* functions to properties --- .../symbols/linux/extensions/__init__.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1b5e1d286..00f6730eb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1780,34 +1780,42 @@ class inode(objects.StructType): # pointer, will easily cause an integer overflow here. return self.i_ino > 0 and self.i_count.counter >= 0 + @property def is_dir(self) -> bool: """Returns True if the inode is a directory""" return stat.S_ISDIR(self.i_mode) != 0 + @property def is_reg(self) -> bool: """Returns True if the inode is a regular file""" return stat.S_ISREG(self.i_mode) != 0 + @property def is_link(self) -> bool: """Returns True if the inode is a symlink""" return stat.S_ISLNK(self.i_mode) != 0 + @property def is_fifo(self) -> bool: """Returns True if the inode is a FIFO""" return stat.S_ISFIFO(self.i_mode) != 0 + @property def is_sock(self) -> bool: """Returns True if the inode is a socket""" return stat.S_ISSOCK(self.i_mode) != 0 + @property def is_block(self) -> bool: """Returns True if the inode is a block device""" return stat.S_ISBLK(self.i_mode) != 0 + @property def is_char(self) -> bool: """Returns True if the inode is a char device""" return stat.S_ISCHR(self.i_mode) != 0 + @property def is_sticky(self) -> bool: """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 @@ -1818,19 +1826,19 @@ class inode(objects.StructType): Returns: The inode type name """ - if self.is_dir(): + if self.is_dir: return "DIR" - elif self.is_reg(): + elif self.is_reg: return "REG" - elif self.is_link(): + elif self.is_link: return "LNK" - elif self.is_fifo(): + elif self.is_fifo: return "FIFO" - elif self.is_sock(): + elif self.is_sock: return "SOCK" - elif self.is_char(): + elif self.is_char: return "CHR" - elif self.is_block(): + elif self.is_block: return "BLK" else: return None From d34b1ded3e673e2d311d14c8e104668d7ebac78c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 19:46:23 +1000 Subject: [PATCH 169/263] PR review fixes: Remove get_inode_number. It's better to use the type's original member name and handle the casting on the consumer side. --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 00f6730eb..05679523f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1843,10 +1843,6 @@ class inode(objects.StructType): else: return None - def get_inode_number(self) -> int: - """Returns the inode number""" - return int(self.i_ino) - def _time_member_to_datetime(self, member) -> datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 From ed208347630428e21dec91f90eb431ed02595bc3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 19:46:23 +1000 Subject: [PATCH 170/263] PR review fixes: Remove get_inode_number. It's better to use the type's original member name and handle the casting on the consumer side. --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 00f6730eb..05679523f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1843,10 +1843,6 @@ class inode(objects.StructType): else: return None - def get_inode_number(self) -> int: - """Returns the inode number""" - return int(self.i_ino) - def _time_member_to_datetime(self, member) -> datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 From 230ea09728dc9b756e27936544f03d43647cc0ba Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Aug 2024 17:52:51 +0200 Subject: [PATCH 171/263] Updating code following #1230 merge --- volatility3/framework/plugins/linux/lsof.py | 26 ++++++++--------- .../framework/symbols/linux/__init__.py | 28 ++++++++----------- .../symbols/linux/extensions/__init__.py | 14 ++++++++++ 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index c1de48c1a..fa9d2bf61 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -21,7 +21,6 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) @classmethod @@ -53,7 +52,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): symbol_table: str, filter_func: Callable[[int], bool] = lambda _: False, ): - linuxutils_symbol_table = None # type: ignore + linuxutils_symbol_table = None for task in pslist.PsList.list_tasks(context, symbol_table, filter_func): if linuxutils_symbol_table is None: if constants.BANG not in task.vol.type_name: @@ -71,21 +70,17 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): fd_num, filp, full_path = fd_fields inode_metadata = linux.LinuxUtilities.get_inode_metadata(context, filp) try: - inode_num, file_size, imode, ctime, mtime, atime = next( + inode_num, itype, file_size, imode, ctime, mtime, atime = next( inode_metadata ) except Exception as e: vollog.warning( f"Can't get inode metadata for file descriptor {fd_num}: {e}" ) - # Yield NotAvailableValue for each field in case of an exception - inode_num = renderers.NotAvailableValue() - file_size = renderers.NotAvailableValue() - imode = renderers.NotAvailableValue() - ctime = renderers.NotAvailableValue() - mtime = renderers.NotAvailableValue() - atime = renderers.NotAvailableValue() - yield pid, task_comm, task, fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size + inode_num = itype = file_size = imode = ctime = mtime = atime = ( + renderers.NotAvailableValue() + ) + yield pid, task_comm, task, fd_num, filp, full_path, inode_num, itype, imode, ctime, mtime, atime, file_size def _generator(self, pids, symbol_table): filter_func = pslist.PsList.create_pid_filter(pids) @@ -100,6 +95,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): filp, full_path, inode_num, + itype, imode, ctime, mtime, @@ -112,6 +108,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): fd_num, full_path, inode_num, + itype, imode, ctime, mtime, @@ -130,6 +127,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ("FD", int), ("Path", str), ("Inode", int), + ("Type", str), ("Mode", str), ("Changed", datetime.datetime), ("Modified", datetime.datetime), @@ -144,6 +142,6 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): for row in self._generator(pids, symbol_table): _depth, row_data = row description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' - yield description, timeliner.TimeLinerType.CHANGED, row_data[6] - yield description, timeliner.TimeLinerType.MODIFIED, row_data[7] - yield description, timeliner.TimeLinerType.ACCESSED, row_data[8] + yield description, timeliner.TimeLinerType.CHANGED, row_data[7] + yield description, timeliner.TimeLinerType.MODIFIED, row_data[8] + yield description, timeliner.TimeLinerType.ACCESSED, row_data[9] diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index a96fe9d2f..d52c43dae 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -280,23 +280,19 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): A helper function that gets the inodes metadata from a file descriptor """ dentry = filp.get_dentry() - if dentry != 0: + if dentry: inode_object = dentry.d_inode - inode_num = inode_object.i_ino - file_size = inode_object.i_size # file size in bytes - imode = stat.filemode(inode_object.i_mode) # file type & Permissions - - # Timestamps - ctime = datetime.datetime.fromtimestamp( - inode_object.i_ctime.tv_sec - ) # last change time - mtime = datetime.datetime.fromtimestamp( - inode_object.i_mtime.tv_sec - ) # last modify time - atime = datetime.datetime.fromtimestamp( - inode_object.i_atime.tv_sec - ) # last access time - yield inode_num, file_size, imode, ctime, mtime, atime + if inode_object and inode_object.is_valid(): + itype = inode_object.get_inode_type() or "?" + yield ( + inode_object.i_ino, + itype, + inode_object.i_size, + inode_object.get_file_mode(), + inode_object.get_change_time(), + inode_object.get_modification_time(), + inode_object.get_access_time(), + ) @classmethod def mask_mods_list( diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 05679523f..0ee6e7d95 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1820,6 +1820,16 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 + @property + def is_whiteout(self) -> bool: + """Returns True if the inode is a whiteout""" + return (self.i_mode & 0o140000) == 0o140000 + + @property + def is_overlay(self) -> bool: + """Returns True if the inode is an overlay""" + return (self.i_mode & 0o40000) == 0o40000 + def get_inode_type(self) -> Union[str, None]: """Returns inode type name @@ -1840,6 +1850,10 @@ class inode(objects.StructType): return "CHR" elif self.is_block: return "BLK" + elif self.is_whiteout: + return "WHT" + elif self.is_overlay: + return "OVL" else: return None From 60b1c49e49864ce7cb5ae3a3491b6a7e9e40eef3 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Aug 2024 18:10:11 +0200 Subject: [PATCH 172/263] removing test code --- .../framework/symbols/linux/extensions/__init__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0ee6e7d95..06d2e2bf4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1820,16 +1820,6 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 - @property - def is_whiteout(self) -> bool: - """Returns True if the inode is a whiteout""" - return (self.i_mode & 0o140000) == 0o140000 - - @property - def is_overlay(self) -> bool: - """Returns True if the inode is an overlay""" - return (self.i_mode & 0o40000) == 0o40000 - def get_inode_type(self) -> Union[str, None]: """Returns inode type name From 2e9b5b62faec7d8e7fa67dd1e9243013af7cbcb4 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Aug 2024 18:11:29 +0200 Subject: [PATCH 173/263] removing test code --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 06d2e2bf4..05679523f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1840,10 +1840,6 @@ class inode(objects.StructType): return "CHR" elif self.is_block: return "BLK" - elif self.is_whiteout: - return "WHT" - elif self.is_overlay: - return "OVL" else: return None From d4ed07f95b883183fcd28871e7fc7858e649347a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 20:51:32 +1000 Subject: [PATCH 174/263] PR review fixes: Add fixme to remember we should move wintime_to_datetime/unixtime_to_datetime out of renderers --- volatility3/framework/renderers/conversion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index 864794860..c8ddc19fd 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -11,6 +11,7 @@ from typing import Union from volatility3.framework import interfaces, renderers +# FIXME: Move wintime_to_datetime() and unixtime_to_datetime() out of renderers, possibly framework.objects.utility def wintime_to_datetime( wintime: int, ) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: From 0dfb9d8a0ff9080eef10b7505f7e1955e0f728e1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 20:57:23 +1000 Subject: [PATCH 175/263] Linux mountinfo: Add a method to yield all filesystem superblocks --- .../framework/plugins/linux/mountinfo.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index da743bb60..319c92cca 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -37,7 +37,7 @@ class MountInfo(plugins.PluginInterface): _required_framework_version = (2, 2, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -146,7 +146,7 @@ class MountInfo(plugins.PluginInterface): def _get_tasks_mountpoints( self, tasks: Iterable[interfaces.objects.ObjectInterface], - filtered_by_pids: bool, + filtered_by_pids: bool = False, ): seen_mountpoints = set() for task in tasks: @@ -184,8 +184,8 @@ class MountInfo(plugins.PluginInterface): self, tasks: Iterable[interfaces.objects.ObjectInterface], mnt_ns_ids: List[int], - mount_format: bool, - filtered_by_pids: bool, + mount_format: bool = False, + filtered_by_pids: bool = False, ) -> Iterable[Tuple[int, Tuple]]: show_filter_warning = False for task, mnt, mnt_ns_id in self._get_tasks_mountpoints( @@ -247,6 +247,31 @@ class MountInfo(plugins.PluginInterface): "Could not filter by mount namespace id. This field is not available in this kernel." ) + def get_superblocks(self): + """Yield file system superblocks based on the task's mounted filesystems. + + Yields: + super_block: Kernel's struct super_block object + """ + # No filter so that we get all the mount namespaces from all tasks + pid_filter = pslist.PsList.create_pid_filter() + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=pid_filter + ) + + seen_sb_ptr = set() + for task, mnt, _mnt_ns_id in self._get_tasks_mountpoints(tasks): + path_root = linux.LinuxUtilities.get_path_mnt(task, mnt) + if not path_root: + continue + + sb_ptr = mnt.get_mnt_sb() + if not sb_ptr or sb_ptr in seen_sb_ptr: + continue + seen_sb_ptr.add(sb_ptr) + + yield sb_ptr.dereference(), path_root + def run(self): pids = self.config.get("pids") mount_ns_ids = self.config.get("mntns") From 231f682b2769d7a5a6de0fc96008d44c06f0825c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:01:59 +1000 Subject: [PATCH 176/263] Linux: Improve mount's object extension method docstrings --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 05679523f..de5e432d3 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -47,7 +47,7 @@ class module(generic.GenericIntelProcess): ).choices except exceptions.SymbolError: vollog.debug( - f"Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" + "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" ) # set to empty dict to show that the enum was not found, and so shouldn't be searched for again self._mod_mem_type = {} @@ -936,7 +936,8 @@ class mount(objects.StructType): MNT_RELATIME: "relatime", } - def get_mnt_sb(self): + def get_mnt_sb(self) -> int: + """Returns a pointer to the super_block""" if self.has_member("mnt"): return self.mnt.mnt_sb elif self.has_member("mnt_sb"): @@ -1251,6 +1252,7 @@ class vfsmount(objects.StructType): return self._get_real_mnt().has_parent() def get_mnt_sb(self): + """Returns a pointer to the super_block""" return self.mnt_sb def get_flags_access(self) -> str: From a369a9e23cea620a791129d687bd2bbe9c4d4442 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:05:23 +1000 Subject: [PATCH 177/263] Linux: dentry object extension: Add a method to walk dentries subdirectories --- .../symbols/linux/extensions/__init__.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index de5e432d3..3bfbe168a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -820,6 +820,26 @@ class dentry(objects.StructType): current_dentry = current_dentry.d_parent return None + def get_subdirs(self) -> interfaces.objects.ObjectInterface: + """Walks dentry subdirs + + Yields: + A dentry object + """ + if self.has_member("d_sib") and self.has_member("d_children"): + # kernels >= 6.8 + walk_member = "d_sib" + list_head_member = self.d_children.first + elif self.has_member("d_child") and self.has_member("d_subdirs"): + # 2.5.0 <= kernels < 6.8 + walk_member = "d_child" + list_head_member = self.d_subdirs + else: + raise exceptions.VolatilityException("Unsupported dentry type") + + dentry_type_name = self.get_symbol_table_name() + constants.BANG + "dentry" + yield from list_head_member.to_list(dentry_type_name, walk_member) + class struct_file(objects.StructType): def get_dentry(self) -> interfaces.objects.ObjectInterface: From 41684478ad7c12c2d862ac8612f44315bc6f855b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:21:49 +1000 Subject: [PATCH 178/263] Linux: Add page cache support, including abstractions like RadixTree, XArray, and IDR, to support both older and latest kernel versions --- .../framework/symbols/linux/__init__.py | 346 +++++++++++++++++- .../symbols/linux/extensions/__init__.py | 244 +++++++++++- 2 files changed, 588 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 03353135d..248cb8d75 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,6 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import math +from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework @@ -30,6 +32,9 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) self.set_type_class("inode", extensions.inode) + self.set_type_class("idr", extensions.IDR) + self.set_type_class("address_space", extensions.address_space) + self.set_type_class("page", extensions.page) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) @@ -67,7 +72,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 0) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -425,3 +430,342 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): kernel = context.modules[kernel_module_name] return kernel + + @classmethod + def choose_kernel_tree(cls, vmlinux: interfaces.context.ModuleInterface) -> "Tree": + """Returns the appropriate tree data structure instance for the current kernel implementation. + This is used by the IDR and the PageCache to choose between the XArray and RadixTree. + + Args: + vmlinux: The kernel module object + + Returns: + The appropriate Tree instance for the current kernel + """ + address_space_type = vmlinux.get_type("address_space") + address_space_has_i_pages = address_space_type.has_member("i_pages") + i_pages_type_name = ( + address_space_type.child_template("i_pages").vol.type_name + if address_space_has_i_pages + else "" + ) + i_pages_is_xarray = i_pages_type_name.endswith(constants.BANG + "xarray") + i_pages_is_radix_tree_root = i_pages_type_name.endswith( + constants.BANG + "radix_tree_root" + ) and vmlinux.get_type("radix_tree_root").has_member("xa_head") + + if i_pages_is_xarray or i_pages_is_radix_tree_root: + return XArray(vmlinux) + else: + return RadixTree(vmlinux) + + +class Tree(ABC): + """Abstraction to support both XArray and RadixTree""" + + # Dynamic values, these will be initialized later + CHUNK_SHIFT = None + CHUNK_SIZE = None + CHUNK_MASK = None + + def __init__(self, vmlinux: interfaces.context.ModuleInterface): + self.vmlinux = vmlinux + self.vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + + self.pointer_size = self.vmlinux.get_type("pointer").size + # Dynamically work out the (XA_CHUNK|RADIX_TREE_MAP)_SHIFT values based on + # the node.slots[] array size + node_type = self.vmlinux.get_type(self.node_type_name) + slots_array_size = node_type.child_template("slots").count + + # Calculate the LSB index - 1 + self.CHUNK_SHIFT = slots_array_size.bit_length() - 1 + self.CHUNK_SIZE = 1 << self.CHUNK_SHIFT + self.CHUNK_MASK = self.CHUNK_SIZE - 1 + + @property + @abstractmethod + def node_type_name(self) -> str: + """Returns the Tree implementation node type name + + Returns: + A string with the node type name + """ + raise NotImplementedError() + + @property + def tag_internal_value(self) -> int: + """Returns the internal node flag for the tree""" + raise NotImplementedError() + + @abstractmethod + def node_is_internal(self, nodep) -> bool: + """Checks if the node is internal""" + raise NotImplementedError + + @abstractmethod + def is_node_tagged(self, nodep) -> bool: + """Checks if the node pointer is tagged""" + raise NotImplementedError + + @abstractmethod + def untag_node(self, nodep) -> int: + """Untags a node pointer""" + raise NotImplementedError + + @abstractmethod + def get_tree_height(self, treep) -> int: + """Returns the tree height""" + raise NotImplementedError + + @abstractmethod + def get_node_height(self, nodep) -> int: + """Returns the node height""" + raise NotImplementedError + + @abstractmethod + def get_head_node(self, tree) -> int: + """Returns a pointer to the tree's head""" + raise NotImplementedError + + @abstractmethod + def is_valid_node(self, nodep) -> bool: + """Validates a node pointer""" + raise NotImplementedError + + def nodep_to_node(self, nodep) -> interfaces.objects.ObjectInterface: + """Instanciates a tree node from its pointer + + Args: + nodep: Pointer to the XArray/RadixTree node + + Returns: + A XArray/RadixTree node instance + """ + node = self.vmlinux.object(self.node_type_name, offset=nodep, absolute=True) + return node + + def _slot_to_nodep(self, slot) -> int: + if self.node_is_internal(slot): + nodep = slot & ~self.tag_internal_value + else: + nodep = slot + + return nodep + + def _iter_node(self, nodep, height) -> int: + node = self.nodep_to_node(nodep) + node_slots = node.slots + for off in range(self.CHUNK_SIZE): + slot = node_slots[off] + if slot == 0: + continue + + nodep = self._slot_to_nodep(slot) + + if height == 1: + if self.is_valid_node(nodep): + yield nodep + else: + for child_node in self._iter_node(nodep, height - 1): + yield child_node + + def get_page_addresses(self, root: interfaces.objects.ObjectInterface) -> int: + """Walks the tree data structure + + Args: + root: The tree root object + + Yields: + A tree node pointer + """ + height = self.get_tree_height(root.vol.offset) + + nodep = self.get_head_node(root) + if not nodep: + return + + # Keep the internal flag before untagging it + is_internal = self.node_is_internal(nodep) + if self.is_node_tagged(nodep): + nodep = self.untag_node(nodep) + + if is_internal: + height = self.get_node_height(nodep) + + if height == 0: + if self.is_valid_node(nodep): + yield nodep + else: + for child_node in self._iter_node(nodep, height): + yield child_node + + +class XArray(Tree): + XARRAY_TAG_MASK = 3 + XARRAY_TAG_INTERNAL = 2 + + def get_tree_height(self, treep) -> int: + return 0 + + @property + def node_type_name(self) -> str: + return "xa_node" + + @property + def tag_internal_value(self) -> int: + return self.XARRAY_TAG_INTERNAL + + def get_node_height(self, nodep) -> int: + node = self.nodep_to_node(nodep) + return (node.shift / self.CHUNK_SHIFT) + 1 + + def get_head_node(self, tree) -> int: + return tree.xa_head + + def node_is_internal(self, nodep) -> bool: + return (nodep & self.XARRAY_TAG_MASK) == self.XARRAY_TAG_INTERNAL + + def is_node_tagged(self, nodep) -> bool: + return (nodep & self.XARRAY_TAG_MASK) != 0 + + def untag_node(self, nodep) -> int: + return nodep & (~self.XARRAY_TAG_MASK) + + def is_valid_node(self, nodep) -> bool: + # It should have the tag mask clear + return not self.is_node_tagged(nodep) + + +class RadixTree(Tree): + RADIX_TREE_INTERNAL_NODE = 1 + RADIX_TREE_EXCEPTIONAL_ENTRY = 2 + RADIX_TREE_ENTRY_MASK = 3 + + # Dynamic values. These will be initialized later + RADIX_TREE_INDEX_BITS = None + RADIX_TREE_MAX_PATH = None + RADIX_TREE_HEIGHT_SHIFT = None + RADIX_TREE_HEIGHT_MASK = None + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + char_bits = 8 + self.RADIX_TREE_INDEX_BITS = char_bits * self.pointer_size + self.RADIX_TREE_MAX_PATH = int( + math.ceil(self.RADIX_TREE_INDEX_BITS / float(self.CHUNK_SHIFT)) + ) + self.RADIX_TREE_HEIGHT_SHIFT = self.RADIX_TREE_MAX_PATH + 1 + self.RADIX_TREE_HEIGHT_MASK = (1 << self.RADIX_TREE_HEIGHT_SHIFT) - 1 + + if not self.vmlinux.has_type("radix_tree_root"): + # In kernels 4.20, RADIX_TREE_INTERNAL_NODE flag took RADIX_TREE_EXCEPTIONAL_ENTRY's + # value. RADIX_TREE_EXCEPTIONAL_ENTRY was removed but that's managed in is_valid_node() + # Note that the Radix Tree is still in use for IDR, even after kernels 4.20 when XArray + # mostly replace it + self.RADIX_TREE_INTERNAL_NODE = 2 + + @property + def node_type_name(self) -> str: + return "radix_tree_node" + + @property + def tag_internal_value(self) -> int: + return self.RADIX_TREE_INTERNAL_NODE + + def get_tree_height(self, treep) -> int: + try: + if self.vmlinux.get_type("radix_tree_root").has_member("height"): + # kernels < 4.7.10 + radix_tree_root = self.vmlinux.object( + "radix_tree_root", offset=treep, absolute=True + ) + return radix_tree_root.height + except exceptions.SymbolError: + pass + + # kernels >= 4.7.10 + return 0 + + def _radix_tree_maxindex(self, node, height) -> int: + """Return the maximum key which can be store into a radix tree with this height.""" + + if not self.vmlinux.has_symbol("height_to_maxindex"): + # Kernels >= 4.7 + return (self.CHUNK_SIZE << node.shift) - 1 + else: + # Kernels < 4.7 + height_to_maxindex_array = self.vmlinux.object_from_symbol( + "height_to_maxindex" + ) + maxindex = height_to_maxindex_array[height] + return maxindex + + def get_node_height(self, nodep) -> int: + node = self.nodep_to_node(nodep) + if hasattr(node, "shift"): + # 4.7 <= Kernels < 4.20 + return (node.shift / self.CHUNK_SHIFT) + 1 + elif hasattr(node, "path"): + # 3.15 <= Kernels < 4.7 + return node.path & self.RADIX_TREE_HEIGHT_MASK + elif hasattr(node, "height"): + # Kernels < 3.15 + return node.height + else: + raise exceptions.VolatilityException("Cannot find radix-tree node height") + + def get_head_node(self, tree) -> int: + return tree.rnode + + def node_is_internal(self, nodep) -> bool: + return (nodep & self.RADIX_TREE_INTERNAL_NODE) != 0 + + def is_node_tagged(self, nodep) -> bool: + return self.node_is_internal(nodep) + + def untag_node(self, nodep) -> int: + return nodep & (~self.RADIX_TREE_ENTRY_MASK) + + def is_valid_node(self, nodep) -> bool: + # In kernels 4.20, exceptional nodes were removed and internal entries took their bitmask + if self.vmlinux.has_type("radix_tree_root"): + return ( + nodep & self.RADIX_TREE_ENTRY_MASK + ) != self.RADIX_TREE_EXCEPTIONAL_ENTRY + + return True + + +class PageCache(object): + """Linux Page Cache abstraction""" + + def __init__( + self, + page_cache: interfaces.objects.ObjectInterface, + vmlinux: interfaces.context.ModuleInterface, + ): + """ + Args: + page_cache: Page cache address space + vmlinux: Kernel module object + """ + self.vmlinux = vmlinux + self._page_cache = page_cache + self._tree = LinuxUtilities.choose_kernel_tree(self.vmlinux) + + def get_cached_pages(self) -> interfaces.objects.ObjectInterface: + """Returns all page cache contents + + Yields: + Page objects + """ + + for page_addr in self._tree.get_page_addresses(self._page_cache.i_pages): + if not page_addr: + continue + + page = self.vmlinux.object("page", offset=page_addr, absolute=True) + if page: + yield page diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3bfbe168a..300ab2ed0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -4,10 +4,11 @@ import collections.abc import logging +import functools import stat from datetime import datetime import socket as socket_module -from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union +from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, Dict from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion @@ -1919,3 +1920,244 @@ class inode(objects.StructType): The inode's file mode string """ return stat.filemode(self.i_mode) + + def get_pages(self) -> interfaces.objects.ObjectInterface: + """Gets the inode's cached pages + + Yields: + The inode's cached pages + """ + if not self.i_size: + return + elif not (self.i_mapping and self.i_mapping.nrpages > 0): + return + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + page_cache = linux.PageCache(self.i_mapping.dereference(), vmlinux) + yield from page_cache.get_cached_pages() + + def get_contents(self): + """Get the inode cached pages from the page cache + + Yields: + page_index (int): The page index in the Tree. File offset is page_index * PAGE_SIZE. + page_content (str): The page content + """ + for page_obj in self.get_pages(): + page_index = int(page_obj.index) + page_content = page_obj.get_content() + yield page_index, page_content + + +class address_space(objects.StructType): + @property + def i_pages(self): + """Returns the appropriate member containing the page cache tree""" + if self.has_member("i_pages"): + # Kernel >= 4.17 + return self.member("i_pages") + elif self.has_member("page_tree"): + # Kernel < 4.17 + return self.member("page_tree") + + raise exceptions.VolatilityException("Unsupported page cache tree") + + +class page(objects.StructType): + @property + @functools.cache + def pageflags_enum(self) -> Dict: + """Returns 'pageflags' enumeration key/values + + Returns: + A dictionary with the pageflags enumeration key/values + """ + # FIXME: It would be even better to use @functools.cached_property instead, + # however, this requires Python +3.8 + try: + pageflags_enum = self._context.symbol_space.get_enumeration( + self.get_symbol_table_name() + constants.BANG + "pageflags" + ).choices + except exceptions.SymbolError: + vollog.debug( + "Unable to find pageflags enum. This can happen in kernels < 2.6.26 or wrong ISF" + ) + # set to empty dict to show that the enum was not found, and so shouldn't be searched for again + pageflags_enum = {} + + return pageflags_enum + + def flags_list(self) -> List[str]: + """Returns a list of page flags + + Returns: + List of page flags + """ + flags = [] + for name, value in self.pageflags_enum.items(): + if self.flags & (1 << value) != 0: + flags.append(name) + + return flags + + def to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current physical memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + + vmemmap_start = None + if vmlinux.has_symbol("mem_section"): + # SPARSEMEM_VMEMMAP physical memory model: memmap is virtually contiguous + if vmlinux.has_symbol("vmemmap_base"): + # CONFIG_DYNAMIC_MEMORY_LAYOUT - KASLR kernels >= 4.9 + vmemmap_start = vmlinux.object_from_symbol("vmemmap_base") + else: + # !CONFIG_DYNAMIC_MEMORY_LAYOUT + if vmlinux_layer._maxvirtaddr < 57: + # 4-Level paging -> VMEMMAP_START = __VMEMMAP_BASE_L4 + vmemmap_base_l4 = 0xFFFFEA0000000000 + vmemmap_start = vmemmap_base_l4 + else: + # 5-Level paging -> VMEMMAP_START = __VMEMMAP_BASE_L5 + vmemmap_base_l5 = 0xFFD4000000000000 + vmemmap_start = vmemmap_base_l5 + + # FIXME: Remove this exception once 5-level paging is supported. + raise exceptions.VolatilityException( + "5-level paging is not yet supported" + ) + + elif vmlinux.has_symbol("mem_map"): + # FLATMEM physical memory model, typically 32bit + vmemmap_start = vmlinux.object_from_symbol("mem_map") + + elif vmlinux.has_symbol("node_data"): + raise exceptions.VolatilityException("NUMA systems are not yet supported") + else: + raise exceptions.VolatilityException("Unsupported Linux memory model") + + if not vmemmap_start: + raise exceptions.VolatilityException( + "Something went wrong, we shouldn't be here" + ) + + page_type_size = vmlinux.get_type("page").size + pagec = vmlinux_layer.canonicalize(self.vol.offset) + pfn = (pagec - vmemmap_start) // page_type_size + page_paddr = pfn * vmlinux_layer.page_size + + return page_paddr + + def get_content(self) -> Union[str, None]: + """Returns the page content + + Returns: + The page content + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + physical_layer = vmlinux.context.layers["memory_layer"] + page_paddr = self.to_paddr() + if not page_paddr: + return + + page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) + return page_data + + +class IDR(objects.StructType): + IDR_BITS = 8 + IDR_MASK = (1 << IDR_BITS) - 1 + INT_SIZE = 4 + MAX_IDR_SHIFT = INT_SIZE * 8 - 1 + MAX_IDR_BIT = 1 << MAX_IDR_SHIFT + + def idr_max(self, num_layers: int) -> int: + """Returns the maximum ID which can be allocated given idr::layers + + Args: + num_layers: Number of layers + + Returns: + Maximum ID for a given number of layers + """ + # Kernel < 4.17 + bits = min([self.INT_SIZE, num_layers * self.IDR_BITS, self.MAX_IDR_SHIFT]) + + return (1 << bits) - 1 + + def idr_find(self, idr_id: int) -> int: + """Finds an ID within the IDR data structure. + Based on idr_find_slowpath(), 3.9 <= Kernel < 4.11 + Args: + idr_id: The IDR element ID + + Returns: + A pointer to the given ID element + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if not vmlinux.get_type("idr_layer").has_member("layer"): + vollog.info( + "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" + ) + return + + if idr_id < 0: + return + + cur_layer = self.top + if not cur_layer: + return + + n = (cur_layer.layer + 1) * self.IDR_BITS + + if idr_id > self.idr_max(cur_layer.layer + 1): + return + + assert n != 0 + + while n > 0 and cur_layer: + n -= self.IDR_BITS + assert n == cur_layer.layer * self.IDR_BITS + cur_layer = cur_layer.ary[(idr_id >> n) & self.IDR_MASK] + + return cur_layer.v() + + def _old_kernel_get_page_addresses(self, in_use) -> int: + # Kernels < 4.11 + total = next_id = 0 + while total < in_use: + page_addr = self.idr_find(next_id) + if page_addr: + yield page_addr + total += 1 + + next_id += 1 + + def _new_kernel_get_page_addresses(self, _in_use) -> int: + # Kernels >= 4.11 + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + tree = linux.LinuxUtilities.choose_kernel_tree(vmlinux) + for page_addr in tree.get_page_addresses(root=self.idr_rt): + yield page_addr + + def get_page_addresses(self, in_use=0) -> int: + """Walks the IDR and yield a pointer associated with each element. + + Args: + in_use (int, optional): _description_. Defaults to 0. + + Yields: + A pointer associated with each element. + """ + if self.has_member("idr_rt"): + get_page_addresses_func = self._new_kernel_get_page_addresses + else: + get_page_addresses_func = self._old_kernel_get_page_addresses + + for page_addr in get_page_addresses_func(in_use): + yield page_addr From ac27d6663a3733e002cb064afd155139103c4c43 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:26:29 +1000 Subject: [PATCH 179/263] Linux: Add two page cache plugins, linux.pagecache.Files and linux.pagecache.InodePages --- .../framework/plugins/linux/pagecache.py | 504 ++++++++++++++++++ 1 file changed, 504 insertions(+) create mode 100644 volatility3/framework/plugins/linux/pagecache.py diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py new file mode 100644 index 000000000..545c243e0 --- /dev/null +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -0,0 +1,504 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import math +import logging +import datetime +from dataclasses import dataclass, astuple +from typing import List + +from volatility3.framework import renderers, interfaces +from volatility3.framework.renderers import format_hints +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements +from volatility3.plugins import timeliner +from volatility3.plugins.linux import mountinfo + +vollog = logging.getLogger(__name__) + + +@dataclass +class InodeUser: + """Inode user representation, featuring augmented information and formatted fields. + This is the data the plugin will eventually display. + """ + + superblock_addr: int + mountpoint: str + device: str + inode_num: int + inode_addr: int + type: str + inode_pages: int + cached_pages: int + file_mode: str + access_time: str + modification_time: str + change_time: str + path: str + + +@dataclass +class InodeInternal: + """Inode internal representation containing only the core objects + + Fields: + superblock: 'super_block' struct + mountpoint: Superblock mountpoint path + inode: 'inode' struct + path: Dentry full path + """ + + superblock: interfaces.objects.ObjectInterface + mountpoint: str + inode: interfaces.objects.ObjectInterface + path: str + + def to_user( + self, kernel_layer: interfaces.layers.TranslationLayerInterface + ) -> InodeUser: + """Augment the inode information to be presented to the user + + Args: + kernel_layer: The kernel layer to obtain the page size + + Returns: + An InodeUser dataclass + """ + # Ensure all types are atomic immutable. Otherwise, astuple() will take a long + # time doing a deepcopy of the Volatility objects. + superblock_addr = self.superblock.vol.offset + device = f"{self.superblock.major}:{self.superblock.minor}" + inode_num = int(self.inode.i_ino) + inode_addr = self.inode.vol.offset + inode_type = renderers.UnparsableValue() + # Round up the number of pages to fit the inode's size + inode_pages = int(math.ceil(self.inode.i_size / float(kernel_layer.page_size))) + cached_pages = int(self.inode.i_mapping.nrpages) + file_mode = self.inode.get_file_mode() + access_time_dt = self.inode.get_access_time() + modification_time_str = self.inode.get_modification_time() + change_time_str = self.inode.get_change_time() + + inode_user = InodeUser( + superblock_addr=superblock_addr, + mountpoint=self.mountpoint, + device=device, + inode_num=inode_num, + inode_addr=inode_addr, + type=inode_type, + inode_pages=inode_pages, + cached_pages=cached_pages, + file_mode=file_mode, + access_time=access_time_dt, + modification_time=modification_time_str, + change_time=change_time_str, + path=self.path, + ) + return inode_user + + +class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): + """Lists files from memory""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 1, 0) + ), + requirements.ListRequirement( + name="type", + description="List of space-separated file type filters i.e. --type REG DIR", + element_type=str, + optional=True, + ), + requirements.StringRequirement( + name="find", + description="Filename (full path) to find", + optional=True, + ), + ] + + @staticmethod + def _follow_symlink(inode, symlink_path) -> str: + """Follows (fast) symlinks (kernels >= 4.2.x). + Fast symlinks are filesystem agnostic. + + Args: + inode: The inode (or pointer) to dump + symlink_path: The symlink name + + Returns: + If it can resolve the symlink, it returns a string "symlink_path -> target_path" + Otherwise, it returns the same symlink_path + """ + # i_link (fast symlinks) were introduced in 4.2 + if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: + i_link_str = inode.i_link.dereference().cast( + "string", max_length=255, encoding="utf-8", errors="replace" + ) + symlink_path = f"{symlink_path} -> {i_link_str}" + + return symlink_path + + @classmethod + def _walk_dentry(cls, seen_dentries, root_dentry, parent): + + for dentry in root_dentry.get_subdirs(): + dentry_addr = dentry.vol.offset + + # corruption + if dentry_addr == root_dentry.vol.offset: + continue + + if dentry_addr in seen_dentries: + continue + + seen_dentries.add(dentry_addr) + + inode = dentry.d_inode + if not (inode and inode.is_valid()): + continue + + # This allows us to have consistent paths + if dentry.d_name.name: + name = dentry.d_name.name_as_str() + # Do NOT use os.path.join() below + new_file = parent + "/" + name + else: + continue + + yield new_file, dentry, dentry.d_parent.vol.offset + + if inode.is_dir: + for new_file, dentry, parent_address in cls._walk_dentry( + seen_dentries, dentry, new_file + ): + yield new_file, dentry, parent_address + + @classmethod + def get_inodes( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + ): + """Retrieves the inodes from the superblocks + + Args: + context: The context that the plugin will operate within + config_path: The path to configuration data within the context configuration data + + Yields: + An InodeInternal object + """ + + superblocks_iter = mountinfo.MountInfo( + context=context, + config_path=config_path, + ).get_superblocks() + + seen_inodes = set() + seen_dentries = set() + for superblock, mountpoint in superblocks_iter: + parent = "" if mountpoint == "/" else mountpoint + + # Superblock root dentry + root_dentry = superblock.s_root + if not root_dentry: + continue + + # Dentry sanity check + if not root_dentry.is_root(): + continue + + # More dentry/inode sanity checks + root_inode_ptr = root_dentry.d_inode + if not root_inode_ptr: + continue + root_inode = root_inode_ptr.dereference() + if not root_inode.is_valid(): + continue + + # Inode already processed? + if root_inode_ptr in seen_inodes: + continue + seen_inodes.add(root_inode_ptr) + + root_path = mountpoint + + inode_in = InodeInternal( + superblock=superblock, + mountpoint=mountpoint, + inode=root_inode, + path=root_path, + ) + yield inode_in + + # Children + for file_path, file_dentry, _ in cls._walk_dentry( + seen_dentries, root_dentry, parent + ): + if not file_dentry: + continue + # Dentry/inode sanity checks + file_inode_ptr = file_dentry.d_inode + if not file_inode_ptr: + continue + file_inode = file_inode_ptr.dereference() + if not file_inode.is_valid(): + continue + + # Inode already processed? + if file_inode_ptr in seen_inodes: + continue + seen_inodes.add(file_inode_ptr) + + file_path = cls._follow_symlink(file_inode_ptr, file_path) + inode_in = InodeInternal( + superblock=superblock, + mountpoint=mountpoint, + inode=file_inode, + path=file_path, + ) + yield inode_in + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + inodes_iter = self.get_inodes( + context=self.context, config_path=self.config_path + ) + + types_filter = self.config["type"] + for inode_in in inodes_iter: + if types_filter and inode_in.inode.get_inode_type() not in types_filter: + continue + + if self.config["find"]: + if inode_in.path == self.config["find"]: + inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) + break # Only the first match + else: + inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) + + def generate_timeline(self): + """Generates tuples of (description, timestamp_type, timestamp) + + These need not be generated in any particular order, sorting + will be done later + """ + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + inodes_iter = self.get_inodes( + context=self.context, config_path=self.config_path + ) + for inode_in in inodes_iter: + inode_out = inode_in.to_user(vmlinux_layer) + description = f"Cached Inode for {inode_out.path}" + yield description, timeliner.TimeLinerType.ACCESSED, inode_out.access_time + yield description, timeliner.TimeLinerType.MODIFIED, inode_out.modification_time + yield description, timeliner.TimeLinerType.CHANGE, inode_out.change_time + + @staticmethod + def format_fields_with_headers(headers, generator): + """Uses the headers type to cast the fields obtained from the generator""" + for level, fields in generator: + formatted_fields = [] + for header, field in zip(headers, fields): + header_type = header[1] + + if isinstance( + field, (header_type, interfaces.renderers.BaseAbsentValue) + ): + formatted_field = field + else: + formatted_field = header_type(field) + + formatted_fields.append(formatted_field) + yield level, formatted_fields + + def run(self): + headers = [ + ("SuperblockAddr", format_hints.Hex), + ("MountPoint", str), + ("Device", str), + ("InodeNum", int), + ("InodeAddr", format_hints.Hex), + ("FileType", str), + ("InodePages", int), + ("CachedPages", int), + ("FileMode", str), + ("AccessTime", datetime.datetime), + ("ModificationTime", datetime.datetime), + ("ChangeTime", datetime.datetime), + ("FilePath", str), + ] + + return renderers.TreeGrid( + headers, self.format_fields_with_headers(headers, self._generator()) + ) + + +class InodePages(plugins.PluginInterface): + """Lists and recovers cached inode pages""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="files", plugin=Files, version=(1, 0, 0) + ), + requirements.StringRequirement( + name="find", + description="Filename (full path) to find ", + optional=True, + ), + requirements.IntRequirement( + name="inode", + description="Inode address", + optional=True, + ), + requirements.StringRequirement( + name="dump", + description="Output file path", + optional=True, + ), + ] + + @staticmethod + def write_inode_content_to_file( + inode: interfaces.objects.ObjectInterface, + filename: str, + vmlinux_layer: interfaces.layers.TranslationLayerInterface, + ) -> None: + """Extracts the inode's contents from the page cache and saves them to a file + + Args: + inode: The inode to dump + filename: Filename for writing the inode content + vmlinux_layer: The kernel layer to obtain the page size + """ + if not inode.is_reg: + vollog.error("The inode is not a regular file") + return + + # By using truncate/seek, provided the filesystem supports it, a sparse file will be + # created, saving both disk space and I/O time. + # Additionally, using the page index will guarantee that each page is written at the + # appropriate file position. + try: + with open(filename, "wb") as f: + inode_size = inode.i_size + f.truncate(inode_size) + + for page_idx, page_content in inode.get_contents(): + current_fp = page_idx * vmlinux_layer.page_size + max_length = inode_size - current_fp + page_bytes = page_content[:max_length] + if current_fp + len(page_bytes) > inode_size: + vollog.error( + "Page out of file bounds: inode 0x%x, inode size %d, page index %d", + inode.vol.object, + inode_size, + page_idx, + ) + f.seek(current_fp) + f.write(page_bytes) + + except IOError as e: + vollog.error("Unable to write to file (%s): %s", filename, e) + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + if self.config["inode"] and self.config["find"]: + vollog.error("Cannot use --inode and --find simultaneously") + return + + if self.config["find"]: + inodes_iter = Files.get_inodes( + context=self.context, config_path=self.config_path + ) + for inode_in in inodes_iter: + if inode_in.path == self.config["find"]: + inode = inode_in.inode + break # Only the first match + + elif self.config["inode"]: + inode = vmlinux.object("inode", self.config["inode"], absolute=True) + else: + vollog.error("You must use either --inode or --find") + return + + if not inode.is_reg: + vollog.error("The inode is not a regular file") + return + + inode_size = inode.i_size + if not inode.is_valid(): + vollog.error("Invalid inode at 0x%x", self.config["inode"]) + return + + for page_obj in inode.get_pages(): + page_vaddr = page_obj.vol.offset + page_paddr = page_obj.to_paddr() + page_mapping_addr = page_obj.mapping + page_index = int(page_obj.index) + page_file_offset = page_index * vmlinux_layer.page_size + dump_safe = page_file_offset < inode_size + page_flags_list = page_obj.get_flags() + page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) + fields = ( + page_vaddr, + page_paddr, + page_mapping_addr, + page_index, + dump_safe, + page_flags, + ) + + yield 0, fields + + if self.config["dump"]: + filename = self.config["dump"] + vollog.info("[*] Writing inode at 0x%x to '%s'", inode.vol.offset, filename) + self.write_inode_content_to_file(inode, filename, vmlinux_layer) + + def run(self): + headers = [ + ("PageVAddr", format_hints.Hex), + ("PagePAddr", format_hints.Hex), + ("MappingAddr", format_hints.Hex), + ("Index", int), + ("DumpSafe", bool), + ("Flags", str), + ] + + return renderers.TreeGrid( + headers, Files.format_fields_with_headers(headers, self._generator()) + ) From 55212008f805abdd47f2f3d7d6211c198097f8d4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:39:51 +1000 Subject: [PATCH 180/263] Linux: Add pidhashtable plugin. This is based on the vol2 plugin, removing ancient kernel support, curating code and enhancing comments, while using the new IDR abstraction included also in this effort. --- .../framework/plugins/linux/pidhashtable.py | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 volatility3/framework/plugins/linux/pidhashtable.py diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py new file mode 100644 index 000000000..b24c73e77 --- /dev/null +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -0,0 +1,249 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List + +from volatility3.framework import renderers, interfaces, constants +from volatility3.framework.symbols import linux +from volatility3.framework.renderers import format_hints +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class PIDHashTable(plugins.PluginInterface): + """Enumerates processes through the PID hash table""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + ), + requirements.BooleanRequirement( + name="decorate_comm", + description="Show `user threads` comm in curly brackets, and `kernel threads` comm in square brackets", + optional=True, + default=False, + ), + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.vmlinux = None + self.vmlinux_layer = None + + def _is_valid_task(self, task): + return task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent) + + def _get_pidtype_pid(self): + # The pid_type enumeration is present since 2.5.37, just in case + pid_type_enum = self.vmlinux.get_enumeration("pid_type") + if not pid_type_enum: + vollog.error("Cannot find pid_type enum. Unsupported kernel") + return + + pidtype_pid = pid_type_enum.choices.get("PIDTYPE_PID") + if pidtype_pid is None: + vollog.error("Cannot find PIDTYPE_PID. Unsupported kernel") + return + + # Typically PIDTYPE_PID = 0 + return pidtype_pid + + def _get_pidhash_array(self): + pidhash_shift = self.vmlinux.object_from_symbol("pidhash_shift") + pidhash_size = 1 << pidhash_shift + + array_type_name = self.vmlinux.symbol_table_name + constants.BANG + "array" + + pidhash_ptr = self.vmlinux.object_from_symbol("pid_hash") + # pidhash is an array of hlist_heads + pidhash = self._context.object( + array_type_name, + offset=pidhash_ptr, + subtype=self.vmlinux.get_type("hlist_head"), + count=pidhash_size, + layer_name=self.vmlinux.layer_name, + ) + + return pidhash + + def _walk_upid(self, seen_upids, upid): + while upid and self.vmlinux_layer.is_valid(upid.vol.offset): + if upid.vol.offset in seen_upids: + break + seen_upids.add(upid.vol.offset) + + pid_chain = upid.pid_chain + if not (pid_chain and self.vmlinux_layer.is_valid(pid_chain.vol.offset)): + break + + upid = linux.LinuxUtilities.container_of( + pid_chain.next, "upid", "pid_chain", self.vmlinux + ) + + def _get_upids(self): + # 2.6.24 <= kernels < 4.15 + pidhash = self._get_pidhash_array() + + seen_upids = set() + for hlist in pidhash: + # each entry in the hlist is a upid which is wrapped in a pid + ent = hlist.first + + while ent and self.vmlinux_layer.is_valid(ent.vol.offset): + # upid->pid_chain exists 2.6.24 <= kernel < 4.15 + upid = linux.LinuxUtilities.container_of( + ent.vol.offset, "upid", "pid_chain", self.vmlinux + ) + + if upid.vol.offset in seen_upids: + break + + self._walk_upid(seen_upids, upid) + + ent = ent.next + + return seen_upids + + def _pid_hash_implementation(self): + # 2.6.24 <= kernels < 4.15 + task_pids_off = self.vmlinux.get_type("task_struct").relative_child_offset( + "pids" + ) + pidtype_pid = self._get_pidtype_pid() + + for upid in self._get_upids(): + pid = linux.LinuxUtilities.container_of( + upid, "pid", "numbers", self.vmlinux + ) + if not pid: + continue + + pid_tasks_0 = pid.tasks[pidtype_pid].first + if not pid_tasks_0: + continue + + task = self.vmlinux.object( + "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True + ) + if self._is_valid_task(task): + yield task + + def _task_for_radix_pid_node(self, nodep): + # kernels >= 4.15 + pid = self.vmlinux.object("pid", offset=nodep, absolute=True) + pidtype_pid = self._get_pidtype_pid() + + pid_tasks_0 = pid.tasks[pidtype_pid].first + if not pid_tasks_0: + return + + task_struct_type = self.vmlinux.get_type("task_struct") + if task_struct_type.has_member("pids"): + member = "pids" + elif task_struct_type.has_member("pid_links"): + member = "pid_links" + else: + return None + + task_pids_off = task_struct_type.relative_child_offset(member) + task = self.vmlinux.object( + "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True + ) + return task + + def _pid_namespace_idr(self): + # kernels >= 4.15 + ns_addr = self.vmlinux.get_symbol("init_pid_ns").address + ns = self.vmlinux.object("pid_namespace", offset=ns_addr) + + for page_addr in ns.idr.get_page_addresses(): + task = self._task_for_radix_pid_node(page_addr) + if self._is_valid_task(task): + yield task + + def _determine_pid_func(self): + pid_hash = self.vmlinux.has_symbol("pid_hash") and self.vmlinux.has_symbol( + "pidhash_shift" + ) # 2.5.55 <= kernels < 4.15 + + has_pid_numbers = self.vmlinux.has_type("pid") and self.vmlinux.get_type( + "pid" + ).has_member( + "numbers" + ) # kernels >= 2.6.24 + + has_pid_numbers = self.vmlinux.has_type("upid") and self.vmlinux.get_type( + "upid" + ).has_member( + "pid_chain" + ) # 2.6.24 <= kernels < 4.15 + + # kernels >= 4.15 + pid_idr = self.vmlinux.has_type("pid_namespace") and self.vmlinux.get_type( + "pid_namespace" + ).has_member("idr") + + if pid_idr: + # kernels >= 4.15 + return self._pid_namespace_idr + elif pid_hash and has_pid_numbers and has_pid_numbers: + # 2.6.24 <= kernels < 4.15 + return self._pid_hash_implementation + + return None + + def get_tasks(self) -> interfaces.objects.ObjectInterface: + """Enumerates processes through the PID hash table + + Yields: + task_struct objects + """ + self.vmlinux = self.context.modules[self.config["kernel"]] + self.vmlinux_layer = self.context.layers[self.vmlinux.layer_name] + pid_func = self._determine_pid_func() + if not pid_func: + vollog.error("Cannot determine which PID hash table this kernel is using") + return + + yield from sorted(pid_func(), key=lambda t: (t.tgid, t.pid)) + + def _generator( + self, decorate_comm: bool = False + ) -> interfaces.objects.ObjectInterface: + for task in self.get_tasks(): + offset, pid, tid, ppid, name = pslist.PsList.get_task_fields( + task, decorate_comm + ) + fields = format_hints.Hex(offset), pid, tid, ppid, name + yield 0, fields + + def run(self): + decorate_comm = self.config.get("decorate_comm") + + headers = [ + ("OFFSET", format_hints.Hex), + ("PID", int), + ("TID", int), + ("PPID", int), + ("COMM", str), + ] + return renderers.TreeGrid(headers, self._generator(decorate_comm=decorate_comm)) From 103537801ee0b49ca3475be11e1fe62670938e91 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:44:38 +1000 Subject: [PATCH 181/263] Linux: Add a basic eBPF program enumeration plugin to test and demonstrate using the IDR abstraction --- volatility3/framework/plugins/linux/ebpf.py | 78 +++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 volatility3/framework/plugins/linux/ebpf.py diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py new file mode 100644 index 000000000..33ba71faf --- /dev/null +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -0,0 +1,78 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import binascii +import logging +from typing import List + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements + +vollog = logging.getLogger(__name__) + + +class EBPF(plugins.PluginInterface): + """Enumerate eBPF programs""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + ] + + def get_ebpf_programs(self, vmlinux) -> interfaces.objects.ObjectInterface: + """Enumerate eBPF programs walking its IDR. + + Args: + vmlinux: The kernel symbols object + + Yields: + eBPF program objects + """ + if not vmlinux.has_symbol("prog_idr"): + raise exceptions.VolatilityException( + "Cannot find the eBPF prog idr. Unsupported kernel" + ) + + prog_idr_addr = vmlinux.get_symbol("prog_idr").address + prog_idr = vmlinux.object("idr", offset=prog_idr_addr) + for page_addr in prog_idr.get_page_addresses(): + bpf_prog = vmlinux.object("bpf_prog", offset=page_addr, absolute=True) + yield bpf_prog + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + bpf_prog_types = vmlinux.get_enumeration("bpf_prog_type") + for prog in self.get_ebpf_programs(vmlinux): + prog_addr = prog.vol.offset + prog_type = bpf_prog_types.lookup(prog.type) + prog_tag_addr = prog.tag.vol.offset + prog_tag_size = prog.tag.count + prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) + prog_tag = binascii.hexlify(prog_tag_bytes).decode() + prog_name = ( + utility.array_to_string(prog.aux.name) or renderers.NotAvailableValue() + ) + fields = (format_hints.Hex(prog_addr), prog_name, prog_tag, prog_type) + yield (0, fields) + + def run(self): + headers = [ + ("Address", format_hints.Hex), + ("Name", str), + ("Tag", str), + ("Type", str), + ] + return renderers.TreeGrid(headers, self._generator()) From cc04f665e35989fea4c108de2bdd1b31016145ed Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 23:47:28 +1000 Subject: [PATCH 182/263] Fix inode type --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 545c243e0..c36f8a339 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -72,7 +72,7 @@ class InodeInternal: device = f"{self.superblock.major}:{self.superblock.minor}" inode_num = int(self.inode.i_ino) inode_addr = self.inode.vol.offset - inode_type = renderers.UnparsableValue() + inode_type = self.inode.get_inode_type() or renderers.UnparsableValue() # Round up the number of pages to fit the inode's size inode_pages = int(math.ceil(self.inode.i_size / float(kernel_layer.page_size))) cached_pages = int(self.inode.i_mapping.nrpages) From 3e75c2ae9d29084485f6d2803c64d2c42e064ee0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 4 Aug 2024 14:40:45 +1000 Subject: [PATCH 183/263] Fix @functools.cache . It's available since Python 3.9 --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 300ab2ed0..2d48f677b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1965,7 +1965,7 @@ class address_space(objects.StructType): class page(objects.StructType): @property - @functools.cache + @functools.lru_cache() def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values From 8d6fd3cd78f0fadd223ae93a70a268048b4ccfe9 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 5 Aug 2024 14:28:43 +0200 Subject: [PATCH 184/263] Moved get_inode_metadata, separated inode and FD processing, error handling precision --- volatility3/framework/plugins/linux/lsof.py | 65 ++++++++++++------- .../framework/symbols/linux/__init__.py | 26 +------- 2 files changed, 45 insertions(+), 46 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index fa9d2bf61..167556e7d 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -6,7 +6,7 @@ found in Linux's /proc file system.""" import logging, datetime from typing import List, Callable -from volatility3.framework import renderers, interfaces, constants +from volatility3.framework import renderers, interfaces, constants, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -46,7 +46,30 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ] @classmethod - def list_fds_and_inodes( + def get_inode_metadata(cls, filp: interfaces.objects.ObjectInterface): + try: + dentry = filp.get_dentry() + if dentry: + inode_object = dentry.d_inode + if inode_object and inode_object.is_valid(): + itype = ( + inode_object.get_inode_type() or renderers.NotAvailableValue() + ) + return ( + inode_object.i_ino, + itype, + inode_object.i_size, + inode_object.get_file_mode(), + inode_object.get_change_time(), + inode_object.get_modification_time(), + inode_object.get_access_time(), + ) + except (exceptions.InvalidAddressException, AttributeError) as e: + vollog.warning(f"Can't get inode metadata: {e}") + return tuple(renderers.NotAvailableValue() for _ in range(7)) + + @classmethod + def list_fds( cls, context: interfaces.context.ContextInterface, symbol_table: str, @@ -67,26 +90,27 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ) for fd_fields in fd_generator: - fd_num, filp, full_path = fd_fields - inode_metadata = linux.LinuxUtilities.get_inode_metadata(context, filp) - try: - inode_num, itype, file_size, imode, ctime, mtime, atime = next( - inode_metadata - ) - except Exception as e: - vollog.warning( - f"Can't get inode metadata for file descriptor {fd_num}: {e}" - ) - inode_num = itype = file_size = imode = ctime = mtime = atime = ( - renderers.NotAvailableValue() - ) - yield pid, task_comm, task, fd_num, filp, full_path, inode_num, itype, imode, ctime, mtime, atime, file_size + yield pid, task_comm, task, fd_fields + + @classmethod + def list_fds_and_inodes( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + filter_func: Callable[[int], bool] = lambda _: False, + ): + for pid, task_comm, task, (fd_num, filp, full_path) in cls.list_fds( + context, symbol_table, filter_func + ): + inode_metadata = cls.get_inode_metadata(filp) + yield pid, task_comm, task, fd_num, filp, full_path, inode_metadata def _generator(self, pids, symbol_table): filter_func = pslist.PsList.create_pid_filter(pids) fds_generator = self.list_fds_and_inodes( self.context, symbol_table, filter_func=filter_func ) + for ( pid, task_comm, @@ -94,14 +118,9 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): fd_num, filp, full_path, - inode_num, - itype, - imode, - ctime, - mtime, - atime, - file_size, + inode_metadata, ) in fds_generator: + inode_num, itype, file_size, imode, ctime, mtime, atime = inode_metadata fields = ( pid, task_comm, diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index d52c43dae..03353135d 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,8 +1,8 @@ -# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # from typing import Iterator, List, Tuple, Optional, Union -import datetime, stat + from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility @@ -67,7 +67,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 2, 0) + _version = (2, 1, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -274,26 +274,6 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): yield fd_num, filp, full_path - @classmethod - def get_inode_metadata(cls, context: interfaces.context.ContextInterface, filp): - """ - A helper function that gets the inodes metadata from a file descriptor - """ - dentry = filp.get_dentry() - if dentry: - inode_object = dentry.d_inode - if inode_object and inode_object.is_valid(): - itype = inode_object.get_inode_type() or "?" - yield ( - inode_object.i_ino, - itype, - inode_object.i_size, - inode_object.get_file_mode(), - inode_object.get_change_time(), - inode_object.get_modification_time(), - inode_object.get_access_time(), - ) - @classmethod def mask_mods_list( cls, From c9eb81c95fa530c58e493d77cde506f001e9e4f3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 04:40:48 -0700 Subject: [PATCH 185/263] PR review fixes: Improve eBPF extension objects: bpf_prog and added bpf_prog_aux. Apply changes to the EBPF and Sockstat plugins. --- volatility3/framework/plugins/linux/ebpf.py | 15 ++---- .../framework/plugins/linux/sockstat.py | 14 +++--- .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 48 +++++++++++++++++-- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 33ba71faf..8df506b06 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -1,12 +1,10 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import binascii import logging from typing import List from volatility3.framework import renderers, interfaces, exceptions -from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins from volatility3.framework.configuration import requirements @@ -53,18 +51,11 @@ class EBPF(plugins.PluginInterface): def _generator(self): vmlinux = self.context.modules[self.config["kernel"]] - vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - bpf_prog_types = vmlinux.get_enumeration("bpf_prog_type") for prog in self.get_ebpf_programs(vmlinux): prog_addr = prog.vol.offset - prog_type = bpf_prog_types.lookup(prog.type) - prog_tag_addr = prog.tag.vol.offset - prog_tag_size = prog.tag.count - prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) - prog_tag = binascii.hexlify(prog_tag_bytes).decode() - prog_name = ( - utility.array_to_string(prog.aux.name) or renderers.NotAvailableValue() - ) + prog_type = prog.get_type() or renderers.NotAvailableValue() + prog_tag = prog.get_tag() or renderers.NotAvailableValue() + prog_name = prog.get_name() or renderers.NotAvailableValue() fields = (format_hints.Hex(prog_addr), prog_name, prog_tag, prog_type) yield (0, fields) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 78217fbec..b0503b105 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -151,17 +151,15 @@ class SockHandlers(interfaces.configuration.VersionableInterface): bpfprog = sock_filter.prog - BPF_PROG_TYPE_UNSPEC = 0 # cBPF filter - try: - bpfprog_type = bpfprog.get_type() - if bpfprog_type == BPF_PROG_TYPE_UNSPEC: - return # cBPF filter - except AttributeError: + bpfprog_type = bpfprog.get_type() + if not bpfprog_type: # kernel < 3.18.140, it's a cBPF filter return None - BPF_PROG_TYPE_SOCKET_FILTER = 1 # eBPF filter - if bpfprog_type != BPF_PROG_TYPE_SOCKET_FILTER: + if bpfprog_type == "BPF_PROG_TYPE_UNSPEC": + return None # cBPF filter + + if bpfprog_type != "BPF_PROG_TYPE_SOCKET_FILTER": socket_filter["bpf_filter_type"] = f"UNK({bpfprog_type})" vollog.warning(f"Unexpected BPF type {bpfprog_type} for a socket") return None diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 248cb8d75..7a87135ff 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -38,6 +38,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) + self.optional_set_type_class("bpf_prog_aux", extensions.bpf_prog_aux) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 2d48f677b..2b971fb79 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -5,6 +5,7 @@ import collections.abc import logging import functools +import binascii import stat from datetime import datetime import socket as socket_module @@ -1607,20 +1608,59 @@ class xdp_sock(objects.StructType): class bpf_prog(objects.StructType): - def get_type(self): + def get_type(self) -> Union[str, None]: + """Returns a string with the eBPF program type""" + # The program type was in `bpf_prog_aux::prog_type` from 3.18.140 to # 4.1.52 before it was moved to `bpf_prog::type` if self.has_member("type"): # kernel >= 4.1.52 - return self.type + return self.type.description if self.has_member("aux") and self.aux: if self.aux.has_member("prog_type"): # 3.18.140 <= kernel < 4.1.52 - return self.aux.prog_type + return self.aux.prog_type.description # kernel < 3.18.140 - raise AttributeError("Unable to find the BPF type") + return None + + def get_tag(self) -> Union[str, None]: + """Returns a string with the eBPF program tag""" + # 'tag' was added in kernels 4.10 + if not self.has_member("tag"): + return None + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + + prog_tag_addr = self.tag.vol.offset + prog_tag_size = self.tag.count + prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) + + prog_tag = binascii.hexlify(prog_tag_bytes).decode() + return prog_tag + + def get_name(self) -> Union[str, None]: + """Returns a string with the eBPF program name""" + if not self.has_member("aux"): + # 'prog_aux' was added in kernels 3.18 + return None + + return self.aux.get_name() + + +class bpf_prog_aux(objects.StructType): + def get_name(self) -> Union[str, None]: + """Returns a string with the eBPF program name""" + if not self.has_member("name"): + # 'name' was added in kernels 4.15 + return None + + if not self.name: + return None + + return utility.array_to_string(self.name) class cred(objects.StructType): From bd37aa3930c056fc8511969592f731a29e253c84 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 04:48:49 -0700 Subject: [PATCH 186/263] PR review fixes: Fix pidhashtable plugin --- volatility3/framework/plugins/linux/pidhashtable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index b24c73e77..ef110bdf1 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -191,7 +191,7 @@ class PIDHashTable(plugins.PluginInterface): "numbers" ) # kernels >= 2.6.24 - has_pid_numbers = self.vmlinux.has_type("upid") and self.vmlinux.get_type( + has_pid_chain = self.vmlinux.has_type("upid") and self.vmlinux.get_type( "upid" ).has_member( "pid_chain" @@ -205,7 +205,7 @@ class PIDHashTable(plugins.PluginInterface): if pid_idr: # kernels >= 4.15 return self._pid_namespace_idr - elif pid_hash and has_pid_numbers and has_pid_numbers: + elif pid_hash and has_pid_numbers and has_pid_numbers and has_pid_chain: # 2.6.24 <= kernels < 4.15 return self._pid_hash_implementation From 7df0636f30d47f1d5b9373671aac410549474766 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 04:57:33 -0700 Subject: [PATCH 187/263] PR review fixes: Fix pidhashtable plugin explicit returns mixed with implicit returns --- volatility3/framework/plugins/linux/pidhashtable.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index ef110bdf1..73cdff452 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -50,19 +50,19 @@ class PIDHashTable(plugins.PluginInterface): self.vmlinux_layer = None def _is_valid_task(self, task): - return task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent) + return bool(task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent)) def _get_pidtype_pid(self): # The pid_type enumeration is present since 2.5.37, just in case pid_type_enum = self.vmlinux.get_enumeration("pid_type") if not pid_type_enum: vollog.error("Cannot find pid_type enum. Unsupported kernel") - return + return None pidtype_pid = pid_type_enum.choices.get("PIDTYPE_PID") if pidtype_pid is None: vollog.error("Cannot find PIDTYPE_PID. Unsupported kernel") - return + return None # Typically PIDTYPE_PID = 0 return pidtype_pid @@ -154,7 +154,7 @@ class PIDHashTable(plugins.PluginInterface): pid_tasks_0 = pid.tasks[pidtype_pid].first if not pid_tasks_0: - return + return None task_struct_type = self.vmlinux.get_type("task_struct") if task_struct_type.has_member("pids"): From 1e3e9e2c78cbfc4e24362c323432660c98373516 Mon Sep 17 00:00:00 2001 From: Arcuri Davide Date: Tue, 6 Aug 2024 16:28:59 +0200 Subject: [PATCH 188/263] add args and kwargs to threads.py init Without args and kwargs there were an issue with timeliner plugin that tried to pass additional parameters like progress_callback raising TypeError --- volatility3/framework/plugins/windows/threads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index ae70e717b..39f3b7d77 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -18,9 +18,9 @@ class Threads(thrdscan.ThrdScan): _required_framework_version = (2, 4, 0) _version = (1, 0, 0) - def __init__(self): + def __init__(self, *args, **kwargs): self.implementation = self.list_process_threads - super().__init__() + super().__init__(*args, **kwargs) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 339a9a94f57adb2039984facf0cd22df0ba4bf90 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 23:00:09 -0700 Subject: [PATCH 189/263] PR review fixes: pidhashtable plugin add missing typing. --- volatility3/framework/plugins/linux/pidhashtable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 73cdff452..3c429dc2f 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -49,7 +49,7 @@ class PIDHashTable(plugins.PluginInterface): self.vmlinux = None self.vmlinux_layer = None - def _is_valid_task(self, task): + def _is_valid_task(self, task) -> bool: return bool(task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent)) def _get_pidtype_pid(self): From c8cb4465da3d71a879b47d981fdd4871afd9d521 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 23:08:21 -0700 Subject: [PATCH 190/263] PR review fixes: Remove filter function, it isn't needed --- volatility3/framework/plugins/linux/mountinfo.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 319c92cca..dfb2e2f52 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -254,10 +254,7 @@ class MountInfo(plugins.PluginInterface): super_block: Kernel's struct super_block object """ # No filter so that we get all the mount namespaces from all tasks - pid_filter = pslist.PsList.create_pid_filter() - tasks = pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=pid_filter - ) + tasks = pslist.PsList.list_tasks(self.context, self.config["kernel"]) seen_sb_ptr = set() for task, mnt, _mnt_ns_id in self._get_tasks_mountpoints(tasks): From f737b88d03d9be9f4f0a9b43a83e02a88a620068 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 23:12:01 -0700 Subject: [PATCH 191/263] PR review fixes: Improve _walk_dentry() and get_inodes() variable names, arguments and return values --- .../framework/plugins/linux/pagecache.py | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index c36f8a339..cf09f23d8 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List +from typing import List, Set from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints @@ -153,7 +153,23 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): return symlink_path @classmethod - def _walk_dentry(cls, seen_dentries, root_dentry, parent): + def _walk_dentry( + cls, + seen_dentries: Set[int], + root_dentry: interfaces.objects.ObjectInterface, + parent_dir: str, + ): + """Walk dentries recursively + + Args: + seen_dentries: A set to ensure each dentry is processed only once + root_dentry: Root dentry object + parent_dir: Parent directory path + + Yields: + file_path: Filename including path + dentry: Dentry object + """ for dentry in root_dentry.get_subdirs(): dentry_addr = dentry.vol.offset @@ -173,19 +189,16 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): # This allows us to have consistent paths if dentry.d_name.name: - name = dentry.d_name.name_as_str() + basename = dentry.d_name.name_as_str() # Do NOT use os.path.join() below - new_file = parent + "/" + name + file_path = parent_dir + "/" + basename else: continue - yield new_file, dentry, dentry.d_parent.vol.offset + yield file_path, dentry if inode.is_dir: - for new_file, dentry, parent_address in cls._walk_dentry( - seen_dentries, dentry, new_file - ): - yield new_file, dentry, parent_address + yield from cls._walk_dentry(seen_dentries, dentry, parent_dir=file_path) @classmethod def get_inodes( @@ -211,13 +224,15 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): seen_inodes = set() seen_dentries = set() for superblock, mountpoint in superblocks_iter: - parent = "" if mountpoint == "/" else mountpoint + parent_dir = "" if mountpoint == "/" else mountpoint # Superblock root dentry - root_dentry = superblock.s_root - if not root_dentry: + root_dentry_ptr = superblock.s_root + if not root_dentry_ptr: continue + root_dentry = root_dentry_ptr.dereference() + # Dentry sanity check if not root_dentry.is_root(): continue @@ -246,8 +261,8 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): yield inode_in # Children - for file_path, file_dentry, _ in cls._walk_dentry( - seen_dentries, root_dentry, parent + for file_path, file_dentry in cls._walk_dentry( + seen_dentries, root_dentry, parent_dir ): if not file_dentry: continue From 805b3514c3b14c53724cd38d722540d84949d51f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 7 Aug 2024 00:23:19 -0700 Subject: [PATCH 192/263] PR review fixes: Fix page flags list method name, this was introduced earlier in another commit of this PR. --- volatility3/framework/plugins/linux/pagecache.py | 2 +- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index cf09f23d8..fd62cc1e4 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -486,7 +486,7 @@ class InodePages(plugins.PluginInterface): page_index = int(page_obj.index) page_file_offset = page_index * vmlinux_layer.page_size dump_safe = page_file_offset < inode_size - page_flags_list = page_obj.get_flags() + page_flags_list = page_obj.get_flags_list() page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) fields = ( page_vaddr, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 2b971fb79..900cc9b6c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2027,7 +2027,7 @@ class page(objects.StructType): return pageflags_enum - def flags_list(self) -> List[str]: + def get_flags_list(self) -> List[str]: """Returns a list of page flags Returns: From 46842981fb25aa6f1b242ef5990642fdeeb0cf00 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 7 Aug 2024 00:24:58 -0700 Subject: [PATCH 193/263] PR review fixes: Use contextlib.suppress() instead of an empty exception handler --- volatility3/framework/symbols/linux/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 7a87135ff..96bc56a18 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import math +import contextlib from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union @@ -676,15 +677,13 @@ class RadixTree(Tree): return self.RADIX_TREE_INTERNAL_NODE def get_tree_height(self, treep) -> int: - try: + with contextlib.suppress(exceptions.SymbolError): if self.vmlinux.get_type("radix_tree_root").has_member("height"): # kernels < 4.7.10 radix_tree_root = self.vmlinux.object( "radix_tree_root", offset=treep, absolute=True ) return radix_tree_root.height - except exceptions.SymbolError: - pass # kernels >= 4.7.10 return 0 From 10376a2687b8546df88bf6b3a3e11126f8816479 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Wed, 7 Aug 2024 10:38:29 +0200 Subject: [PATCH 194/263] use one class with flag --- README.md | 2 +- pyproject.toml | 2 +- volatility3/framework/__init__.py | 2 +- .../framework/plugins/windows/vadyarascan.py | 7 +- volatility3/framework/plugins/yarascan.py | 81 ++++++++----------- 5 files changed, 38 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 886790df8..1463c2bde 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.7.3 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.8.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/pyproject.toml b/pyproject.toml index 207f762cc..2e1636a43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, ] -requires-python = ">=3.7.3" +requires-python = ">=3.8.0" license = { text = "VSL" } dynamic = ["dependencies", "optional-dependencies", "version"] diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 74db773cf..51310bfa2 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 7, 3) +required_python_version = (3, 8, 0) if ( sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 4a84a1285..7bc3377c3 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -73,8 +73,9 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ) continue + data = layer.read(start, size, True) if not yarascan.YaraScan._yara_x: - for match in rules.match(data=layer.read(start, size, True)): + for match in rules.match(data=data): if yarascan.YaraScan.yara_returns_instances(): for match_string in match.strings: for instance in match_string.instances: @@ -95,9 +96,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): value, ) else: - data = layer.read(start, size, True) - results = rules.scan(data) - for match in results.matching_rules: + for match in rules.scan(data).matching_rules: for match_string in match.patterns: for instance in match_string.matches: yield 0, ( diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 6a4dd9251..310bbd072 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -36,7 +36,7 @@ except ImportError: raise -class BaseYaraScanner(interfaces.layers.ScannerInterface): +class YaraScanner(interfaces.layers.ScannerInterface): _version = (2, 1, 0) # yara.Rules isn't exposed, so we can't type this properly @@ -45,32 +45,44 @@ class BaseYaraScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules - - -class YaraPythonScanner(BaseYaraScanner): - def __init__(self, rules) -> None: - super().__init__(rules) - self.st_object = not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) + self.st_object = ( + None + if USE_YARA_X + else not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) + ) def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: - for match in self._rules.match(data=data): - if YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: + if USE_YARA_X: + for match in self._rules.scan(data).matching_rules: + for match_string in match.patterns: + for instance in match_string.matches: yield ( instance.offset + data_offset, - match.rule, + f"{match.namespace}.{match.identifier}", match_string.identifier, - instance.matched_data, + data[instance.offset : instance.offset + instance.length], ) - else: - for offset, name, value in match.strings: - yield (offset + data_offset, match.rule, name, value) + else: + for match in self._rules.match(data=data): + if YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield ( + instance.offset + data_offset, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield (offset + data_offset, match.rule, name, value) @staticmethod def get_rule(rule): + if USE_YARA_X: + return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") return yara.compile( sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} ) @@ -78,47 +90,18 @@ class YaraPythonScanner(BaseYaraScanner): @staticmethod def from_compiled_file(filepath): with resources.ResourceAccessor().open(filepath, "rb") as fp: + if USE_YARA_X: + return yara_x.Rules.deserialize_from(file=fp) return yara.load(file=fp) @staticmethod def from_file(filepath): with resources.ResourceAccessor().open(filepath, "rb") as fp: + if USE_YARA_X: + return yara_x.compile(fp.read().decode()) return yara.compile(file=fp) -class YaraXScanner(BaseYaraScanner): - def __call__( - self, data: bytes, data_offset: int - ) -> Iterable[Tuple[int, str, str, bytes]]: - results = self._rules.scan(data) - for match in results.matching_rules: - for match_string in match.patterns: - for instance in match_string.matches: - yield ( - instance.offset + data_offset, - f"{match.namespace}.{match.identifier}", - match_string.identifier, - data[instance.offset : instance.offset + instance.length], - ) - - @staticmethod - def get_rule(rule): - return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") - - @staticmethod - def from_compiled_file(filepath): - with resources.ResourceAccessor().open(filepath, "rb") as fp: - return yara_x.Rules.deserialize_from(file=fp) - - @staticmethod - def from_file(filepath): - with resources.ResourceAccessor().open(filepath, "rb") as fp: - return yara_x.compile(fp.read().decode()) - - -YaraScanner = YaraXScanner if USE_YARA_X else YaraPythonScanner - - class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" From de6637c9871ac16f1a5278c4831dfe60778852b9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 00:52:34 -0700 Subject: [PATCH 195/263] PR review fixes: ebpf plugin code improvement. Use the object_from_symbol() instead --- volatility3/framework/plugins/linux/ebpf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 8df506b06..9d41c0ffb 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -43,8 +43,7 @@ class EBPF(plugins.PluginInterface): "Cannot find the eBPF prog idr. Unsupported kernel" ) - prog_idr_addr = vmlinux.get_symbol("prog_idr").address - prog_idr = vmlinux.object("idr", offset=prog_idr_addr) + prog_idr = vmlinux.object_from_symbol("prog_idr") for page_addr in prog_idr.get_page_addresses(): bpf_prog = vmlinux.object("bpf_prog", offset=page_addr, absolute=True) yield bpf_prog From ee10ba8abb7b8c932d9af85a37edefc3fd03fa63 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 01:46:53 -0700 Subject: [PATCH 196/263] PR review fixes: Fix IDR explicit returns mixed with implicit returns and improve and fix code. --- .../symbols/linux/extensions/__init__.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 900cc9b6c..6eafd4173 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2134,7 +2134,7 @@ class IDR(objects.StructType): """Finds an ID within the IDR data structure. Based on idr_find_slowpath(), 3.9 <= Kernel < 4.11 Args: - idr_id: The IDR element ID + idr_id: The IDR lookup ID Returns: A pointer to the given ID element @@ -2144,28 +2144,28 @@ class IDR(objects.StructType): vollog.info( "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" ) - return + return None if idr_id < 0: - return + return None - cur_layer = self.top - if not cur_layer: - return + idr_layer = self.top + if not idr_layer: + return None - n = (cur_layer.layer + 1) * self.IDR_BITS + n = (idr_layer.layer + 1) * self.IDR_BITS - if idr_id > self.idr_max(cur_layer.layer + 1): - return + if idr_id > self.idr_max(idr_layer.layer + 1): + return None assert n != 0 - while n > 0 and cur_layer: + while n > 0 and idr_layer: n -= self.IDR_BITS - assert n == cur_layer.layer * self.IDR_BITS - cur_layer = cur_layer.ary[(idr_id >> n) & self.IDR_MASK] + assert n == idr_layer.layer * self.IDR_BITS + idr_layer = idr_layer.ary[(idr_id >> n) & self.IDR_MASK] - return cur_layer.v() + return idr_layer def _old_kernel_get_page_addresses(self, in_use) -> int: # Kernels < 4.11 From 0f3f33863370f68b7da3069db5b29282dabab932 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 01:52:57 -0700 Subject: [PATCH 197/263] PR review fixes: Since the IDR, XArray and RadixTree can store any value, it renames the function names to a more generic name --- volatility3/framework/plugins/linux/ebpf.py | 2 +- .../framework/plugins/linux/pidhashtable.py | 2 +- volatility3/framework/symbols/linux/__init__.py | 4 ++-- .../symbols/linux/extensions/__init__.py | 16 +++++++++------- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 9d41c0ffb..70082daf5 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -44,7 +44,7 @@ class EBPF(plugins.PluginInterface): ) prog_idr = vmlinux.object_from_symbol("prog_idr") - for page_addr in prog_idr.get_page_addresses(): + for page_addr in prog_idr.get_entries(): bpf_prog = vmlinux.object("bpf_prog", offset=page_addr, absolute=True) yield bpf_prog diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 3c429dc2f..c384cb5cd 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -175,7 +175,7 @@ class PIDHashTable(plugins.PluginInterface): ns_addr = self.vmlinux.get_symbol("init_pid_ns").address ns = self.vmlinux.object("pid_namespace", offset=ns_addr) - for page_addr in ns.idr.get_page_addresses(): + for page_addr in ns.idr.get_entries(): task = self._task_for_radix_pid_node(page_addr) if self._is_valid_task(task): yield task diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 96bc56a18..89f970275 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -572,7 +572,7 @@ class Tree(ABC): for child_node in self._iter_node(nodep, height - 1): yield child_node - def get_page_addresses(self, root: interfaces.objects.ObjectInterface) -> int: + def get_entries(self, root: interfaces.objects.ObjectInterface) -> int: """Walks the tree data structure Args: @@ -762,7 +762,7 @@ class PageCache(object): Page objects """ - for page_addr in self._tree.get_page_addresses(self._page_cache.i_pages): + for page_addr in self._tree.get_entries(self._page_cache.i_pages): if not page_addr: continue diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6eafd4173..29af5c510 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2167,7 +2167,7 @@ class IDR(objects.StructType): return idr_layer - def _old_kernel_get_page_addresses(self, in_use) -> int: + def _old_kernel_get_entries(self) -> int: # Kernels < 4.11 total = next_id = 0 while total < in_use: @@ -2178,14 +2178,14 @@ class IDR(objects.StructType): next_id += 1 - def _new_kernel_get_page_addresses(self, _in_use) -> int: + def _new_kernel_get_entries(self) -> int: # Kernels >= 4.11 vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) tree = linux.LinuxUtilities.choose_kernel_tree(vmlinux) - for page_addr in tree.get_page_addresses(root=self.idr_rt): + for page_addr in tree.get_entries(root=self.idr_rt): yield page_addr - def get_page_addresses(self, in_use=0) -> int: + def get_entries(self) -> int: """Walks the IDR and yield a pointer associated with each element. Args: @@ -2195,9 +2195,11 @@ class IDR(objects.StructType): A pointer associated with each element. """ if self.has_member("idr_rt"): - get_page_addresses_func = self._new_kernel_get_page_addresses + # Kernels >= 4.11 + get_entries_func = self._new_kernel_get_entries else: - get_page_addresses_func = self._old_kernel_get_page_addresses + # Kernels < 4.11 + get_entries_func = self._old_kernel_get_entries - for page_addr in get_page_addresses_func(in_use): + for page_addr in get_entries_func(): yield page_addr From 7a8dea3356c709cda2bdd20b7cb3ff97f1f7987d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 01:57:35 -0700 Subject: [PATCH 198/263] PR review fixes: Code scanning complains about these unused variables. Let's comment them and adapt the FIXME message --- volatility3/framework/symbols/linux/extensions/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 29af5c510..9b03235cb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2063,10 +2063,9 @@ class page(objects.StructType): vmemmap_start = vmemmap_base_l4 else: # 5-Level paging -> VMEMMAP_START = __VMEMMAP_BASE_L5 - vmemmap_base_l5 = 0xFFD4000000000000 - vmemmap_start = vmemmap_base_l5 - - # FIXME: Remove this exception once 5-level paging is supported. + # FIXME: Once 5-level paging is supported, uncomment the following lines and remove the exception + # vmemmap_base_l5 = 0xFFD4000000000000 + # vmemmap_start = vmemmap_base_l5 raise exceptions.VolatilityException( "5-level paging is not yet supported" ) From 06508a4afba235812e7d1b7bfb79467adf2ed6fc Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 02:00:27 -0700 Subject: [PATCH 199/263] PR review fixes: Fix the IDR's old kernel get_entries --- .../framework/symbols/linux/extensions/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9b03235cb..f7b0df6be 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2168,11 +2168,12 @@ class IDR(objects.StructType): def _old_kernel_get_entries(self) -> int: # Kernels < 4.11 + cur = self.cur total = next_id = 0 - while total < in_use: - page_addr = self.idr_find(next_id) - if page_addr: - yield page_addr + while next_id < cur: + entry = self.idr_find(next_id) + if entry: + yield entry total += 1 next_id += 1 From 2c85ea525e15c0cb745f3b806836f22ea44a4b4c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 02:03:28 -0700 Subject: [PATCH 200/263] PR review fixes: Fix page extension object get_content() explicit returns mixed with implicit returns. --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f7b0df6be..be4df6a13 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2102,7 +2102,7 @@ class page(objects.StructType): physical_layer = vmlinux.context.layers["memory_layer"] page_paddr = self.to_paddr() if not page_paddr: - return + return None page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) return page_data From f22575669a6ccd9afaeef126e81a6adad8b880f6 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 9 Aug 2024 10:20:28 +0200 Subject: [PATCH 201/263] Modifications following the review --- volatility3/framework/plugins/linux/lsof.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 167556e7d..9a0fd7417 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -66,7 +66,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ) except (exceptions.InvalidAddressException, AttributeError) as e: vollog.warning(f"Can't get inode metadata: {e}") - return tuple(renderers.NotAvailableValue() for _ in range(7)) + return None @classmethod def list_fds( @@ -103,6 +103,10 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): context, symbol_table, filter_func ): inode_metadata = cls.get_inode_metadata(filp) + if inode_metadata is None: + inode_metadata = tuple( + interfaces.renderers.BaseAbsentValue() for _ in range(7) + ) yield pid, task_comm, task, fd_num, filp, full_path, inode_metadata def _generator(self, pids, symbol_table): From 53f3d12341e722f1058c42436adfea600af94bab Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 9 Aug 2024 23:35:41 -0700 Subject: [PATCH 202/263] linuxutilities code improvement. Remove code duplication --- volatility3/framework/symbols/linux/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 89f970275..90f5cc8a2 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -419,9 +419,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): Returns: A kernel object (vmlinux) """ - symbol_table_arr = volobj.vol.type_name.split("!", 1) - symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None - + symbol_table = volobj.get_symbol_table_name() module_names = context.modules.get_modules_by_symbol_tables(symbol_table) module_names = list(module_names) From 17861618df3744d572955b50b2b1b1ad1d0961e5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 10 Aug 2024 00:13:38 -0700 Subject: [PATCH 203/263] PR review fixes: Rename Tree to IDStorage. Move choose_id_storage() form LinuxUtilities to IDStorage. Use context and kernel_module_name instead of vmlinux --- .../framework/symbols/linux/__init__.py | 87 +++++++++++-------- .../symbols/linux/extensions/__init__.py | 14 +-- 2 files changed, 59 insertions(+), 42 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 90f5cc8a2..632ac2f6b 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -431,17 +431,51 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return kernel + +class IDStorage(ABC): + """Abstraction to support both XArray and RadixTree""" + + # Dynamic values, these will be initialized later + CHUNK_SHIFT = None + CHUNK_SIZE = None + CHUNK_MASK = None + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.vmlinux = context.modules[kernel_module_name] + self.vmlinux_layer = self.vmlinux.context.layers[self.vmlinux.layer_name] + + self.pointer_size = self.vmlinux.get_type("pointer").size + # Dynamically work out the (XA_CHUNK|RADIX_TREE_MAP)_SHIFT values based on + # the node.slots[] array size + node_type = self.vmlinux.get_type(self.node_type_name) + slots_array_size = node_type.child_template("slots").count + + # Calculate the LSB index - 1 + self.CHUNK_SHIFT = slots_array_size.bit_length() - 1 + self.CHUNK_SIZE = 1 << self.CHUNK_SHIFT + self.CHUNK_MASK = self.CHUNK_SIZE - 1 + @classmethod - def choose_kernel_tree(cls, vmlinux: interfaces.context.ModuleInterface) -> "Tree": - """Returns the appropriate tree data structure instance for the current kernel implementation. + def choose_id_storage( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ) -> "IDStorage": + """Returns the appropriate ID storage data structure instance for the current kernel implementation. This is used by the IDR and the PageCache to choose between the XArray and RadixTree. Args: - vmlinux: The kernel module object + context: The context to retrieve required elements (layers, symbol tables) from + kernel_module_name: The name of the kernel module on which to operate Returns: - The appropriate Tree instance for the current kernel + The appropriate ID storage instance for the current kernel """ + vmlinux = context.modules[kernel_module_name] address_space_type = vmlinux.get_type("address_space") address_space_has_i_pages = address_space_type.has_member("i_pages") i_pages_type_name = ( @@ -455,33 +489,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ) and vmlinux.get_type("radix_tree_root").has_member("xa_head") if i_pages_is_xarray or i_pages_is_radix_tree_root: - return XArray(vmlinux) + return XArray(context, kernel_module_name) else: - return RadixTree(vmlinux) - - -class Tree(ABC): - """Abstraction to support both XArray and RadixTree""" - - # Dynamic values, these will be initialized later - CHUNK_SHIFT = None - CHUNK_SIZE = None - CHUNK_MASK = None - - def __init__(self, vmlinux: interfaces.context.ModuleInterface): - self.vmlinux = vmlinux - self.vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - - self.pointer_size = self.vmlinux.get_type("pointer").size - # Dynamically work out the (XA_CHUNK|RADIX_TREE_MAP)_SHIFT values based on - # the node.slots[] array size - node_type = self.vmlinux.get_type(self.node_type_name) - slots_array_size = node_type.child_template("slots").count - - # Calculate the LSB index - 1 - self.CHUNK_SHIFT = slots_array_size.bit_length() - 1 - self.CHUNK_SIZE = 1 << self.CHUNK_SHIFT - self.CHUNK_MASK = self.CHUNK_SIZE - 1 + return RadixTree(context, kernel_module_name) @property @abstractmethod @@ -601,7 +611,7 @@ class Tree(ABC): yield child_node -class XArray(Tree): +class XArray(IDStorage): XARRAY_TAG_MASK = 3 XARRAY_TAG_INTERNAL = 2 @@ -637,7 +647,7 @@ class XArray(Tree): return not self.is_node_tagged(nodep) -class RadixTree(Tree): +class RadixTree(IDStorage): RADIX_TREE_INTERNAL_NODE = 1 RADIX_TREE_EXCEPTIONAL_ENTRY = 2 RADIX_TREE_ENTRY_MASK = 3 @@ -741,17 +751,20 @@ class PageCache(object): def __init__( self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, page_cache: interfaces.objects.ObjectInterface, - vmlinux: interfaces.context.ModuleInterface, ): """ Args: + context: interfaces.context.ContextInterface, + kernel_module_name: The name of the kernel module on which to operate page_cache: Page cache address space - vmlinux: Kernel module object """ - self.vmlinux = vmlinux + self.vmlinux = context.modules[kernel_module_name] + self._page_cache = page_cache - self._tree = LinuxUtilities.choose_kernel_tree(self.vmlinux) + self._idstorage = IDStorage.choose_id_storage(context, kernel_module_name) def get_cached_pages(self) -> interfaces.objects.ObjectInterface: """Returns all page cache contents @@ -760,7 +773,7 @@ class PageCache(object): Page objects """ - for page_addr in self._tree.get_entries(self._page_cache.i_pages): + for page_addr in self._idstorage.get_entries(self._page_cache.i_pages): if not page_addr: continue diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index be4df6a13..d00af7a3f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1972,8 +1972,11 @@ class inode(objects.StructType): elif not (self.i_mapping and self.i_mapping.nrpages > 0): return - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - page_cache = linux.PageCache(self.i_mapping.dereference(), vmlinux) + page_cache = linux.PageCache( + context=self._context, + kernel_module_name="kernel", + page_cache=self.i_mapping.dereference(), + ) yield from page_cache.get_cached_pages() def get_contents(self): @@ -2180,9 +2183,10 @@ class IDR(objects.StructType): def _new_kernel_get_entries(self) -> int: # Kernels >= 4.11 - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - tree = linux.LinuxUtilities.choose_kernel_tree(vmlinux) - for page_addr in tree.get_entries(root=self.idr_rt): + id_storage = linux.IDStorage.choose_id_storage( + self._context, kernel_module_name="kernel" + ) + for page_addr in id_storage.get_entries(root=self.idr_rt): yield page_addr def get_entries(self) -> int: From 8f9d565f6300750ad30c0aaca0aa01cfbcb1a417 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 10 Aug 2024 00:49:18 -0700 Subject: [PATCH 204/263] PR review fixes: Fix minor typo to match verb form from other docstrings --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index fd62cc1e4..f062655a6 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -159,7 +159,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): root_dentry: interfaces.objects.ObjectInterface, parent_dir: str, ): - """Walk dentries recursively + """Walks dentries recursively Args: seen_dentries: A set to ensure each dentry is processed only once From f804b44ff6f8451e8f1091383b8d9c2300f7bfbb Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Aug 2024 01:00:57 -0700 Subject: [PATCH 205/263] Fix test.yaml, it should remove *.bin and not *.lime. There is no *.lime atm. --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 55b2e4b60..668b814ce 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,7 +46,7 @@ jobs: - name: Clean up post-test run: | - rm -rf *.lime + rm -rf *.bin rm -rf *.img cd volatility3/symbols rm -rf linux From 7efa2210a550d55f356b6ebb35cafcdfd4a58d1f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Aug 2024 01:05:44 -0700 Subject: [PATCH 206/263] PR review fixes: Adjust LinuxUtilities version since we moved choose_id_storage() back to the IDStorage class. --- volatility3/framework/plugins/linux/pidhashtable.py | 2 +- volatility3/framework/symbols/linux/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index c384cb5cd..9a2528543 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -34,7 +34,7 @@ class PIDHashTable(plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.VersionRequirement( - name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), requirements.BooleanRequirement( name="decorate_comm", diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 632ac2f6b..91abf7db4 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -74,7 +74,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 2, 0) + _version = (2, 1, 1) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 918584537fafd67b2367e283871889bf291b2951 Mon Sep 17 00:00:00 2001 From: Steven Luke <97394870+sluke-nuix@users.noreply.github.com> Date: Tue, 13 Aug 2024 08:34:35 -0400 Subject: [PATCH 207/263] Update the modules.Modules version requirement. This is a response to PR [#1173](https://github.com/volatilityfoundation/volatility3/pull/1173) --- volatility3/framework/plugins/windows/truecrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index 81250a749..7fd26cb4e 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -33,7 +33,7 @@ class Passphrase(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name="modules", component=modules.Modules, version=(1, 1, 0) + name="modules", component=modules.Modules, version=(2, 0, 0) ), requirements.IntRequirement( name="min-length", From 8d7edfdca6fa84983b4ee734e3f45a9d07c28ff1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Aug 2024 20:36:09 +0100 Subject: [PATCH 208/263] Bump as the release branch has been cut --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index f219fb0af..4df0b9041 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 8 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From 71cdca5883b234680773e000a663750964d4860e Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Thu, 22 Aug 2024 16:45:04 +0200 Subject: [PATCH 209/263] Updating version + docstring --- volatility3/framework/plugins/linux/lsof.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 9a0fd7417..360f89749 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -18,10 +18,10 @@ vollog = logging.getLogger(__name__) class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): - """Lists all memory maps for all processes.""" + """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (1, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From feae6a9aa0f5c7869174e0b906ec83e61b3e14e3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 00:42:52 +1000 Subject: [PATCH 210/263] PR review fixes: Use plugin's open method instead of the builtin open() --- volatility3/framework/plugins/linux/pagecache.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index f062655a6..cf4151c85 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set +from typing import List, Set, Type from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints @@ -408,6 +408,7 @@ class InodePages(plugins.PluginInterface): def write_inode_content_to_file( inode: interfaces.objects.ObjectInterface, filename: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a file @@ -415,6 +416,7 @@ class InodePages(plugins.PluginInterface): Args: inode: The inode to dump filename: Filename for writing the inode content + open_method: class for constructing output files vmlinux_layer: The kernel layer to obtain the page size """ if not inode.is_reg: @@ -426,7 +428,7 @@ class InodePages(plugins.PluginInterface): # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. try: - with open(filename, "wb") as f: + with open_method(filename) as f: inode_size = inode.i_size f.truncate(inode_size) @@ -502,7 +504,7 @@ class InodePages(plugins.PluginInterface): if self.config["dump"]: filename = self.config["dump"] vollog.info("[*] Writing inode at 0x%x to '%s'", inode.vol.offset, filename) - self.write_inode_content_to_file(inode, filename, vmlinux_layer) + self.write_inode_content_to_file(inode, filename, self.open, vmlinux_layer) def run(self): headers = [ From 3c70c1b9f7c2251e35eea276f5d5099b68392a48 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 00:58:04 +1000 Subject: [PATCH 211/263] PR review fixes: Use context and module_name instead of vmlinux in ebpf plugin --- volatility3/framework/plugins/linux/ebpf.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 70082daf5..2267dd922 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -29,15 +29,21 @@ class EBPF(plugins.PluginInterface): ), ] - def get_ebpf_programs(self, vmlinux) -> interfaces.objects.ObjectInterface: + def get_ebpf_programs( + self, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> interfaces.objects.ObjectInterface: """Enumerate eBPF programs walking its IDR. Args: - vmlinux: The kernel symbols object - + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate Yields: eBPF program objects """ + vmlinux = context.modules[vmlinux_module_name] + if not vmlinux.has_symbol("prog_idr"): raise exceptions.VolatilityException( "Cannot find the eBPF prog idr. Unsupported kernel" @@ -49,8 +55,7 @@ class EBPF(plugins.PluginInterface): yield bpf_prog def _generator(self): - vmlinux = self.context.modules[self.config["kernel"]] - for prog in self.get_ebpf_programs(vmlinux): + for prog in self.get_ebpf_programs(self.context, self.config["kernel"]): prog_addr = prog.vol.offset prog_type = prog.get_type() or renderers.NotAvailableValue() prog_tag = prog.get_tag() or renderers.NotAvailableValue() From 695635199e2bd3af11bee522e054719ee1898db6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 01:14:09 +1000 Subject: [PATCH 212/263] PR review fixes: Avoid saving state in the pidhashtable plugin --- .../framework/plugins/linux/pidhashtable.py | 79 +++++++++++-------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 9a2528543..3223aed4a 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -44,17 +44,16 @@ class PIDHashTable(plugins.PluginInterface): ), ] - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.vmlinux = None - self.vmlinux_layer = None - def _is_valid_task(self, task) -> bool: - return bool(task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent)) + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + return bool(task and task.pid > 0 and vmlinux_layer.is_valid(task.parent)) def _get_pidtype_pid(self): + vmlinux = self.context.modules[self.config["kernel"]] + # The pid_type enumeration is present since 2.5.37, just in case - pid_type_enum = self.vmlinux.get_enumeration("pid_type") + pid_type_enum = vmlinux.get_enumeration("pid_type") if not pid_type_enum: vollog.error("Cannot find pid_type enum. Unsupported kernel") return None @@ -68,38 +67,46 @@ class PIDHashTable(plugins.PluginInterface): return pidtype_pid def _get_pidhash_array(self): - pidhash_shift = self.vmlinux.object_from_symbol("pidhash_shift") + vmlinux = self.context.modules[self.config["kernel"]] + + pidhash_shift = vmlinux.object_from_symbol("pidhash_shift") pidhash_size = 1 << pidhash_shift - array_type_name = self.vmlinux.symbol_table_name + constants.BANG + "array" + array_type_name = vmlinux.symbol_table_name + constants.BANG + "array" - pidhash_ptr = self.vmlinux.object_from_symbol("pid_hash") + pidhash_ptr = vmlinux.object_from_symbol("pid_hash") # pidhash is an array of hlist_heads pidhash = self._context.object( array_type_name, offset=pidhash_ptr, - subtype=self.vmlinux.get_type("hlist_head"), + subtype=vmlinux.get_type("hlist_head"), count=pidhash_size, - layer_name=self.vmlinux.layer_name, + layer_name=vmlinux.layer_name, ) return pidhash def _walk_upid(self, seen_upids, upid): - while upid and self.vmlinux_layer.is_valid(upid.vol.offset): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + while upid and vmlinux_layer.is_valid(upid.vol.offset): if upid.vol.offset in seen_upids: break seen_upids.add(upid.vol.offset) pid_chain = upid.pid_chain - if not (pid_chain and self.vmlinux_layer.is_valid(pid_chain.vol.offset)): + if not (pid_chain and vmlinux_layer.is_valid(pid_chain.vol.offset)): break upid = linux.LinuxUtilities.container_of( - pid_chain.next, "upid", "pid_chain", self.vmlinux + pid_chain.next, "upid", "pid_chain", vmlinux ) def _get_upids(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + # 2.6.24 <= kernels < 4.15 pidhash = self._get_pidhash_array() @@ -108,10 +115,10 @@ class PIDHashTable(plugins.PluginInterface): # each entry in the hlist is a upid which is wrapped in a pid ent = hlist.first - while ent and self.vmlinux_layer.is_valid(ent.vol.offset): + while ent and vmlinux_layer.is_valid(ent.vol.offset): # upid->pid_chain exists 2.6.24 <= kernel < 4.15 upid = linux.LinuxUtilities.container_of( - ent.vol.offset, "upid", "pid_chain", self.vmlinux + ent.vol.offset, "upid", "pid_chain", vmlinux ) if upid.vol.offset in seen_upids: @@ -124,16 +131,14 @@ class PIDHashTable(plugins.PluginInterface): return seen_upids def _pid_hash_implementation(self): + vmlinux = self.context.modules[self.config["kernel"]] + # 2.6.24 <= kernels < 4.15 - task_pids_off = self.vmlinux.get_type("task_struct").relative_child_offset( - "pids" - ) + task_pids_off = vmlinux.get_type("task_struct").relative_child_offset("pids") pidtype_pid = self._get_pidtype_pid() for upid in self._get_upids(): - pid = linux.LinuxUtilities.container_of( - upid, "pid", "numbers", self.vmlinux - ) + pid = linux.LinuxUtilities.container_of(upid, "pid", "numbers", vmlinux) if not pid: continue @@ -141,22 +146,24 @@ class PIDHashTable(plugins.PluginInterface): if not pid_tasks_0: continue - task = self.vmlinux.object( + task = vmlinux.object( "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True ) if self._is_valid_task(task): yield task def _task_for_radix_pid_node(self, nodep): + vmlinux = self.context.modules[self.config["kernel"]] + # kernels >= 4.15 - pid = self.vmlinux.object("pid", offset=nodep, absolute=True) + pid = vmlinux.object("pid", offset=nodep, absolute=True) pidtype_pid = self._get_pidtype_pid() pid_tasks_0 = pid.tasks[pidtype_pid].first if not pid_tasks_0: return None - task_struct_type = self.vmlinux.get_type("task_struct") + task_struct_type = vmlinux.get_type("task_struct") if task_struct_type.has_member("pids"): member = "pids" elif task_struct_type.has_member("pid_links"): @@ -165,15 +172,17 @@ class PIDHashTable(plugins.PluginInterface): return None task_pids_off = task_struct_type.relative_child_offset(member) - task = self.vmlinux.object( + task = vmlinux.object( "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True ) return task def _pid_namespace_idr(self): + vmlinux = self.context.modules[self.config["kernel"]] + # kernels >= 4.15 - ns_addr = self.vmlinux.get_symbol("init_pid_ns").address - ns = self.vmlinux.object("pid_namespace", offset=ns_addr) + ns_addr = vmlinux.get_symbol("init_pid_ns").address + ns = vmlinux.object("pid_namespace", offset=ns_addr) for page_addr in ns.idr.get_entries(): task = self._task_for_radix_pid_node(page_addr) @@ -181,24 +190,26 @@ class PIDHashTable(plugins.PluginInterface): yield task def _determine_pid_func(self): - pid_hash = self.vmlinux.has_symbol("pid_hash") and self.vmlinux.has_symbol( + vmlinux = self.context.modules[self.config["kernel"]] + + pid_hash = vmlinux.has_symbol("pid_hash") and vmlinux.has_symbol( "pidhash_shift" ) # 2.5.55 <= kernels < 4.15 - has_pid_numbers = self.vmlinux.has_type("pid") and self.vmlinux.get_type( + has_pid_numbers = vmlinux.has_type("pid") and vmlinux.get_type( "pid" ).has_member( "numbers" ) # kernels >= 2.6.24 - has_pid_chain = self.vmlinux.has_type("upid") and self.vmlinux.get_type( + has_pid_chain = vmlinux.has_type("upid") and vmlinux.get_type( "upid" ).has_member( "pid_chain" ) # 2.6.24 <= kernels < 4.15 # kernels >= 4.15 - pid_idr = self.vmlinux.has_type("pid_namespace") and self.vmlinux.get_type( + pid_idr = vmlinux.has_type("pid_namespace") and vmlinux.get_type( "pid_namespace" ).has_member("idr") @@ -217,8 +228,6 @@ class PIDHashTable(plugins.PluginInterface): Yields: task_struct objects """ - self.vmlinux = self.context.modules[self.config["kernel"]] - self.vmlinux_layer = self.context.layers[self.vmlinux.layer_name] pid_func = self._determine_pid_func() if not pid_func: vollog.error("Cannot determine which PID hash table this kernel is using") From d964e6f61de00b5a8608d48d468d9a59b7923ec7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 02:06:06 +1000 Subject: [PATCH 213/263] PR review fixes: Check for LinuxUtilities version everywhere we use it --- .../symbols/linux/extensions/__init__.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d00af7a3f..51dc37d31 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,6 +13,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion +from volatility3.framework.configuration import requirements from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1608,6 +1609,19 @@ class xdp_sock(objects.StructType): class bpf_prog(objects.StructType): + def _get_vmlinux(self): + linuxutils_required_version = (2, 1, 1) + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + return vmlinux + def get_type(self) -> Union[str, None]: """Returns a string with the eBPF program type""" @@ -1631,7 +1645,7 @@ class bpf_prog(objects.StructType): if not self.has_member("tag"): return None - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] prog_tag_addr = self.tag.vol.offset @@ -2043,13 +2057,26 @@ class page(objects.StructType): return flags + def _get_vmlinux(self): + linuxutils_required_version = (2, 1, 1) + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + return vmlinux + def to_paddr(self) -> int: """Converts a page's virtual address to its physical address using the current physical memory model. Returns: int: page physical address """ - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] vmemmap_start = None @@ -2100,7 +2127,7 @@ class page(objects.StructType): Returns: The page content """ - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] physical_layer = vmlinux.context.layers["memory_layer"] page_paddr = self.to_paddr() @@ -2118,6 +2145,19 @@ class IDR(objects.StructType): MAX_IDR_SHIFT = INT_SIZE * 8 - 1 MAX_IDR_BIT = 1 << MAX_IDR_SHIFT + def _get_vmlinux(self): + linuxutils_required_version = (2, 1, 1) + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + return vmlinux + def idr_max(self, num_layers: int) -> int: """Returns the maximum ID which can be allocated given idr::layers @@ -2141,7 +2181,7 @@ class IDR(objects.StructType): Returns: A pointer to the given ID element """ - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() if not vmlinux.get_type("idr_layer").has_member("layer"): vollog.info( "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" From d627f243e259efce2f730860f47d9116b3c78f3d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 10:55:01 +1000 Subject: [PATCH 214/263] PR review fixes: Add typing info to the get_inodes() class method. --- volatility3/framework/plugins/linux/pagecache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index cf4151c85..e384cbabf 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type +from typing import List, Set, Type, Iterable from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints @@ -205,7 +205,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, config_path: str, - ): + ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: From 90b327e63253404a98cc5a79e3cecaa1b773048c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 11:45:29 +1000 Subject: [PATCH 215/263] PR review fixes: Make mountinfo.get_superblocks() a classmethod and adapt the code using it. --- .../framework/plugins/linux/mountinfo.py | 19 +++++++++--- .../framework/plugins/linux/pagecache.py | 31 ++++++++++++------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index dfb2e2f52..1eaec77bf 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -37,7 +37,7 @@ class MountInfo(plugins.PluginInterface): _required_framework_version = (2, 2, 0) - _version = (1, 1, 0) + _version = (1, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -143,8 +143,8 @@ class MountInfo(plugins.PluginInterface): sb_opts, ) + @staticmethod def _get_tasks_mountpoints( - self, tasks: Iterable[interfaces.objects.ObjectInterface], filtered_by_pids: bool = False, ): @@ -247,17 +247,26 @@ class MountInfo(plugins.PluginInterface): "Could not filter by mount namespace id. This field is not available in this kernel." ) - def get_superblocks(self): + @classmethod + def get_superblocks( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Yield file system superblocks based on the task's mounted filesystems. + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + Yields: super_block: Kernel's struct super_block object """ # No filter so that we get all the mount namespaces from all tasks - tasks = pslist.PsList.list_tasks(self.context, self.config["kernel"]) + tasks = pslist.PsList.list_tasks(context, vmlinux_module_name) seen_sb_ptr = set() - for task, mnt, _mnt_ns_id in self._get_tasks_mountpoints(tasks): + for task, mnt, _mnt_ns_id in cls._get_tasks_mountpoints(tasks): path_root = linux.LinuxUtilities.get_path_mnt(task, mnt) if not path_root: continue diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index e384cbabf..e54891480 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -115,7 +115,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 1, 0) + name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 2, 0) ), requirements.ListRequirement( name="type", @@ -204,22 +204,22 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): def get_inodes( cls, context: interfaces.context.ContextInterface, - config_path: str, + vmlinux_module_name: str, ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: context: The context that the plugin will operate within - config_path: The path to configuration data within the context configuration data + vmlinux_module_name: The name of the kernel module on which to operate Yields: An InodeInternal object """ - superblocks_iter = mountinfo.MountInfo( + superblocks_iter = mountinfo.MountInfo.get_superblocks( context=context, - config_path=config_path, - ).get_superblocks() + vmlinux_module_name=vmlinux_module_name, + ) seen_inodes = set() seen_dentries = set() @@ -289,11 +289,13 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): yield inode_in def _generator(self): - vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] vmlinux_layer = self.context.layers[vmlinux.layer_name] inodes_iter = self.get_inodes( - context=self.context, config_path=self.config_path + context=self.context, + vmlinux_module_name=vmlinux_module_name, ) types_filter = self.config["type"] @@ -316,12 +318,15 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): These need not be generated in any particular order, sorting will be done later """ - vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] vmlinux_layer = self.context.layers[vmlinux.layer_name] inodes_iter = self.get_inodes( - context=self.context, config_path=self.config_path + context=self.context, + vmlinux_module_name=vmlinux_module_name, ) + for inode_in in inodes_iter: inode_out = inode_in.to_user(vmlinux_layer) description = f"Cached Inode for {inode_out.path}" @@ -450,7 +455,8 @@ class InodePages(plugins.PluginInterface): vollog.error("Unable to write to file (%s): %s", filename, e) def _generator(self): - vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] vmlinux_layer = self.context.layers[vmlinux.layer_name] if self.config["inode"] and self.config["find"]: @@ -459,7 +465,8 @@ class InodePages(plugins.PluginInterface): if self.config["find"]: inodes_iter = Files.get_inodes( - context=self.context, config_path=self.config_path + context=self.context, + vmlinux_module_name=vmlinux_module_name, ) for inode_in in inodes_iter: if inode_in.path == self.config["find"]: From 3bf9f8cec0e1c4c75088abe17baadd0bc6d4c3d6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 12:03:10 +1000 Subject: [PATCH 216/263] PR review fixes: Add typing info to pagecache.Files._follow_symlink() --- volatility3/framework/plugins/linux/pagecache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index e54891480..b6c5f7cc0 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -131,7 +131,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): ] @staticmethod - def _follow_symlink(inode, symlink_path) -> str: + def _follow_symlink( + inode: interfaces.objects.ObjectInterface, + symlink_path: str, + ) -> str: """Follows (fast) symlinks (kernels >= 4.2.x). Fast symlinks are filesystem agnostic. From e97abae156721bc1c625729166c08325c164bd3a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 24 Aug 2024 15:02:31 +0100 Subject: [PATCH 217/263] Remove get_vmlinux calls Sorry, I know I asked for it, but I hadn't quite figured out what was going on. Things in the symbols/linux code are considered part of the framework, and therefore it's the framework version that should have been bumped. Since the framework comes packages with LinuxUtilities we can rely on the version numbers to be suitable. This cleans up the mess I caused, sorry for the extra work! 5:S --- volatility3/framework/constants/_version.py | 4 +- .../symbols/linux/extensions/__init__.py | 47 ++----------------- 2 files changed, 6 insertions(+), 45 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 4df0b9041..d30446d63 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 8 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 9 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 51dc37d31..fdd34403a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1609,19 +1609,6 @@ class xdp_sock(objects.StructType): class bpf_prog(objects.StructType): - def _get_vmlinux(self): - linuxutils_required_version = (2, 1, 1) - linuxutils_current_version = linux.LinuxUtilities._version - if not requirements.VersionRequirement.matches_required( - linuxutils_required_version, linuxutils_current_version - ): - raise exceptions.PluginRequirementException( - f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" - ) - - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - return vmlinux - def get_type(self) -> Union[str, None]: """Returns a string with the eBPF program type""" @@ -1645,7 +1632,7 @@ class bpf_prog(objects.StructType): if not self.has_member("tag"): return None - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] prog_tag_addr = self.tag.vol.offset @@ -2057,26 +2044,13 @@ class page(objects.StructType): return flags - def _get_vmlinux(self): - linuxutils_required_version = (2, 1, 1) - linuxutils_current_version = linux.LinuxUtilities._version - if not requirements.VersionRequirement.matches_required( - linuxutils_required_version, linuxutils_current_version - ): - raise exceptions.PluginRequirementException( - f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" - ) - - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - return vmlinux - def to_paddr(self) -> int: """Converts a page's virtual address to its physical address using the current physical memory model. Returns: int: page physical address """ - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] vmemmap_start = None @@ -2127,7 +2101,7 @@ class page(objects.StructType): Returns: The page content """ - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] physical_layer = vmlinux.context.layers["memory_layer"] page_paddr = self.to_paddr() @@ -2145,19 +2119,6 @@ class IDR(objects.StructType): MAX_IDR_SHIFT = INT_SIZE * 8 - 1 MAX_IDR_BIT = 1 << MAX_IDR_SHIFT - def _get_vmlinux(self): - linuxutils_required_version = (2, 1, 1) - linuxutils_current_version = linux.LinuxUtilities._version - if not requirements.VersionRequirement.matches_required( - linuxutils_required_version, linuxutils_current_version - ): - raise exceptions.PluginRequirementException( - f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" - ) - - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - return vmlinux - def idr_max(self, num_layers: int) -> int: """Returns the maximum ID which can be allocated given idr::layers @@ -2181,7 +2142,7 @@ class IDR(objects.StructType): Returns: A pointer to the given ID element """ - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) if not vmlinux.get_type("idr_layer").has_member("layer"): vollog.info( "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" From 6a157a785f3364635de2c7a43eeab054ab7c6b3d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 27 Aug 2024 14:45:52 +0100 Subject: [PATCH 218/263] Bump the modules version number Pull-request #1173 bumped the version of the modules plugin (even though this only needed to be a MINOR version bump, see https://github.com/volatilityfoundation/volatility3/pull/1173#discussion_r1649614761), but failed to verify that other plugins which relied on it were also updated to make use of the new plugin. This was the version system working as intended, but highlighted a review failure that the neither the author, nor the reviewers, verified that the rest of the framework (specifically other plugins which relied on modules) worked correctly with the new code (which this kind of error is designed to fix). Fixes #1244. --- volatility3/framework/plugins/windows/ssdt.py | 2 +- volatility3/framework/plugins/windows/verinfo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/ssdt.py b/volatility3/framework/plugins/windows/ssdt.py index 6a47c36e9..1fcb6cc91 100644 --- a/volatility3/framework/plugins/windows/ssdt.py +++ b/volatility3/framework/plugins/windows/ssdt.py @@ -30,7 +30,7 @@ class SSDT(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="modules", plugin=modules.Modules, version=(1, 0, 0) + name="modules", plugin=modules.Modules, version=(2, 0, 0) ), ] diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 1c6615804..5b3c52bf6 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -46,7 +46,7 @@ class VerInfo(interfaces.plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="modules", plugin=modules.Modules, version=(1, 0, 0) + name="modules", plugin=modules.Modules, version=(2, 0, 0) ), requirements.VersionRequirement( name="dlllist", component=dlllist.DllList, version=(2, 0, 0) From a2196850bc01d5a1a41a2aa29685cdb1d2146e87 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Wed, 28 Aug 2024 10:39:28 +0200 Subject: [PATCH 219/263] increase python version to 3.8 --- .github/workflows/build-pypi.yml | 2 +- .github/workflows/install.yml | 2 +- .github/workflows/test.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 346d0337e..d1a63b4da 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.7"] + python-version: ["3.8"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index e13161085..398ff8ae3 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: host: [ ubuntu-latest, windows-latest ] - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + python-version: [ "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 55b2e4b60..65dfb13c8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.7"] + python-version: ["3.8"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 7a479d617f8107590533eaed9afe105c3cd3b34f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 28 Aug 2024 18:40:41 +0200 Subject: [PATCH 220/263] add OS and framework architectures constants --- .../framework/constants/architectures.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 volatility3/framework/constants/architectures.py diff --git a/volatility3/framework/constants/architectures.py b/volatility3/framework/constants/architectures.py new file mode 100644 index 000000000..a8822d7d3 --- /dev/null +++ b/volatility3/framework/constants/architectures.py @@ -0,0 +1,21 @@ +from volatility3.framework.layers import intel + +WIN_ARCHS = ["Intel32", "Intel64"] +"""Windows supported architectures""" +WIN_ARCHS_LAYERS = [intel.Intel] +"""Windows supported architectures layers""" + +LINUX_ARCHS = ["Intel32", "Intel64"] +"""Linux supported architectures""" +LINUX_ARCHS_LAYERS = [intel.Intel] +"""Linux supported architectures layers""" + +MAC_ARCHS = ["Intel32", "Intel64"] +"""Mac supported architectures""" +MAC_ARCHS_LAYERS = [intel.Intel] +"""Mac supported architectures layers""" + +FRAMEWORK_ARCHS = ["Intel32", "Intel64"] +"""Framework supported architectures""" +FRAMEWORK_ARCHS_LAYERS = [intel.Intel] +"""Framework supported architectures layers""" From 46a26c7dc5f5bc87d93027a6c005acbf85bc8716 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 12:17:16 -0500 Subject: [PATCH 221/263] Address feedback --- .../framework/plugins/windows/orphan_kernel_threads.py | 8 +++++--- volatility3/framework/plugins/windows/thrdscan.py | 3 +-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/orphan_kernel_threads.py b/volatility3/framework/plugins/windows/orphan_kernel_threads.py index 1cdb1dcdf..f4901dc8c 100644 --- a/volatility3/framework/plugins/windows/orphan_kernel_threads.py +++ b/volatility3/framework/plugins/windows/orphan_kernel_threads.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # @@ -48,8 +48,9 @@ class Threads(thrdscan.ThrdScan): """Yields thread objects of kernel threads that do not map to a module Args: - kernel - + cls + context: the context to operate upon + module_name: name of the module to use for scanning Returns: A generator of thread objects of orphaned threads """ @@ -61,6 +62,7 @@ class Threads(thrdscan.ThrdScan): context, layer_name, symbol_table ) + # FIXME - use a proper constant once established # used to filter out smeared pointers if symbols.symbol_table_is_64bit(context, symbol_table): kernel_start = 0xFFFFF80000000000 diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index ad885e8e9..b812a15ff 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -48,8 +48,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) Args: context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - symbol_table: The name of the table containing the kernel symbols + module_name: Name of the module to use for scanning Returns: A list of _ETHREAD objects found by scanning memory for the "Thre" / "Thr\\xE5" pool signatures From 483cb7e4eebbb6b536cdec883caa3f75fa182ce2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 15:08:29 -0500 Subject: [PATCH 222/263] Add smear checks in MFT parsing code --- .../framework/symbols/windows/extensions/mft.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 14c1f08d6..c51dd0348 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -27,6 +27,11 @@ class MFTAttribute(objects.StructType): """This represents an MFT ATTRIBUTE""" def get_resident_filename(self) -> str: + # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems + # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous + if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.NameLength > 512: + return None + # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data try: name = self._context.object( @@ -42,6 +47,11 @@ class MFTAttribute(objects.StructType): return None def get_resident_filecontent(self) -> bytes: + # smear observed in mass testing of samples + # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems + if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.ContentLength > 4194304: + return None + # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data try: bytesobj = self._context.object( From 660e8a7b89387bfc3e883a10a3662708a09fa146 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 15:09:20 -0500 Subject: [PATCH 223/263] Add smear checks in MFT parsing code --- .../framework/symbols/windows/extensions/mft.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index c51dd0348..c0303eafb 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -29,7 +29,10 @@ class MFTAttribute(objects.StructType): def get_resident_filename(self) -> str: # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous - if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.NameLength > 512: + if ( + self.Attr_Header.ContentOffset > 4194304 + or self.Attr_Header.NameLength > 512 + ): return None # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data @@ -49,7 +52,10 @@ class MFTAttribute(objects.StructType): def get_resident_filecontent(self) -> bytes: # smear observed in mass testing of samples # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems - if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.ContentLength > 4194304: + if ( + self.Attr_Header.ContentOffset > 4194304 + or self.Attr_Header.ContentLength > 4194304 + ): return None # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data From 9c058936a88893d0b5dd6a3ee68ab87b037bacf4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 3 Sep 2024 10:30:05 -0500 Subject: [PATCH 224/263] Address feedback --- .../framework/symbols/windows/extensions/mft.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index c0303eafb..ebba882c0 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,6 +2,8 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +from typing import Optional + from volatility3.framework import objects, constants, exceptions @@ -26,11 +28,11 @@ class MFTFileName(objects.StructType): class MFTAttribute(objects.StructType): """This represents an MFT ATTRIBUTE""" - def get_resident_filename(self) -> str: + def get_resident_filename(self) -> Optional[str]: # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous if ( - self.Attr_Header.ContentOffset > 4194304 + self.Attr_Header.ContentOffset > 0x400000 or self.Attr_Header.NameLength > 512 ): return None @@ -49,12 +51,12 @@ class MFTAttribute(objects.StructType): except exceptions.InvalidAddressException: return None - def get_resident_filecontent(self) -> bytes: + def get_resident_filecontent(self) -> Optional[bytes]: # smear observed in mass testing of samples # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems if ( - self.Attr_Header.ContentOffset > 4194304 - or self.Attr_Header.ContentLength > 4194304 + self.Attr_Header.ContentOffset > 0x400000 + or self.Attr_Header.ContentLength > 0x400000 ): return None From 69e6e59daf39ec7a0cf9a82fb13762d4389c95a4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:01:21 -0500 Subject: [PATCH 225/263] Add new pe_symbols API, debug registers plugin, unhooked system calls plugin --- .../plugins/windows/debugregisters.py | 223 ++++++ .../framework/plugins/windows/pe_symbols.py | 732 ++++++++++++++++++ .../plugins/windows/unhooked_system_calls.py | 183 +++++ .../framework/plugins/windows/vadinfo.py | 88 ++- 4 files changed, 1219 insertions(+), 7 deletions(-) create mode 100644 volatility3/framework/plugins/windows/debugregisters.py create mode 100644 volatility3/framework/plugins/windows/pe_symbols.py create mode 100644 volatility3/framework/plugins/windows/unhooked_system_calls.py diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py new file mode 100644 index 000000000..5f3c2e519 --- /dev/null +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -0,0 +1,223 @@ +import logging + +from typing import Tuple, Optional, Generator, List, Dict + +from functools import partial + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +import volatility3.plugins.windows.pslist as pslist +import volatility3.plugins.windows.threads as threads +import volatility3.plugins.windows.vadinfo as vadinfo +import volatility3.plugins.windows.pe_symbols as pe_symbols + +vollog = logging.getLogger(__name__) + + +class DebugRegisters(interfaces.plugins.PluginInterface): + # version 2.6.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 6, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) + ), + ] + + def _get_debug_info( + self, ethread: interfaces.objects.ObjectInterface + ) -> Optional[Tuple[interfaces.objects.ObjectInterface, int, int, int, int, int]]: + """ + Gathers information related to the debug registers for the given thread + """ + try: + dr7 = ethread.Tcb.TrapFrame.Dr7 + state = ethread.Tcb.State + except exceptions.InvalidAddressException: + return None + + # 0 = debug registers not active + # 4 = terminated + if dr7 == 0 or state == 4: + return None + + try: + owner_proc = ethread.owning_process() + except (AttributeError, exceptions.InvalidAddressException): + return None + + dr0 = ethread.Tcb.TrapFrame.Dr0 + dr1 = ethread.Tcb.TrapFrame.Dr1 + dr2 = ethread.Tcb.TrapFrame.Dr2 + dr3 = ethread.Tcb.TrapFrame.Dr3 + + # bail if all are 0 + if not (dr0 or dr1 or dr2 or dr3): + return None + + return owner_proc, dr7, dr0, dr1, dr2, dr3 + + def _get_vads( + self, + vads_cache: Dict[int, List[Tuple[int, int, str]]], + owner_proc: interfaces.objects.ObjectInterface, + ) -> Optional[List[Tuple[int, int, str]]]: + if owner_proc.vol.offset in vads_cache: + vads = vads_cache[owner_proc.vol.offset] + else: + vads = vadinfo.VadInfo.get_proc_vads_with_file_paths(owner_proc) + vads_cache[owner_proc.vol.offset] = vads + + # smear or terminated process + if len(vads) == 0: + return None + + return vads + + def _generator( + self, + ) -> Generator[ + Tuple[ + int, + Tuple[ + str, + int, + int, + int, + int, + format_hints.Hex, + str, + str, + format_hints.Hex, + str, + str, + format_hints.Hex, + str, + str, + format_hints.Hex, + str, + str, + ], + ], + None, + None, + ]: + kernel = self.context.modules[self.config["kernel"]] + + vads_cache: Dict[int, List[Tuple[int, int, str]]] = {} + + proc_modules = None + + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) + + for proc in procs: + for thread in threads.Threads.list_threads(kernel, proc): + debug_info = self._get_debug_info(thread) + if not debug_info: + continue + + owner_proc, dr7, dr0, dr1, dr2, dr3 = debug_info + + vads = self._get_vads(vads_cache, owner_proc) + if not vads: + continue + + # this lookup takes a while, so only perform if we need to + if not proc_modules: + proc_modules = pe_symbols.PESymbols.get_process_modules( + self.context, kernel.layer_name, kernel.symbol_table_name, None + ) + path_and_symbol = partial( + pe_symbols.PESymbols.path_and_symbol_for_address, + self.context, + self.config_path, + proc_modules, + ) + + file0, sym0 = path_and_symbol(vads, dr0) + file1, sym1 = path_and_symbol(vads, dr1) + file2, sym2 = path_and_symbol(vads, dr2) + file3, sym3 = path_and_symbol(vads, dr3) + + # if none map to an actual file VAD then bail + if not ( + isinstance(file0, str) + or isinstance(file1, str) + or isinstance(file2, str) + or isinstance(file3, str) + ): + continue + + process_name = owner_proc.ImageFileName.cast( + "string", + max_length=owner_proc.ImageFileName.vol.count, + errors="replace", + ) + + thread_tid = thread.Cid.UniqueThread + + yield ( + 0, + ( + process_name, + owner_proc.UniqueProcessId, + thread_tid, + thread.Tcb.State, + dr7, + format_hints.Hex(dr0), + file0, + sym0, + format_hints.Hex(dr1), + file1, + sym1, + format_hints.Hex(dr2), + file2, + sym2, + format_hints.Hex(dr3), + file3, + sym3, + ), + ) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("TID", int), + ("State", int), + ("Dr7", int), + ("Dr0", format_hints.Hex), + ("Range0", str), + ("Symbol0", str), + ("Dr1", format_hints.Hex), + ("Range1", str), + ("Symbol1", str), + ("Dr2", format_hints.Hex), + ("Range2", str), + ("Symbol2", str), + ("Dr3", format_hints.Hex), + ("Range3", str), + ("Symbol3", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py new file mode 100644 index 000000000..b7faeaf55 --- /dev/null +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -0,0 +1,732 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + + +import io +import logging + +from typing import Dict, Tuple, Optional, List, Generator, Union + +import pefile + +from volatility3.framework import interfaces, exceptions +from volatility3.framework import renderers, constants +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, vadinfo, modules + +vollog = logging.getLogger(__name__) + + +class PESymbolFinder: + """ + Interface for PE symbol finding classes + This interface provides a standard way for the calling code to + lookup symbols by name or address + """ + + cached_str = Union[str, None] + cached_str_dict = Dict[str, cached_str] + + cached_int = Union[int, None] + cached_int_dict = Dict[str, cached_int] + + cached_value = Union[int, str, None] + cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + + def __init__( + self, + layer_name: str, + mod_name: str, + module_start: int, + symbol_module: Union[interfaces.context.ModuleInterface, pefile.ExportDirData], + ): + self._layer_name = layer_name + self._mod_name = mod_name + self._module_start = module_start + self._symbol_module = symbol_module + + self._address_cache: PESymbolFinder.cached_int_dict = {} + self._name_cache: PESymbolFinder.cached_str_dict = {} + + def _get_cache_key(self, value: cached_value) -> str: + """ + Maintain a cache for symbol lookups to avoid re-walking of PDB symbols or export tables + within the same module for the same address in the same layer + """ + return f"{self._layer_name}|{self._mod_name}|{value}" + + def get_name_for_address(self, address: int) -> cached_str: + cached_key = self._get_cache_key(address) + if cached_key not in self._name_cache: + name = self._do_get_name(address) + self._name_cache[cached_key] = name + + return self._name_cache[cached_key] + + def get_address_for_name(self, name: str) -> cached_int: + cached_key = self._get_cache_key(name) + if cached_key not in self._address_cache: + address = self._do_get_address(name) + self._address_cache[cached_key] = address + + return self._address_cache[cached_key] + + def _do_get_name(self, address: int) -> cached_str: + raise NotImplementedError("_do_get_name must be overwritten") + + def _do_get_address(self, name: str) -> cached_int: + raise NotImplementedError("_do_get_address must be overwritten") + + +class PDBSymbolFinder(PESymbolFinder): + """ + PESymbolFinder implementation for PDB modules + """ + + def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + try: + return self._symbol_module.get_absolute_symbol_address(name) + except exceptions.SymbolError: + return None + + def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + try: + name = self._symbol_module.get_symbols_by_absolute_location(address)[0] + return name.split(constants.BANG)[1] + except (exceptions.SymbolError, IndexError): + return None + + +class ExportSymbolFinder(PESymbolFinder): + """ + PESymbolFinder implementation for PDB modules + """ + + def _get_name(self, export: pefile.ExportData) -> Optional[str]: + # AttributeError throws on empty or ordinal-only exports + try: + return export.name.decode("ascii") + except AttributeError: + return None + + def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + for export in self._symbol_module: + if export.address + self._module_start == address: + return self._get_name(export) + + return None + + def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + for export in self._symbol_module: + sym_name = self._get_name(export) + if sym_name and sym_name == name: + return self._module_start + export.address + + return None + + +class PESymbols(interfaces.plugins.PluginInterface): + """Prints symbols in PE files in process and kernel memory""" + + _required_framework_version = (2, 7, 0) + + _version = (1, 0, 0) + + # used for special handling of the kernel PDB file. See later notes + os_module_name = "ntoskrnl.exe" + + # keys for specifying wanted names and/or addresses + # used for consistent access between the API and plugins + wanted_names = "names" + wanted_addresses = "addresses" + + # how wanted modules/symbols are specified, such as: + # {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} + # {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} + filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + + # holds resolved symbols + # {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} + found_symbols_type = Dict[str, List[Tuple[str, int]]] + + # used to hold informatin about a range (VAD or kernel module) + # (start address, size, file path) + range_type = Tuple[int, int, str] + ranges_type = List[range_type] + + @classmethod + def get_requirements(cls) -> List: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="modules", component=modules.Modules, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pdbutil", component=pdbutil.PDBUtility, version=(1, 0, 0) + ), + requirements.ChoiceRequirement( + name="source", + description="Where to resolve symbols.", + choices=["kernel", "processes"], + optional=False, + ), + requirements.StringRequirement( + name="module", + description='Module in which to resolve symbols. Use "ntoskrnl.exe" to resolve in the base kernel executable.', + optional=False, + ), + requirements.StringRequirement( + name="symbol", + description="Symbol name to resolve", + optional=True, + ), + requirements.IntRequirement( + name="address", + description="Address of symbol to resolve", + optional=True, + ), + ] + + @staticmethod + def _get_pefile_obj( + context: interfaces.context.ContextInterface, + pe_table_name: str, + layer_name: str, + base_address: int, + ) -> Optional[pefile.PE]: + """ + Attempts to pefile object from the bytes of the PE file + + Args: + pe_table_name: name of the pe types table + layer_name: name of the process layer + base_address: base address of the module + + Returns: + the constructed pefile object + """ + pe_data = io.BytesIO() + + try: + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base_address, + layer_name=layer_name, + ) + + for offset, data in dos_header.reconstruct(): + pe_data.seek(offset) + pe_data.write(data) + + pe_ret = pefile.PE(data=pe_data.getvalue(), fast_load=True) + + except exceptions.InvalidAddressException: + pe_ret = None + + return pe_ret + + @staticmethod + def range_info_for_address( + ranges: ranges_type, address: int + ) -> Optional[range_type]: + """ + Helper for getting the range information for an address + """ + for start, size, filepath in ranges: + if start <= address < start + size: + return start, size, filepath + + return None + + @staticmethod + def filepath_for_address(ranges: ranges_type, address: int) -> Optional[str]: + """ + Helper to get the file path for an address + """ + info = PESymbols.range_info_for_address(ranges, address) + if info: + return info[2] + + return None + + @staticmethod + def filename_for_path(filepath: str) -> str: + """ + Consistent way to get the filename + """ + return filepath.split("\\")[-1] + + @staticmethod + def addresses_for_process_symbols( + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + symbol_table_name: str, + symbols: filter_modules_type, + ) -> found_symbols_type: + collected_modules = PESymbols.get_process_modules( + context, layer_name, symbol_table_name, symbols + ) + + found_symbols = PESymbols.find_symbols( + context, config_path, symbols, collected_modules + ) + + for mod_name, unresolved_symbols in symbols.items(): + for symbol in unresolved_symbols: + vollog.debug(f"Unable to resolve symbol {symbol} in module {mod_name}") + + return found_symbols + + @staticmethod + def path_and_symbol_for_address( + context: interfaces.context.ContextInterface, + config_path: str, + collected_modules: Dict[str, List[Tuple[str, int, int]]], + ranges: ranges_type, + address: int, + ) -> Tuple[str, str]: + """ + Method for plugins to determine the file path and symbol name for a given address + + collected_modules: return value from `get_kernel_modules` or `get_process_modules` + ranges: the memory ranges to examine in this layer. + address: address to resolve to its symbol name + """ + + if not address: + return renderers.NotApplicableValue(), renderers.NotApplicableValue() + + filepath = PESymbols.filepath_for_address(ranges, address) + + if not filepath: + return renderers.NotAvailableValue(), renderers.NotAvailableValue() + + filename = PESymbols.filename_for_path(filepath).lower() + + # setup to resolve the address + filter_module: PESymbols.filter_modules_type = { + filename: {PESymbols.wanted_addresses: [address]} + } + + found_symbols = PESymbols.find_symbols( + context, config_path, filter_module, collected_modules + ) + + if not found_symbols or not found_symbols[filename]: + return renderers.NotAvailableValue(), renderers.NotAvailableValue() + + return filepath, found_symbols[filename][0][0] + + @staticmethod + def _get_exported_symbols( + context: interfaces.context.ContextInterface, + pe_table_name: str, + mod_name: str, + module_info: Tuple[str, int, int], + ) -> Optional[ExportSymbolFinder]: + """ + Attempts to locate symbols based on export analysis + + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + """ + + layer_name = module_info[0] + module_start = module_info[1] + + # we need a valid PE with an export table + pe_module = PESymbols._get_pefile_obj( + context, pe_table_name, layer_name, module_start + ) + if not pe_module: + return None + + pe_module.parse_data_directories( + directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_EXPORT"]] + ) + if not hasattr(pe_module, "DIRECTORY_ENTRY_EXPORT"): + return None + + return ExportSymbolFinder( + layer_name, mod_name, module_start, pe_module.DIRECTORY_ENTRY_EXPORT.symbols + ) + + @staticmethod + def _get_pdb_module( + context: interfaces.context.ContextInterface, + config_path: str, + mod_name: str, + module_info: Tuple[str, int, int], + ) -> Optional[PDBSymbolFinder]: + """ + Attempts to locate symbols based on PDB analysis + + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + """ + + mod_symbols = None + + layer_name, module_start, module_size = module_info + + # the PDB name of the kernel file is not consistent for an exe, for example, + # a `ntoskrnl.exe` can have an internal PDB name of any of the ones in the following list + # The code attempts to find all possible PDBs to ensure the best chance of recovery + if mod_name == PESymbols.os_module_name: + pdb_names = ["ntkrnlmp.pdb", "ntkrnlpa.pdb", "ntkrpamp.pdb", "ntoskrnl.pdb"] + + # for non-kernel files, replace the exe, sys, or dll extension with pdb + else: + mod_name = mod_name[:-3] + "pdb" + first_upper = mod_name[0].upper() + mod_name[1:] + pdb_names = [mod_name, first_upper] + + # loop through each PDB name (will be just one for all but the kernel) + for pdb_name in pdb_names: + try: + mod_symbols = pdbutil.PDBUtility.symbol_table_from_pdb( + context, + interfaces.configuration.path_join(config_path, mod_name), + layer_name, + pdb_name, + module_start, + module_size, + ) + + if mod_symbols: + break + + # this exception is expected when the PDB can't be found or downloaded + except exceptions.VolatilityException: + continue + + # this is not expected - it means pdbconv broke when parsing the PDB + except TypeError as e: + vollog.error( + f"Unable to parse PDB file for module {pdb_name} -> {e}. Please file a bug on the GitHub issue tracker." + ) + + # cannot do anything without the symbols + if not mod_symbols: + return None + + pdb_module = context.module( + mod_symbols, layer_name=layer_name, offset=module_start + ) + + return PDBSymbolFinder(layer_name, mod_name, module_start, pdb_module) + + @staticmethod + def _find_symbols_through_pdb( + context: interfaces.context.ContextInterface, + config_path: str, + module_instances: List[Tuple[str, int, int]], + mod_name: str, + ) -> Generator[PDBSymbolFinder, None, None]: + """ + Attempts to resolve the symbols in `wanted_symbols` through PDB analysis + """ + for module_info in module_instances: + mod_module = PESymbols._get_pdb_module( + context, config_path, mod_name, module_info + ) + if mod_module: + yield mod_module + + @staticmethod + def _find_symbols_through_exports( + context: interfaces.context.ContextInterface, + config_path: str, + module_instances: List[Tuple[str, int, int]], + mod_name: str, + ) -> Generator[ExportSymbolFinder, None, None]: + """ + Attempts to resolve the symbols in `wanted_symbols` through export analysis + """ + pe_table_name = intermed.IntermediateSymbolTable.create( + context, config_path, "windows", "pe", class_types=pe.class_types + ) + + # for each process layer and VAD, construct a PE and examine the export table + for module_info in module_instances: + exported_symbols = PESymbols._get_exported_symbols( + context, pe_table_name, mod_name, module_info + ) + if exported_symbols: + yield exported_symbols + + @staticmethod + def _get_symbol_value( + wanted_modules: PESymbolFinder.cached_value_dict, + mod_name: str, + symbol_resolver: PESymbolFinder, + ) -> Generator[Tuple[str, int], None, None]: + """ + Enumerates the symbols specified as wanted by the calling plugin + """ + wanted_symbols = wanted_modules[mod_name] + + if ( + PESymbols.wanted_names not in wanted_symbols + and PESymbols.wanted_addresses not in wanted_symbols + ): + vollog.warning( + f"Invalid `wanted_symbols` sent to `find_symbols` for module {mod_name}. addresses and names keys both misssing." + ) + return + + symbol_keys = [ + (PESymbols.wanted_names, "get_address_for_name"), + (PESymbols.wanted_addresses, "get_name_for_address"), + ] + + for symbol_key, symbol_getter in symbol_keys: + # address or name + if symbol_key in wanted_symbols: + # walk each wanted address or name + for wanted_value in wanted_symbols[symbol_key]: + symbol_value = symbol_resolver.__getattribute__(symbol_getter)( + wanted_value + ) + if symbol_value: + # yield out symbol name, symbol address + if symbol_key == PESymbols.wanted_names: + yield wanted_value, symbol_value # type: ignore + else: + yield symbol_value, wanted_value # type: ignore + + index = wanted_modules[mod_name][symbol_key].index(wanted_value) # type: ignore + + del wanted_modules[mod_name][symbol_key][index] + + # if all names or addresses from a module are found, delete the key + if not wanted_modules[mod_name][symbol_key]: + del wanted_modules[mod_name][symbol_key] + break + + @staticmethod + def _resolve_symbols_through_methods( + context: interfaces.context.ContextInterface, + config_path: str, + module_instances: List[Tuple[str, int, int]], + wanted_modules: PESymbolFinder.cached_value_dict, + mod_name: str, + ) -> Generator[Tuple[str, int], None, None]: + """ + Attempts to resolve every wanted symbol in `mod_name` + Every layer is enumerated for maximum chance of recovery + """ + symbol_resolving_methods = [ + PESymbols._find_symbols_through_pdb, + PESymbols._find_symbols_through_exports, + ] + + for method in symbol_resolving_methods: + for symbol_resolver in method( + context, config_path, module_instances, mod_name + ): + vollog.debug(f"Have resolver for method {method}") + yield from PESymbols._get_symbol_value( + wanted_modules, mod_name, symbol_resolver + ) + + if not wanted_modules[mod_name]: + break + + if not wanted_modules[mod_name]: + break + + @staticmethod + def find_symbols( + context: interfaces.context.ContextInterface, + config_path: str, + wanted_modules: PESymbolFinder.cached_value_dict, + collected_modules: Dict[str, List[Tuple[str, int, int]]], + ) -> found_symbols_type: + """ + Loops through each method of symbol analysis until each wanted symbol is found + Returns the resolved symbols as a dictionary that includes the name and runtime address + """ + found_symbols: PESymbols.found_symbols_type = {} + + for mod_name in wanted_modules: + if mod_name not in collected_modules: + continue + + module_instances = collected_modules[mod_name] + + # try to resolve the symbols for `mod_name` through each method (PDB and export table currently) + for symbol_name, address in PESymbols._resolve_symbols_through_methods( + context, config_path, module_instances, wanted_modules, mod_name + ): + if mod_name not in found_symbols: + found_symbols[mod_name] = [] + + found_symbols[mod_name].append((symbol_name, address)) + + # stop processing the layers (processes) if we found all the symbols for this module + if not wanted_modules[mod_name]: + break + + # stop processing this module if/when all symbols are found + if not wanted_modules[mod_name]: + del wanted_modules[mod_name] + break + + return found_symbols + + @staticmethod + def get_kernel_modules( + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + filter_modules: Optional[filter_modules_type], + ) -> Dict[str, List[Tuple[str, int, int]]]: + """ + Walks the kernel module list and finds the session layer, base, and size of each wanted module + """ + found_modules: Dict[str, List[Tuple[str, int, int]]] = {} + + if filter_modules: + # create a tuple of module names for use with `endswith` + filter_modules_check = tuple([key.lower() for key in filter_modules.keys()]) + else: + filter_modules_check = None + + session_layers = list( + modules.Modules.get_session_layers(context, layer_name, symbol_table) + ) + + # special handling for the kernel + gather_kernel = ( + filter_modules_check and PESymbols.os_module_name in filter_modules_check + ) + + for index, mod in enumerate( + modules.Modules.list_modules(context, layer_name, symbol_table) + ): + try: + mod_name = str(mod.BaseDllName.get_string().lower()) + except exceptions.InvalidAddressException: + continue + + # to analyze, it must either be the kernel or a wanted module + if not filter_modules_check or (gather_kernel and index == 0): + mod_name = PESymbols.os_module_name + elif filter_modules_check and not mod_name.endswith(filter_modules_check): + continue + + # we won't find symbol information if we can't analyze the module + session_layer_name = modules.Modules.find_session_layer( + context, session_layers, mod.DllBase + ) + if not session_layer_name: + continue + + if mod_name not in found_modules: + found_modules[mod_name] = [] + + found_modules[mod_name].append( + (session_layer_name, mod.DllBase, mod.SizeOfImage) + ) + + return found_modules + + @staticmethod + def get_process_modules( + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + filter_modules: Optional[filter_modules_type], + ) -> Dict[str, List[Tuple[str, int, int]]]: + """ + Walks the process list and each process' VAD to determine the base address and size of wanted modules + """ + proc_modules: Dict[str, List[Tuple[str, int, int]]] = {} + + if filter_modules: + # create a tuple of module names for use with `endswith` + filter_modules_check = tuple([key.lower() for key in filter_modules.keys()]) + else: + filter_modules_check = None + + for _, proc_layer_name, vads in vadinfo.VadInfo.get_all_vads_with_file_paths( + context, layer_name, symbol_table + ): + for vad_start, vad_size, filepath in vads: + filename = PESymbols.filename_for_path(filepath) + + if filter_modules_check and not filename.endswith(filter_modules_check): + continue + + # track each module along with the process layer and range to find it + if filename not in proc_modules: + proc_modules[filename] = [] + + proc_modules[filename].append((proc_layer_name, vad_start, vad_size)) + + return proc_modules + + def _generator(self) -> Generator[Tuple[int, Tuple[str, str, int]], None, None]: + kernel = self.context.modules[self.config["kernel"]] + + if self.config["symbol"]: + filter_module = { + self.config["module"].lower(): { + PESymbols.wanted_names: [self.config["symbol"]] + } + } + + elif self.config["address"]: + filter_module = { + self.config["module"].lower(): { + PESymbols.wanted_addresses: [self.config["address"]] + } + } + + else: + vollog.error("--address or --symbol must be specified") + return + + if self.config["source"] == "kernel": + module_resolver = self.get_kernel_modules + else: + module_resolver = self.get_process_modules + + collected_modules = module_resolver( + self.context, kernel.layer_name, kernel.symbol_table_name, filter_module + ) + + found_symbols = PESymbols.find_symbols( + self.context, self.config_path, filter_module, collected_modules + ) + + for module, symbols in found_symbols.items(): + for symbol, address in symbols: + yield (0, (module, symbol, format_hints.Hex(address))) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Module", str), + ("Symbol", str), + ("Address", format_hints.Hex), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py new file mode 100644 index 000000000..0438bc9e3 --- /dev/null +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -0,0 +1,183 @@ +import logging + +from typing import Dict, Tuple, List, Generator + +from volatility3.framework import interfaces, exceptions +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.plugins.windows import pslist, pe_symbols + +vollog = logging.getLogger(__name__) + + +class unhooked_system_calls(interfaces.plugins.PluginInterface): + """Looks for signs of Skeleton Key malware""" + + _required_framework_version = (2, 4, 0) + + system_calls = { + "ntdll.dll": { + pe_symbols.PESymbols.wanted_names: [ + "NtCreateThread", + "NtProtectVirtualMemory", + "NtReadVirtualMemory", + "NtOpenProcess", + "NtWriteFile", + "NtQueryVirtualMemory", + "NtAllocateVirtualMemory", + "NtWorkerFactoryWorkerReady", + "NtAcceptConnectPort", + "NtAddDriverEntry", + "NtAdjustPrivilegesToken", + "NtAlpcCreatePort", + "NtClose", + "NtCreateFile", + "NtCreateMutant", + "NtOpenFile", + "NtOpenIoCompletion", + "NtOpenJobObject", + "NtOpenKey", + "NtOpenKeyEx", + "NtOpenThread", + "NtOpenThreadToken", + "NtOpenThreadTokenEx", + "NtWriteVirtualMemory", + "NtTraceEvent", + "NtTranslateFilePath", + "NtUmsThreadYield", + "NtUnloadDriver", + "NtUnloadKey", + "NtUnloadKey2", + "NtUnloadKeyEx", + "NtCreateKey", + "NtCreateSection", + "NtDeleteKey", + "NtDeleteValueKey", + "NtDuplicateObject", + "NtQueryValueKey", + "NtReplaceKey", + "NtRequestWaitReplyPort", + "NtRestoreKey", + "NtSetContextThread", + "NtSetSecurityObject", + "NtSetValueKey", + "NtSystemDebugControl", + "NtTerminateProcess", + ] + } + } + + _code_bytes_type = Dict[str, Dict[str, Dict[bytes, List[Tuple[int, str]]]]] + + @classmethod + def get_requirements(cls) -> List: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="pe_symbols", plugin=pe_symbols.PESymbols, version=(1, 0, 0) + ), + ] + + def _gather_code_bytes( + self, + kernel: interfaces.context.ModuleInterface, + found_symbols: pe_symbols.PESymbols.found_symbols_type, + ) -> _code_bytes_type: + """ + Enumerates the desired DLLs and function implementations in each process + Groups based on unique implementations of each DLLs' functions + The purpose is to detect when a function has different implementations (code) + in different processes. + This very effectively detects code injection. + """ + code_bytes: unhooked_system_calls._code_bytes_type = {} + + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) + + for proc in procs: + try: + proc_id = proc.UniqueProcessId + proc_name = utility.array_to_string(proc.ImageFileName) + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + for dll_name, functions in found_symbols.items(): + for func_name, func_addr in functions: + try: + fbytes = self.context.layers[proc_layer_name].read( + func_addr, 0x20 + ) + except exceptions.InvalidAddressException: + continue + + if dll_name not in code_bytes: + code_bytes[dll_name] = {} + + if func_name not in code_bytes[dll_name]: + code_bytes[dll_name][func_name] = {} + + if fbytes not in code_bytes[dll_name][func_name]: + code_bytes[dll_name][func_name][fbytes] = [] + + code_bytes[dll_name][func_name][fbytes].append((proc_id, proc_name)) + + return code_bytes + + def _generator(self) -> Generator[Tuple[int, Tuple[str, str, int]], None, None]: + kernel = self.context.modules[self.config["kernel"]] + + found_symbols = pe_symbols.PESymbols.addresses_for_process_symbols( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + unhooked_system_calls.system_calls, + ) + + # code_bytes[dll_name][func_name][func_bytes] + code_bytes = self._gather_code_bytes(kernel, found_symbols) + + for functions in code_bytes.values(): + for func_name, cbb in functions.items(): + cb = list(cbb.values()) + + # same implementation in all + if len(cb) == 1: + yield 0, (func_name, "", len(cb[0])) + else: + # find the processes that are hooked for reporting + max_idx = 0 if len(cb[0]) > len(cb[1]) else 1 + small_idx = (~max_idx) & 1 + + ps = [] + + for pid, pname in cb[small_idx]: + ps.append("{:d}:{}".format(pid, pname)) + + proc_names = ", ".join(ps) + + yield 0, (func_name, proc_names, len(cb[max_idx])) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Function", str), + ("Distinct Implementations", str), + ("Total Implementations", int), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index abc6142fe..97a2f2455 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -3,13 +3,13 @@ # import logging -from typing import Callable, List, Generator, Iterable, Type, Optional +from typing import Callable, List, Generator, Iterable, Type, Optional, Tuple -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import renderers, interfaces, exceptions, symbols from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist +from volatility3.plugins.windows import pslist, pe_symbols vollog = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class VadInfo(interfaces.plugins.PluginInterface): _version = (2, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): # type: ignore super().__init__(*args, **kwargs) self._protect_values = None @@ -107,6 +107,58 @@ class VadInfo(interfaces.plugins.PluginInterface): ) return values # type: ignore + @staticmethod + def get_proc_vads_with_file_paths( + proc: interfaces.objects.ObjectInterface, + ) -> pe_symbols.PESymbols.ranges_type: + """ + Returns a list of the process' vads that map a file + """ + vads = [] + + for vad in proc.get_vad_root().traverse(): + filepath = vad.get_file_name() + if not isinstance(filepath, str) or filepath.count("\\") == 0: + continue + + vads.append((vad.get_start(), vad.get_size(), filepath)) + + return vads + + @classmethod + def get_all_vads_with_file_paths( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table_name: str, + ) -> Generator[ + Tuple[ + interfaces.objects.ObjectInterface, str, pe_symbols.PESymbols.ranges_type + ], + None, + None, + ]: + """ + Yields each set of vads for a process that have a file mapped, along with the process itself and its layer + """ + is_32bit_arch = not symbols.symbol_table_is_64bit(context, symbol_table_name) + + procs = pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, + ) + + for proc in procs: + try: + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + vads = cls.get_proc_vads_with_file_paths(proc) + + yield proc, proc_layer_name, vads + @classmethod def list_vads( cls, @@ -196,11 +248,33 @@ class VadInfo(interfaces.plugins.PluginInterface): return file_handle - def _generator(self, procs): + def _generator( + self, procs: List[interfaces.objects.ObjectInterface] + ) -> Generator[ + Tuple[ + int, + Tuple[ + int, + str, + format_hints.Hex, + format_hints.Hex, + format_hints.Hex, + str, + str, + int, + int, + format_hints.Hex, + str, + str, + ], + ], + None, + None, + ]: kernel = self.context.modules[self.config["kernel"]] kernel_layer = self.context.layers[kernel.layer_name] - def passthrough(_: interfaces.objects.ObjectInterface) -> bool: + def passthrough(x: interfaces.objects.ObjectInterface) -> bool: return False filter_func = passthrough @@ -250,7 +324,7 @@ class VadInfo(interfaces.plugins.PluginInterface): ), ) - def run(self): + def run(self) -> renderers.TreeGrid: kernel = self.context.modules[self.config["kernel"]] filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) From 3bb9264a5770d6828487129b6ebfec509ade8680 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:11:16 -0500 Subject: [PATCH 226/263] formatting --- volatility3/framework/plugins/windows/debugregisters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 5f3c2e519..c70f922ae 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -37,7 +37,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) ), - ] + ] def _get_debug_info( self, ethread: interfaces.objects.ObjectInterface From 30cb5bd3ab4bac9f3ebb224d2f37cb98d6960545 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:14:06 -0500 Subject: [PATCH 227/263] Formatting that my local black doesn't understand --- volatility3/framework/plugins/windows/vadinfo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 97a2f2455..b3b7bf5fb 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -249,8 +249,7 @@ class VadInfo(interfaces.plugins.PluginInterface): return file_handle def _generator( - self, procs: List[interfaces.objects.ObjectInterface] - ) -> Generator[ + self, procs: List[interfaces.objects.ObjectInterface]) -> Generator[ Tuple[ int, Tuple[ From c223ac6e762d1072f04c21276645d7c2dda79dde Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:31:55 -0500 Subject: [PATCH 228/263] more black help --- volatility3/framework/plugins/windows/vadinfo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index b3b7bf5fb..655e54be6 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -141,8 +141,6 @@ class VadInfo(interfaces.plugins.PluginInterface): """ Yields each set of vads for a process that have a file mapped, along with the process itself and its layer """ - is_32bit_arch = not symbols.symbol_table_is_64bit(context, symbol_table_name) - procs = pslist.PsList.list_processes( context=context, layer_name=layer_name, From 7b48ee4be489b4667c57cca55da84abd3c1c3d12 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:36:13 -0500 Subject: [PATCH 229/263] more black help --- volatility3/framework/plugins/windows/vadinfo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 655e54be6..594c0d67c 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -246,8 +246,7 @@ class VadInfo(interfaces.plugins.PluginInterface): return file_handle - def _generator( - self, procs: List[interfaces.objects.ObjectInterface]) -> Generator[ + def _generator(self, procs: List[interfaces.objects.ObjectInterface]) -> Generator[ Tuple[ int, Tuple[ From bcd93616c0628fdb991866290845bbb861889f8f Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:36:35 -0500 Subject: [PATCH 230/263] more black help --- volatility3/framework/plugins/windows/vadinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 594c0d67c..e129ad4ef 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -5,7 +5,7 @@ import logging from typing import Callable, List, Generator, Iterable, Type, Optional, Tuple -from volatility3.framework import renderers, interfaces, exceptions, symbols +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints From fd8777287736baee75e6f1a8843b471a47b6ced2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:47:42 -0500 Subject: [PATCH 231/263] Move VAD enumeration into pe_symbols --- .../framework/plugins/windows/pe_symbols.py | 57 +++++++++++++++++-- .../framework/plugins/windows/vadinfo.py | 50 ---------------- 2 files changed, 52 insertions(+), 55 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index b7faeaf55..51558f71b 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -16,7 +16,7 @@ from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbutil from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, vadinfo, modules +from volatility3.plugins.windows import pslist, modules vollog = logging.getLogger(__name__) @@ -170,9 +170,6 @@ class PESymbols(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) - ), requirements.VersionRequirement( name="modules", component=modules.Modules, version=(2, 0, 0) ), @@ -648,6 +645,56 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_modules + @staticmethod + def get_proc_vads_with_file_paths( + proc: interfaces.objects.ObjectInterface, + ) -> ranges_type: + """ + Returns a list of the process' vads that map a file + """ + vads = [] + + for vad in proc.get_vad_root().traverse(): + filepath = vad.get_file_name() + if not isinstance(filepath, str) or filepath.count("\\") == 0: + continue + + vads.append((vad.get_start(), vad.get_size(), filepath)) + + return vads + + @classmethod + def get_all_vads_with_file_paths( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table_name: str, + ) -> Generator[ + Tuple[ + interfaces.objects.ObjectInterface, str, ranges_type + ], + None, + None, + ]: + """ + Yields each set of vads for a process that have a file mapped, along with the process itself and its layer + """ + procs = pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, + ) + + for proc in procs: + try: + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + vads = PESymbols.get_proc_vads_with_file_paths(proc) + + yield proc, proc_layer_name, vads + @staticmethod def get_process_modules( context: interfaces.context.ContextInterface, @@ -666,7 +713,7 @@ class PESymbols(interfaces.plugins.PluginInterface): else: filter_modules_check = None - for _, proc_layer_name, vads in vadinfo.VadInfo.get_all_vads_with_file_paths( + for _, proc_layer_name, vads in PESymbols.get_all_vads_with_file_paths( context, layer_name, symbol_table ): for vad_start, vad_size, filepath in vads: diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index e129ad4ef..974793a71 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -107,56 +107,6 @@ class VadInfo(interfaces.plugins.PluginInterface): ) return values # type: ignore - @staticmethod - def get_proc_vads_with_file_paths( - proc: interfaces.objects.ObjectInterface, - ) -> pe_symbols.PESymbols.ranges_type: - """ - Returns a list of the process' vads that map a file - """ - vads = [] - - for vad in proc.get_vad_root().traverse(): - filepath = vad.get_file_name() - if not isinstance(filepath, str) or filepath.count("\\") == 0: - continue - - vads.append((vad.get_start(), vad.get_size(), filepath)) - - return vads - - @classmethod - def get_all_vads_with_file_paths( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table_name: str, - ) -> Generator[ - Tuple[ - interfaces.objects.ObjectInterface, str, pe_symbols.PESymbols.ranges_type - ], - None, - None, - ]: - """ - Yields each set of vads for a process that have a file mapped, along with the process itself and its layer - """ - procs = pslist.PsList.list_processes( - context=context, - layer_name=layer_name, - symbol_table=symbol_table_name, - ) - - for proc in procs: - try: - proc_layer_name = proc.add_process_layer() - except exceptions.InvalidAddressException: - continue - - vads = cls.get_proc_vads_with_file_paths(proc) - - yield proc, proc_layer_name, vads - @classmethod def list_vads( cls, From 61cf58d97794359e3e093a97f3b8d2562661aea4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:49:01 -0500 Subject: [PATCH 232/263] black fixes --- volatility3/framework/plugins/windows/pe_symbols.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 51558f71b..10e87c552 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -670,9 +670,7 @@ class PESymbols(interfaces.plugins.PluginInterface): layer_name: str, symbol_table_name: str, ) -> Generator[ - Tuple[ - interfaces.objects.ObjectInterface, str, ranges_type - ], + Tuple[interfaces.objects.ObjectInterface, str, ranges_type], None, None, ]: From 802fc024b5a7666f1e56a641af4fe219c856f6b4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 13:27:04 -0500 Subject: [PATCH 233/263] switch api place --- volatility3/framework/plugins/windows/debugregisters.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index c70f922ae..931ccef37 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -9,7 +9,6 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints import volatility3.plugins.windows.pslist as pslist import volatility3.plugins.windows.threads as threads -import volatility3.plugins.windows.vadinfo as vadinfo import volatility3.plugins.windows.pe_symbols as pe_symbols vollog = logging.getLogger(__name__) @@ -31,9 +30,6 @@ class DebugRegisters(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) - ), requirements.VersionRequirement( name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) ), @@ -80,7 +76,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): if owner_proc.vol.offset in vads_cache: vads = vads_cache[owner_proc.vol.offset] else: - vads = vadinfo.VadInfo.get_proc_vads_with_file_paths(owner_proc) + vads = pe_symbols.PESymbols.get_proc_vads_with_file_paths(owner_proc) vads_cache[owner_proc.vol.offset] = vads # smear or terminated process From 26cd73115b58cf5b1adfb430c9f1070e45dbe727 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 13:31:09 +0100 Subject: [PATCH 234/263] CLI: Filter on rendered values --- volatility3/cli/text_renderer.py | 38 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ab9e44141..1f9cfa9d8 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -179,7 +179,15 @@ class QuickTextRenderer(CLIRenderer): outfd.write("\n{}\n".format("\t".join(line))) def visitor(node: interfaces.renderers.TreeNode, accumulator): - if self.filter and self.filter.filter(node.values): + line = [] + for column_index in range(len(grid.columns)): + column = grid.columns[column_index] + renderer = self._type_renderers.get( + column.type, self._type_renderers["default"] + ) + line.append(renderer(node.values[column_index])) + + if self.filter and self.filter.filter(line): return accumulator accumulator.write("\n") @@ -188,13 +196,6 @@ class QuickTextRenderer(CLIRenderer): "*" * max(0, node.path_depth - 1) + ("" if (node.path_depth <= 1) else " ") ) - line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] - renderer = self._type_renderers.get( - column.type, self._type_renderers["default"] - ) - line.append(renderer(node.values[column_index])) accumulator.write("{}".format("\t".join(line))) accumulator.flush() return accumulator @@ -259,12 +260,18 @@ class CSVRenderer(CLIRenderer): def visitor(node: interfaces.renderers.TreeNode, accumulator): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case row = {"TreeDepth": str(max(0, node.path_depth - 1))} + line = [] for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) row[f"{column.name}"] = renderer(node.values[column_index]) + line.append(row[f"{column.name}"]) + + if self.filter and self.filter.filter(line): + return accumulator + accumulator.writerow(row) return accumulator @@ -317,10 +324,8 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths.get(tree_indent_column, 0), node.path_depth ) - if self.filter and self.filter.filter(node.values): - return accumulator - line = {} + rendered_line = [] for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get( @@ -334,6 +339,11 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths.get(column.name, len(column.name)), field_width ) line[column] = data.split("\n") + rendered_line.append(data) + + if self.filter and self.filter.filter(rendered_line): + return accumulator + accumulator.append((node.path_depth, line)) return accumulator @@ -437,6 +447,7 @@ class JsonRenderer(CLIRenderer): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case acc_map, final_tree = accumulator node_dict: Dict[str, Any] = {"__children": []} + line = [] for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get( @@ -446,6 +457,11 @@ class JsonRenderer(CLIRenderer): if isinstance(data, interfaces.renderers.BaseAbsentValue): data = None node_dict[column.name] = data + line.append(data) + + if self.filter and self.filter.filter(line): + return accumulator + if node.parent: acc_map[node.parent.path]["__children"].append(node_dict) else: From 8259ca90199f548e1592aa3ab05444c9353fe66a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 13:47:26 +0100 Subject: [PATCH 235/263] Core: Bump the framework number so we can differentiate CLI versions --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index d30446d63..a131609c9 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 9 # Number of changes that only add to the interface +VERSION_MINOR = 10 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 4361c3857393b7f7311e9b600d1ea5d4966cdb93 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 14:55:04 +0100 Subject: [PATCH 236/263] Core: Verify plugin requirements of plugins --- .../framework/configuration/requirements.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 1c0622574..931995bb8 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -585,6 +585,24 @@ class PluginRequirement(VersionRequirement): version=version, ) + def unsatisfied( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: + result = super().unsatisfied(context, config_path) + if not result: + component: Type[interfaces.plugins.PluginInterface] = self._component + for requirement in component.get_requirements(): + if isinstance(requirement, PluginRequirement): + result.update( + requirement.unsatisfied( + context, + interfaces.configuration.path_join(config_path, self.name), + ) + ) + if result: + result[config_path] = self + return result + class ModuleRequirement( interfaces.configuration.ConstructableRequirementInterface, From f3085b6c59b05190320722feef58f276357d2155 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 16:06:23 +0100 Subject: [PATCH 237/263] Core: Move the pluginrequirement check to generic versionrequirement --- .../framework/configuration/requirements.py | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 931995bb8..f95dac307 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -527,12 +527,14 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, optional: bool = False, component: Type[interfaces.configuration.VersionableInterface] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: + if description is None: + description = f"Version {".".join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) @@ -550,9 +552,29 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} + + # Check for child requirements + if issubclass(self._component, interfaces.configuration.ConfigurableInterface): + result = {} + for requirement in self._component.get_requirements(): + if not requirement.optional and isinstance( + requirement, VersionRequirement + ): + result.update( + requirement.unsatisfied( + context, + config_path, + ) + ) + + if result: + result.update({config_path: self}) + return result + context.config[interfaces.configuration.path_join(config_path, self.name)] = ( True ) + return {} @classmethod @@ -585,24 +607,6 @@ class PluginRequirement(VersionRequirement): version=version, ) - def unsatisfied( - self, context: interfaces.context.ContextInterface, config_path: str - ) -> Dict[str, interfaces.configuration.RequirementInterface]: - result = super().unsatisfied(context, config_path) - if not result: - component: Type[interfaces.plugins.PluginInterface] = self._component - for requirement in component.get_requirements(): - if isinstance(requirement, PluginRequirement): - result.update( - requirement.unsatisfied( - context, - interfaces.configuration.path_join(config_path, self.name), - ) - ) - if result: - result[config_path] = self - return result - class ModuleRequirement( interfaces.configuration.ConstructableRequirementInterface, From ace590e8f0669797b5770773eeadcbcb27752a78 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 16:08:48 +0100 Subject: [PATCH 238/263] Core: Fix up f-string containing a string --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index f95dac307..3a862a132 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -534,7 +534,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): version: Optional[Tuple[int, ...]] = None, ) -> None: if description is None: - description = f"Version {".".join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" + description = f"Version {'.'.join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) From bf000ff0a0bafc0197ee1dd074f1df450362d02f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 21:42:59 +0100 Subject: [PATCH 239/263] Core: Add recursion protection to VersionRequirement check --- .../framework/configuration/requirements.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 3a862a132..49ca49b59 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -546,13 +546,26 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): self._version = version def unsatisfied( - self, context: interfaces.context.ContextInterface, config_path: str + self, + context: interfaces.context.ContextInterface, + config_path: str, + accumulator: Optional[ + List[interfaces.configuration.VersionableInterface] + ] = None, ) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} + if accumulator is None: + accumulator = set([self._component]) + else: + if self._component in accumulator: + return {config_path: self} + else: + accumulator.add(self._component) + # Check for child requirements if issubclass(self._component, interfaces.configuration.ConfigurableInterface): result = {} @@ -562,8 +575,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): ): result.update( requirement.unsatisfied( - context, - config_path, + context, config_path, accumulator.copy() ) ) From 9152f33181fd347366468d00c47a2c22ae71cfc2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 21:58:09 +0100 Subject: [PATCH 240/263] Core: Allow circular dependencies as long as they are all met --- volatility3/framework/configuration/requirements.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 49ca49b59..dcb7505ea 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -558,16 +558,20 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): if not self.matches_required(self._version, self._component.version): return {config_path: self} + recurse = True if accumulator is None: accumulator = set([self._component]) else: if self._component in accumulator: - return {config_path: self} + recurse = False else: accumulator.add(self._component) # Check for child requirements - if issubclass(self._component, interfaces.configuration.ConfigurableInterface): + if ( + issubclass(self._component, interfaces.configuration.ConfigurableInterface) + and recurse + ): result = {} for requirement in self._component.get_requirements(): if not requirement.optional and isinstance( From fd2c97e1bafb5b30403a1788eae4c32723438746 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 9 Sep 2024 00:44:00 +0100 Subject: [PATCH 241/263] CLI: Use enumerate for renderers (thanks @gcmoreira) --- volatility3/cli/text_renderer.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 1f9cfa9d8..da0cdf62a 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -180,8 +180,7 @@ class QuickTextRenderer(CLIRenderer): def visitor(node: interfaces.renderers.TreeNode, accumulator): line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) @@ -261,8 +260,7 @@ class CSVRenderer(CLIRenderer): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case row = {"TreeDepth": str(max(0, node.path_depth - 1))} line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) @@ -326,8 +324,7 @@ class PrettyTextRenderer(CLIRenderer): line = {} rendered_line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) @@ -357,8 +354,7 @@ class PrettyTextRenderer(CLIRenderer): format_string_list = [ "{0:<" + str(max_column_widths.get(tree_indent_column, 0)) + "s}" ] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): format_string_list.append( "{" + str(column_index + 1) @@ -448,8 +444,7 @@ class JsonRenderer(CLIRenderer): acc_map, final_tree = accumulator node_dict: Dict[str, Any] = {"__children": []} line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) From b7e604d63f667663f9a00415493ec4f8b12a5024 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 10 Sep 2024 14:50:40 -0500 Subject: [PATCH 242/263] Address feedback --- .../plugins/windows/debugregisters.py | 33 +- .../framework/plugins/windows/pe_symbols.py | 394 ++++++++++++++---- .../plugins/windows/unhooked_system_calls.py | 7 +- .../framework/plugins/windows/vadinfo.py | 4 +- 4 files changed, 331 insertions(+), 107 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 931ccef37..65b2e625b 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -1,3 +1,6 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + import logging from typing import Tuple, Optional, Generator, List, Dict @@ -35,11 +38,16 @@ class DebugRegisters(interfaces.plugins.PluginInterface): ), ] + @staticmethod def _get_debug_info( - self, ethread: interfaces.objects.ObjectInterface + ethread: interfaces.objects.ObjectInterface, ) -> Optional[Tuple[interfaces.objects.ObjectInterface, int, int, int, int, int]]: """ Gathers information related to the debug registers for the given thread + Args: + ethread: the thread (_ETHREAD) to examine + Returns: + Tuple[interfaces.objects.ObjectInterface, int, int, int, int, int]: The owner process of the thread and the values for dr7, dr0, dr1, dr2, dr3 """ try: dr7 = ethread.Tcb.TrapFrame.Dr7 @@ -68,23 +76,6 @@ class DebugRegisters(interfaces.plugins.PluginInterface): return owner_proc, dr7, dr0, dr1, dr2, dr3 - def _get_vads( - self, - vads_cache: Dict[int, List[Tuple[int, int, str]]], - owner_proc: interfaces.objects.ObjectInterface, - ) -> Optional[List[Tuple[int, int, str]]]: - if owner_proc.vol.offset in vads_cache: - vads = vads_cache[owner_proc.vol.offset] - else: - vads = pe_symbols.PESymbols.get_proc_vads_with_file_paths(owner_proc) - vads_cache[owner_proc.vol.offset] = vads - - # smear or terminated process - if len(vads) == 0: - return None - - return vads - def _generator( self, ) -> Generator[ @@ -115,7 +106,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): ]: kernel = self.context.modules[self.config["kernel"]] - vads_cache: Dict[int, List[Tuple[int, int, str]]] = {} + vads_cache: Dict[int, pe_symbols.ranges_type] = {} proc_modules = None @@ -133,7 +124,9 @@ class DebugRegisters(interfaces.plugins.PluginInterface): owner_proc, dr7, dr0, dr1, dr2, dr3 = debug_info - vads = self._get_vads(vads_cache, owner_proc) + vads = pe_symbols.PESymbols.get_vads_for_process_cache( + vads_cache, owner_proc + ) if not vads: continue diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 10e87c552..c9785a1c9 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -1,9 +1,9 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 - import io import logging +import ntpath from typing import Dict, Tuple, Optional, List, Generator, Union @@ -17,9 +17,37 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbutil from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import pslist, modules +from volatility3.framework.constants.windows import KERNEL_MODULE_NAMES vollog = logging.getLogger(__name__) +# keys for specifying wanted names and/or addresses +# used for consistent access between the API and plugins +wanted_names_identifier = "names" +wanted_addresses_identifier = "addresses" + +# how wanted modules/symbols are specified, such as: +# {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} +# {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} +filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + +# holds resolved symbols +# {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} +found_symbols_type = Dict[str, List[Tuple[str, int]]] + +# used to hold informatin about a range (VAD or kernel module) +# (start address, size, file path) +range_type = Tuple[int, int, str] +ranges_type = List[range_type] + +# collected_modules are modules and their symbols found when walking vads or kernel modules +# Tuple of (process or kernel layer name, range start, range size) +collected_module_instance = Tuple[str, int, int] +collected_modules_info = List[collected_module_instance] +collected_modules_type = Dict[str, collected_modules_info] + +PESymbolFinders = Union[interfaces.context.ModuleInterface, pefile.ExportDirData] + class PESymbolFinder: """ @@ -28,21 +56,19 @@ class PESymbolFinder: lookup symbols by name or address """ - cached_str = Union[str, None] - cached_str_dict = Dict[str, cached_str] + cached_str_dict = Dict[str, Optional[str]] - cached_int = Union[int, None] - cached_int_dict = Dict[str, cached_int] + cached_int_dict = Dict[str, Optional[int]] cached_value = Union[int, str, None] - cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + cached_value_dict = Dict[str, Dict[str, List[str]] | Dict[str, List[int]]] def __init__( self, layer_name: str, mod_name: str, module_start: int, - symbol_module: Union[interfaces.context.ModuleInterface, pefile.ExportDirData], + symbol_module: PESymbolFinders, ): self._layer_name = layer_name self._mod_name = mod_name @@ -56,10 +82,25 @@ class PESymbolFinder: """ Maintain a cache for symbol lookups to avoid re-walking of PDB symbols or export tables within the same module for the same address in the same layer + + Args: + value: The value (address or name) being cached + + Returns: + str: The constructed cache key that includes the layer and module name """ return f"{self._layer_name}|{self._mod_name}|{value}" - def get_name_for_address(self, address: int) -> cached_str: + def get_name_for_address(self, address: int) -> Optional[str]: + """ + Returns the name for the given address within the particular layer and module + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ cached_key = self._get_cache_key(address) if cached_key not in self._name_cache: name = self._do_get_name(address) @@ -67,7 +108,16 @@ class PESymbolFinder: return self._name_cache[cached_key] - def get_address_for_name(self, name: str) -> cached_int: + def get_address_for_name(self, name: str) -> Optional[int]: + """ + Returns the name for the given address within the particular layer and module + + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ cached_key = self._get_cache_key(name) if cached_key not in self._address_cache: address = self._do_get_address(name) @@ -75,10 +125,30 @@ class PESymbolFinder: return self._address_cache[cached_key] - def _do_get_name(self, address: int) -> cached_str: + def _do_get_name(self, address: int) -> Optional[str]: + """ + Returns the name for the given address within the particular layer and module. + This method must be overwritten by sub classes. + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ raise NotImplementedError("_do_get_name must be overwritten") - def _do_get_address(self, name: str) -> cached_int: + def _do_get_address(self, name: str) -> Optional[int]: + """ + Returns the name for the given address within the particular layer and module + This method must be overwritten by sub classes. + + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ raise NotImplementedError("_do_get_address must be overwritten") @@ -87,13 +157,31 @@ class PDBSymbolFinder(PESymbolFinder): PESymbolFinder implementation for PDB modules """ - def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + def _do_get_address(self, name: str) -> Optional[int]: + """ + _do_get_address implementation for PDBSymbolFinder + + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ try: return self._symbol_module.get_absolute_symbol_address(name) except exceptions.SymbolError: return None - def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + def _do_get_name(self, address: int) -> Optional[str]: + """ + _do_get_name implementation for PDBSymbolFinder + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ try: name = self._symbol_module.get_symbols_by_absolute_location(address)[0] return name.split(constants.BANG)[1] @@ -113,14 +201,32 @@ class ExportSymbolFinder(PESymbolFinder): except AttributeError: return None - def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + def _do_get_name(self, address: int) -> Optional[str]: + """ + _do_get_name implementation for ExportSymbolFinder + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ for export in self._symbol_module: if export.address + self._module_start == address: return self._get_name(export) return None - def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + def _do_get_address(self, name: str) -> Optional[int]: + """ + _do_get_address implementation for ExportSymbolFinder + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ + for export in self._symbol_module: sym_name = self._get_name(export) if sym_name and sym_name == name: @@ -139,25 +245,6 @@ class PESymbols(interfaces.plugins.PluginInterface): # used for special handling of the kernel PDB file. See later notes os_module_name = "ntoskrnl.exe" - # keys for specifying wanted names and/or addresses - # used for consistent access between the API and plugins - wanted_names = "names" - wanted_addresses = "addresses" - - # how wanted modules/symbols are specified, such as: - # {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} - # {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} - filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] - - # holds resolved symbols - # {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} - found_symbols_type = Dict[str, List[Tuple[str, int]]] - - # used to hold informatin about a range (VAD or kernel module) - # (start address, size, file path) - range_type = Tuple[int, int, str] - ranges_type = List[range_type] - @classmethod def get_requirements(cls) -> List: # Since we're calling the plugin, make sure we have the plugin's requirements @@ -187,13 +274,15 @@ class PESymbols(interfaces.plugins.PluginInterface): description='Module in which to resolve symbols. Use "ntoskrnl.exe" to resolve in the base kernel executable.', optional=False, ), - requirements.StringRequirement( - name="symbol", + requirements.ListRequirement( + name="symbols", + element_type=str, description="Symbol name to resolve", optional=True, ), - requirements.IntRequirement( - name="address", + requirements.ListRequirement( + name="addresses", + element_type=int, description="Address of symbol to resolve", optional=True, ), @@ -242,7 +331,15 @@ class PESymbols(interfaces.plugins.PluginInterface): ranges: ranges_type, address: int ) -> Optional[range_type]: """ - Helper for getting the range information for an address + Helper for getting the range information for an address. + Finds the range holding the `address` parameter + + Args: + address: the address to find the range for + + Returns: + Tuple[int, int, str]: The starting address, size, and file path of the range + """ for start, size, filepath in ranges: if start <= address < start + size: @@ -254,6 +351,13 @@ class PESymbols(interfaces.plugins.PluginInterface): def filepath_for_address(ranges: ranges_type, address: int) -> Optional[str]: """ Helper to get the file path for an address + + Args: + ranges: The set of VADs with mapped files to find the address + address: The address to find inside of the VADs set + + Returns: + str: The full path of the file, if found and present """ info = PESymbols.range_info_for_address(ranges, address) if info: @@ -264,9 +368,15 @@ class PESymbols(interfaces.plugins.PluginInterface): @staticmethod def filename_for_path(filepath: str) -> str: """ - Consistent way to get the filename + Consistent way to get the filename regardless of platform + + Args: + str: the file path from `filepath_for_address` + + Returns: + str: the bsae file name of the full path """ - return filepath.split("\\")[-1] + return ntpath.basename(filepath) @staticmethod def addresses_for_process_symbols( @@ -276,6 +386,18 @@ class PESymbols(interfaces.plugins.PluginInterface): symbol_table_name: str, symbols: filter_modules_type, ) -> found_symbols_type: + """ + Used to easily resolve the addresses of names inside of modules. + + See the usage of this function for system call resolution in unhooked_system_calls.py + for an easy to understand example. + + Args: + symbols: The dictionary of symbols requested by the caller + + Returns: + found_symbols_type: The dictionary of symbols that were resolved + """ collected_modules = PESymbols.get_process_modules( context, layer_name, symbol_table_name, symbols ) @@ -294,16 +416,22 @@ class PESymbols(interfaces.plugins.PluginInterface): def path_and_symbol_for_address( context: interfaces.context.ContextInterface, config_path: str, - collected_modules: Dict[str, List[Tuple[str, int, int]]], + collected_modules: collected_modules_type, ranges: ranges_type, address: int, ) -> Tuple[str, str]: """ Method for plugins to determine the file path and symbol name for a given address - collected_modules: return value from `get_kernel_modules` or `get_process_modules` - ranges: the memory ranges to examine in this layer. - address: address to resolve to its symbol name + See debugregisters.py for an example of how this function is used along with get_vads_for_process_cache + for resolving symbols in processes. + + Args: + collected_modules: return value from `get_kernel_modules` or `get_process_modules` + ranges: the memory ranges to examine in this layer. + address: address to resolve to its symbol name + Returns: + Tuple[str|renderers.NotApplicableValue|renderers.NotAvailableValue, str|renderers.NotApplicableValue|renderers.NotAvailableValue] """ if not address: @@ -317,15 +445,15 @@ class PESymbols(interfaces.plugins.PluginInterface): filename = PESymbols.filename_for_path(filepath).lower() # setup to resolve the address - filter_module: PESymbols.filter_modules_type = { - filename: {PESymbols.wanted_addresses: [address]} + filter_module: filter_modules_type = { + filename: {wanted_addresses_identifier: [address]} } found_symbols = PESymbols.find_symbols( context, config_path, filter_module, collected_modules ) - if not found_symbols or not found_symbols[filename]: + if not found_symbols or filename not in found_symbols: return renderers.NotAvailableValue(), renderers.NotAvailableValue() return filepath, found_symbols[filename][0][0] @@ -335,13 +463,18 @@ class PESymbols(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, pe_table_name: str, mod_name: str, - module_info: Tuple[str, int, int], + module_info: collected_module_instance, ) -> Optional[ExportSymbolFinder]: """ Attempts to locate symbols based on export analysis - mod_name: lower case name of the module to resolve symbols in - module_info: (layer_name, module_start, module_size) of the module to examine + Args: + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + + Returns: + Optional[ExportSymbolFinder]: If the export table can be resolved, then the ExportSymbolFinder + instance for it """ layer_name = module_info[0] @@ -361,7 +494,10 @@ class PESymbols(interfaces.plugins.PluginInterface): return None return ExportSymbolFinder( - layer_name, mod_name, module_start, pe_module.DIRECTORY_ENTRY_EXPORT.symbols + layer_name, + mod_name.lower(), + module_start, + pe_module.DIRECTORY_ENTRY_EXPORT.symbols, ) @staticmethod @@ -369,13 +505,17 @@ class PESymbols(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, config_path: str, mod_name: str, - module_info: Tuple[str, int, int], + module_info: collected_module_instance, ) -> Optional[PDBSymbolFinder]: """ - Attempts to locate symbols based on PDB analysis + Attempts to locate symbols based on PDB analysis through each layer where the mod_name module was found - mod_name: lower case name of the module to resolve symbols in - module_info: (layer_name, module_start, module_size) of the module to examine + Args: + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + + Returns: + Optional[PDBSymbolFinder]: If the export table can be resolved, then the ExportSymbolFinder """ mod_symbols = None @@ -386,15 +526,17 @@ class PESymbols(interfaces.plugins.PluginInterface): # a `ntoskrnl.exe` can have an internal PDB name of any of the ones in the following list # The code attempts to find all possible PDBs to ensure the best chance of recovery if mod_name == PESymbols.os_module_name: - pdb_names = ["ntkrnlmp.pdb", "ntkrnlpa.pdb", "ntkrpamp.pdb", "ntoskrnl.pdb"] + pdb_names = [fn + ".pdb" for fn in KERNEL_MODULE_NAMES] # for non-kernel files, replace the exe, sys, or dll extension with pdb else: + # in testing we found where some DLLs, such amsi.dll, have its PDB string as Amsi.dll + # in certain Windows versions mod_name = mod_name[:-3] + "pdb" first_upper = mod_name[0].upper() + mod_name[1:] pdb_names = [mod_name, first_upper] - # loop through each PDB name (will be just one for all but the kernel) + # loop through each PDB name (all the kernel names or the dll name as lower() + first char upper case) for pdb_name in pdb_names: try: mod_symbols = pdbutil.PDBUtility.symbol_table_from_pdb( @@ -433,11 +575,17 @@ class PESymbols(interfaces.plugins.PluginInterface): def _find_symbols_through_pdb( context: interfaces.context.ContextInterface, config_path: str, - module_instances: List[Tuple[str, int, int]], + module_instances: collected_modules_info, mod_name: str, ) -> Generator[PDBSymbolFinder, None, None]: """ - Attempts to resolve the symbols in `wanted_symbols` through PDB analysis + Attempts to resolve the symbols in `mod_name` through PDB analysis + + Args: + module_instances: the set of layers in which the module was found + mod_name: name of the module to resolve symbols in + Returns: + Generator[PDBSymbolFinder]: a PDBSymbolFinder instance for each layer in which the module was found """ for module_info in module_instances: mod_module = PESymbols._get_pdb_module( @@ -450,11 +598,17 @@ class PESymbols(interfaces.plugins.PluginInterface): def _find_symbols_through_exports( context: interfaces.context.ContextInterface, config_path: str, - module_instances: List[Tuple[str, int, int]], + module_instances: collected_modules_info, mod_name: str, ) -> Generator[ExportSymbolFinder, None, None]: """ - Attempts to resolve the symbols in `wanted_symbols` through export analysis + Attempts to resolve the symbols in `mod_name` through export analysis + + Args: + module_instances: the set of layers in which the module was found + mod_name: name of the module to resolve symbols in + Returns: + Generator[ExportSymbolFinder]: an ExportSymbolFinder instance for each layer in which the module was found """ pe_table_name = intermed.IntermediateSymbolTable.create( context, config_path, "windows", "pe", class_types=pe.class_types @@ -476,12 +630,21 @@ class PESymbols(interfaces.plugins.PluginInterface): ) -> Generator[Tuple[str, int], None, None]: """ Enumerates the symbols specified as wanted by the calling plugin + + removes entries from wanted_modules as they found to avoid PDB or export analysis after resolving all symbols + + Args: + wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. + mod_name: the name of module to resolve symbols in + + Returns: + Tuple[str, int]: the name and address of resolved symbols """ wanted_symbols = wanted_modules[mod_name] if ( - PESymbols.wanted_names not in wanted_symbols - and PESymbols.wanted_addresses not in wanted_symbols + wanted_names_identifier not in wanted_symbols + and wanted_addresses_identifier not in wanted_symbols ): vollog.warning( f"Invalid `wanted_symbols` sent to `find_symbols` for module {mod_name}. addresses and names keys both misssing." @@ -489,8 +652,8 @@ class PESymbols(interfaces.plugins.PluginInterface): return symbol_keys = [ - (PESymbols.wanted_names, "get_address_for_name"), - (PESymbols.wanted_addresses, "get_name_for_address"), + (wanted_names_identifier, "get_address_for_name"), + (wanted_addresses_identifier, "get_name_for_address"), ] for symbol_key, symbol_getter in symbol_keys: @@ -503,7 +666,7 @@ class PESymbols(interfaces.plugins.PluginInterface): ) if symbol_value: # yield out symbol name, symbol address - if symbol_key == PESymbols.wanted_names: + if symbol_key == wanted_names_identifier: yield wanted_value, symbol_value # type: ignore else: yield symbol_value, wanted_value # type: ignore @@ -521,13 +684,20 @@ class PESymbols(interfaces.plugins.PluginInterface): def _resolve_symbols_through_methods( context: interfaces.context.ContextInterface, config_path: str, - module_instances: List[Tuple[str, int, int]], + module_instances: collected_modules_info, wanted_modules: PESymbolFinder.cached_value_dict, mod_name: str, ) -> Generator[Tuple[str, int], None, None]: """ Attempts to resolve every wanted symbol in `mod_name` Every layer is enumerated for maximum chance of recovery + + Args: + module_instances: the set of layers in which the module was found + wanted_modules: The symbols to resolve tied to their module names + mod_name: name of the module to resolve symbols in + Returns: + Generator[Tuple[str, int]]: resolved symbol names and addresses """ symbol_resolving_methods = [ PESymbols._find_symbols_through_pdb, @@ -554,13 +724,19 @@ class PESymbols(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, config_path: str, wanted_modules: PESymbolFinder.cached_value_dict, - collected_modules: Dict[str, List[Tuple[str, int, int]]], + collected_modules: collected_modules_type, ) -> found_symbols_type: """ Loops through each method of symbol analysis until each wanted symbol is found Returns the resolved symbols as a dictionary that includes the name and runtime address + + Args: + wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. + collected_modules: return value from `get_kernel_modules` or `get_process_modules` + Returns: + found_symbols_type: The set of symbols resolved to their name and/or address """ - found_symbols: PESymbols.found_symbols_type = {} + found_symbols: found_symbols_type = {} for mod_name in wanted_modules: if mod_name not in collected_modules: @@ -594,11 +770,16 @@ class PESymbols(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, filter_modules: Optional[filter_modules_type], - ) -> Dict[str, List[Tuple[str, int, int]]]: + ) -> collected_modules_type: """ Walks the kernel module list and finds the session layer, base, and size of each wanted module + + Args: + filter_modules: The modules to filter the gathering to. If left as None, all kernel modules are gathered. + Returns: + collected_modules_type: The collection of modules found with at least one layer present """ - found_modules: Dict[str, List[Tuple[str, int, int]]] = {} + found_modules: collected_modules_type = {} if filter_modules: # create a tuple of module names for use with `endswith` @@ -645,16 +826,55 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_modules + @staticmethod + def get_vads_for_process_cache( + vads_cache: Dict[int, ranges_type], + owner_proc: interfaces.objects.ObjectInterface, + ) -> Optional[ranges_type]: + """ + Creates and utilizes a cache of a process' VADs for efficient lookups + + Returns the vad information of the VAD hosting the address, if found + + Args: + vads_cache: The existing cache of VADs + owner_proc: The process being inspected + Returns: + Optional[ranges_type]: The range holding the address, if found + """ + if owner_proc.vol.offset in vads_cache: + vads = vads_cache[owner_proc.vol.offset] + else: + vads = PESymbols.get_proc_vads_with_file_paths(owner_proc) + vads_cache[owner_proc.vol.offset] = vads + + # smear or terminated process + if len(vads) == 0: + return None + + return vads + @staticmethod def get_proc_vads_with_file_paths( proc: interfaces.objects.ObjectInterface, ) -> ranges_type: """ Returns a list of the process' vads that map a file - """ - vads = [] - for vad in proc.get_vad_root().traverse(): + Args: + proc: The process to gather the VADs for + + Returns: + ranges_type: The list of VADs for this process that map a file + """ + vads: ranges_type = [] + + try: + vad_root = proc.get_vad_root() + except exceptions.InvalidAddressException: + return vads + + for vad in vad_root.traverse(): filepath = vad.get_file_name() if not isinstance(filepath, str) or filepath.count("\\") == 0: continue @@ -676,6 +896,9 @@ class PESymbols(interfaces.plugins.PluginInterface): ]: """ Yields each set of vads for a process that have a file mapped, along with the process itself and its layer + + Args: + Generator[Tuple[interfaces.objects.ObjectInterface, str, ranges_type]]: Yields tuple of process objects, layers, and VADs mapping files """ procs = pslist.PsList.list_processes( context=context, @@ -689,7 +912,7 @@ class PESymbols(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: continue - vads = PESymbols.get_proc_vads_with_file_paths(proc) + vads = cls.get_proc_vads_with_file_paths(proc) yield proc, proc_layer_name, vads @@ -699,11 +922,16 @@ class PESymbols(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, filter_modules: Optional[filter_modules_type], - ) -> Dict[str, List[Tuple[str, int, int]]]: + ) -> collected_modules_type: """ Walks the process list and each process' VAD to determine the base address and size of wanted modules + + Args: + filter_modules: The modules to filter the gathering to. If left as None, all process modules are gathered. + Returns: + collected_modules_type: The collection of modules found with at least one layer present """ - proc_modules: Dict[str, List[Tuple[str, int, int]]] = {} + proc_modules: collected_modules_type = {} if filter_modules: # create a tuple of module names for use with `endswith` @@ -711,7 +939,7 @@ class PESymbols(interfaces.plugins.PluginInterface): else: filter_modules_check = None - for _, proc_layer_name, vads in PESymbols.get_all_vads_with_file_paths( + for _proc, proc_layer_name, vads in PESymbols.get_all_vads_with_file_paths( context, layer_name, symbol_table ): for vad_start, vad_size, filepath in vads: @@ -731,17 +959,17 @@ class PESymbols(interfaces.plugins.PluginInterface): def _generator(self) -> Generator[Tuple[int, Tuple[str, str, int]], None, None]: kernel = self.context.modules[self.config["kernel"]] - if self.config["symbol"]: + if self.config["symbols"]: filter_module = { self.config["module"].lower(): { - PESymbols.wanted_names: [self.config["symbol"]] + wanted_names_identifier: self.config["symbols"] } } - elif self.config["address"]: + elif self.config["addresses"]: filter_module = { self.config["module"].lower(): { - PESymbols.wanted_addresses: [self.config["address"]] + wanted_addresses_identifier: self.config["addresses"] } } diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index 0438bc9e3..68f4c4b80 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -1,3 +1,6 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + import logging from typing import Dict, Tuple, List, Generator @@ -18,7 +21,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): system_calls = { "ntdll.dll": { - pe_symbols.PESymbols.wanted_names: [ + pe_symbols.wanted_names_identifier: [ "NtCreateThread", "NtProtectVirtualMemory", "NtReadVirtualMemory", @@ -90,7 +93,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): def _gather_code_bytes( self, kernel: interfaces.context.ModuleInterface, - found_symbols: pe_symbols.PESymbols.found_symbols_type, + found_symbols: pe_symbols.found_symbols_type, ) -> _code_bytes_type: """ Enumerates the desired DLLs and function implementations in each process diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 974793a71..2c6ed4daf 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -9,7 +9,7 @@ from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist, pe_symbols +from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class VadInfo(interfaces.plugins.PluginInterface): _version = (2, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb - def __init__(self, *args, **kwargs): # type: ignore + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._protect_values = None From b40c20dd7fe0d1e348f94531c290a9aceee34dcb Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 10 Sep 2024 14:55:00 -0500 Subject: [PATCH 243/263] Revert back to union to avoid failed tests --- volatility3/framework/plugins/windows/pe_symbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index c9785a1c9..30e9b49d1 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -61,7 +61,7 @@ class PESymbolFinder: cached_int_dict = Dict[str, Optional[int]] cached_value = Union[int, str, None] - cached_value_dict = Dict[str, Dict[str, List[str]] | Dict[str, List[int]]] + cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] def __init__( self, From 037eb1ce036ae7dea48c427742ae889c9b2ec7a3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 11 Sep 2024 16:30:41 +1000 Subject: [PATCH 244/263] Linux Check_creds plugins pointer verification improvements --- volatility3/framework/plugins/linux/check_creds.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index ab6ee4935..45df966d2 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -16,6 +16,8 @@ class Check_creds(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) + @classmethod def get_requirements(cls): return [ @@ -46,7 +48,11 @@ class Check_creds(interfaces.plugins.PluginInterface): tasks = pslist.PsList.list_tasks(self.context, vmlinux.name) for task in tasks: - cred_addr = task.cred.dereference().vol.offset + task_cred_ptr = task.cred + if not (task_cred_ptr and task_cred_ptr.is_readable()): + continue + + cred_addr = task_cred_ptr.dereference().vol.offset if cred_addr not in creds: creds[cred_addr] = [] From c77c662b70c6751087bf947c400a045c81e7a8ec Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 11 Sep 2024 21:09:08 +1000 Subject: [PATCH 245/263] Linux pidhashtable plugin pointer verification improvements --- .../framework/plugins/linux/pidhashtable.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 3223aed4a..edafe97e0 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -20,7 +20,7 @@ class PIDHashTable(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -45,9 +45,7 @@ class PIDHashTable(plugins.PluginInterface): ] def _is_valid_task(self, task) -> bool: - vmlinux = self.context.modules[self.config["kernel"]] - vmlinux_layer = self.context.layers[vmlinux.layer_name] - return bool(task and task.pid > 0 and vmlinux_layer.is_valid(task.parent)) + return bool(task and task.pid > 0 and task.parent.is_readable()) def _get_pidtype_pid(self): vmlinux = self.context.modules[self.config["kernel"]] @@ -96,7 +94,7 @@ class PIDHashTable(plugins.PluginInterface): seen_upids.add(upid.vol.offset) pid_chain = upid.pid_chain - if not (pid_chain and vmlinux_layer.is_valid(pid_chain.vol.offset)): + if not (pid_chain.next and pid_chain.next.is_readable()): break upid = linux.LinuxUtilities.container_of( @@ -105,7 +103,6 @@ class PIDHashTable(plugins.PluginInterface): def _get_upids(self): vmlinux = self.context.modules[self.config["kernel"]] - vmlinux_layer = self.context.layers[vmlinux.layer_name] # 2.6.24 <= kernels < 4.15 pidhash = self._get_pidhash_array() @@ -115,7 +112,7 @@ class PIDHashTable(plugins.PluginInterface): # each entry in the hlist is a upid which is wrapped in a pid ent = hlist.first - while ent and vmlinux_layer.is_valid(ent.vol.offset): + while ent and ent.is_readable(): # upid->pid_chain exists 2.6.24 <= kernel < 4.15 upid = linux.LinuxUtilities.container_of( ent.vol.offset, "upid", "pid_chain", vmlinux @@ -143,7 +140,7 @@ class PIDHashTable(plugins.PluginInterface): continue pid_tasks_0 = pid.tasks[pidtype_pid].first - if not pid_tasks_0: + if not (pid_tasks_0 and pid_tasks_0.is_readable()): continue task = vmlinux.object( @@ -160,7 +157,7 @@ class PIDHashTable(plugins.PluginInterface): pidtype_pid = self._get_pidtype_pid() pid_tasks_0 = pid.tasks[pidtype_pid].first - if not pid_tasks_0: + if not (pid_tasks_0 and pid_tasks_0.is_readable()): return None task_struct_type = vmlinux.get_type("task_struct") From 9e8471799adcf090e20ea98a20309076193e9009 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 10:44:43 +1000 Subject: [PATCH 246/263] Improving code and adding the credential virtual addresses to the output. --- .../framework/plugins/linux/check_creds.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 45df966d2..3e292ae33 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -2,21 +2,18 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import logging - from volatility3.framework import interfaces, renderers +from volatility3.framework.renderers import format_hints from volatility3.framework.configuration import requirements from volatility3.plugins.linux import pslist -vollog = logging.getLogger(__name__) - class Check_creds(interfaces.plugins.PluginInterface): """Checks if any processes are sharing credential structures""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls): @@ -54,18 +51,22 @@ class Check_creds(interfaces.plugins.PluginInterface): cred_addr = task_cred_ptr.dereference().vol.offset - if cred_addr not in creds: - creds[cred_addr] = [] - + creds.setdefault(cred_addr, []) creds[cred_addr].append(task.pid) - for _, pids in creds.items(): + for cred_addr, pids in creds.items(): if len(pids) > 1: - pid_str = "" - for pid in pids: - pid_str = pid_str + f"{pid:d}, " - pid_str = pid_str[:-2] - yield (0, [str(pid_str)]) + pid_str = ", ".join([str(pid) for pid in pids]) + + fields = [ + format_hints.Hex(cred_addr), + pid_str, + ] + yield (0, fields) def run(self): - return renderers.TreeGrid([("PIDs", str)], self._generator()) + headers = [ + ("CredVAddr", format_hints.Hex), + ("PIDs", str), + ] + return renderers.TreeGrid(headers, self._generator()) From 57de357ffdfe87dbcad8c219228a4a0d0e17c173 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 16:05:40 +1000 Subject: [PATCH 247/263] Timeliner plugin: Fix issue with filtering TimeLinerInterface plugins and using the filter argument --- volatility3/framework/plugins/timeliner.py | 23 +++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index f657a2918..abe802e8f 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -45,6 +45,7 @@ class Timeliner(interfaces.plugins.PluginInterface): orders the results by time.""" _required_framework_version = (2, 0, 0) + _version = (1, 1, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -245,6 +246,17 @@ class Timeliner(interfaces.plugins.PluginInterface): filter_list = self.config["plugin-filter"] # Identify plugins that we can run which output datetimes for plugin_class in self.usable_plugins: + if not issubclass(plugin_class, TimeLinerInterface): + continue + + if filter_list and not any( + [ + filter in plugin_class.__module__ + "." + plugin_class.__name__ + for filter in filter_list + ] + ): + continue + try: automagics = automagic.choose_automagic(self.automagics, plugin_class) @@ -276,15 +288,8 @@ class Timeliner(interfaces.plugins.PluginInterface): config_value, ) - if isinstance(plugin, TimeLinerInterface): - if not len(filter_list) or any( - [ - filter - in plugin.__module__ + "." + plugin.__class__.__name__ - for filter in filter_list - ] - ): - plugins_to_run.append(plugin) + plugins_to_run.append(plugin) + except exceptions.UnsatisfiedException as excp: # Remove the failed plugin from the list and continue vollog.debug( From be05ace29b134156fe5f7584921887426fc2f41f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 16:06:40 +1000 Subject: [PATCH 248/263] Timeliner plugin: Add exception information --- volatility3/framework/plugins/timeliner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index abe802e8f..56fe465e4 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -199,9 +199,10 @@ class Timeliner(interfaces.plugins.PluginInterface): ), ) ) - except Exception: + except Exception as e: vollog.log( - logging.INFO, f"Exception occurred running plugin: {plugin_name}" + logging.INFO, + f"Exception occurred running plugin: {plugin_name}: {e}", ) vollog.log(logging.DEBUG, traceback.format_exc()) From a7dcd6d9e8adfe3124aa13db1b1536e6799d822c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 16:19:13 +1000 Subject: [PATCH 249/263] Minor: Add comment on TimeLinerInterface subclass filter --- volatility3/framework/plugins/timeliner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 56fe465e4..d1cb9f460 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -248,6 +248,7 @@ class Timeliner(interfaces.plugins.PluginInterface): # Identify plugins that we can run which output datetimes for plugin_class in self.usable_plugins: if not issubclass(plugin_class, TimeLinerInterface): + # get_usable_plugins() should filter this, but adding a safeguard just in case continue if filter_list and not any( From 48ae43d64edd457b65eb40174fb00b54202aabda Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 17:22:52 +1000 Subject: [PATCH 250/263] Bumping the major version since the output changed --- volatility3/framework/plugins/linux/check_creds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 3e292ae33..b7f73c3eb 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -13,7 +13,7 @@ class Check_creds(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): From 997abeda6d9014a028f6c2b7a9e11352320c142f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 17:27:02 +1000 Subject: [PATCH 251/263] Linux lsof: Add namespace dentry name --- .../framework/symbols/linux/__init__.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 91abf7db4..57b45667e 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -169,13 +169,30 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): Returns: str: Sock pipe pathname relative to the task's root directory. """ + # FIXME: This function must be moved to the 'dentry' object extension + # Also, the scope of this function went beyond the sock pipe path, so we need to rename this. + # Once https://github.com/volatilityfoundation/volatility3/pull/1263 is merged, replace the + # dentry inode getters + + if not (filp and filp.is_readable()): + return f" {filp:x}" + dentry = filp.get_dentry() + if not (dentry and dentry.is_readable()): + return f" {dentry:x}" kernel_module = cls.get_module_from_volobj_type(context, dentry) sym_addr = dentry.d_op.d_dname + if not (sym_addr and sym_addr.is_readable()): + return f" {sym_addr:x}" + symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) + inode = dentry.d_inode + if not (inode and inode.is_readable() and inode.is_valid()): + return f" {inode:x}" + if len(symbs) == 1: sym = symbs[0].split(constants.BANG)[1] @@ -191,13 +208,36 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): elif sym == "simple_dname": pre_name = cls._get_path_file(task, filp) - else: - pre_name = f"" + elif sym == "ns_dname": + # From Kernels 3.19 - ret = f"{pre_name}:[{dentry.d_inode.i_ino:d}]" + # In Kernels >= 6.9, see Linux kernel commit 1fa08aece42512be072351f482096d5796edf7ca + # ns_common->stashed change from 'atomic64_t' to 'dentry*' + try: + ns_common_type = kernel_module.get_type("ns_common") + stashed_template = ns_common_type.child_template("stashed") + stashed_type_full_name = stashed_template.vol.type_name + stashed_type_name = stashed_type_full_name.split(constants.BANG)[-1] + if stashed_type_name == "atomic64_t": + # 3.19 <= Kernels < 6.9 + ns_ops = dentry.d_fsdata.dereference().cast( + "proc_ns_operations" + ) + else: + # Kernels >= 6.9 + ns_common = inode.i_private.dereference().cast("ns_common") + ns_ops = ns_common.ops + + pre_name = utility.pointer_to_string(ns_ops.name, 255) + except IndexError: + ret = "" + else: + pre_name = f" {sym}" + + ret = f"{pre_name}:[{inode.i_ino:d}]" else: - ret = f" {sym_addr:x}" + ret = f" {sym_addr:x}" return ret From cd2af74e6d0c554e81d1e67a8020195cfca59983 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 17:58:21 +1000 Subject: [PATCH 252/263] Improve pointers address verification and return message chain --- .../framework/symbols/linux/__init__.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 57b45667e..2410d2627 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -217,29 +217,32 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ns_common_type = kernel_module.get_type("ns_common") stashed_template = ns_common_type.child_template("stashed") stashed_type_full_name = stashed_template.vol.type_name - stashed_type_name = stashed_type_full_name.split(constants.BANG)[-1] + stashed_type_name = stashed_type_full_name.split(constants.BANG)[1] if stashed_type_name == "atomic64_t": # 3.19 <= Kernels < 6.9 - ns_ops = dentry.d_fsdata.dereference().cast( - "proc_ns_operations" - ) + fsdata_ptr = dentry.d_fsdata + if not (fsdata_ptr and fsdata_ptr.is_readable()): + raise IndexError + + ns_ops = fsdata_ptr.dereference().cast("proc_ns_operations") else: # Kernels >= 6.9 - ns_common = inode.i_private.dereference().cast("ns_common") + private_ptr = inode.i_private + if not (private_ptr and private_ptr.is_readable()): + raise IndexError + + ns_common = private_ptr.dereference().cast("ns_common") ns_ops = ns_common.ops pre_name = utility.pointer_to_string(ns_ops.name, 255) except IndexError: - ret = "" + pre_name = "" else: pre_name = f" {sym}" - - ret = f"{pre_name}:[{inode.i_ino:d}]" - else: - ret = f" {sym_addr:x}" + pre_name = f" {sym_addr:x}" - return ret + return f"{pre_name}:[{inode.i_ino:d}]" @classmethod def path_for_file(cls, context, task, filp) -> str: From 67ee382c3229f10d4e29958b6a5bf257e29ed8f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 13 Sep 2024 18:53:33 +0200 Subject: [PATCH 253/263] use default req value in config_value call --- volatility3/framework/interfaces/configuration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 3bb3cb019..da0a4556c 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -494,8 +494,7 @@ class SimpleTypeRequirement(RequirementInterface): """Validates the instance requirement based upon its `instance_type`.""" config_path = path_join(config_path, self.name) - - value = self.config_value(context, config_path, None) + value = self.config_value(context, config_path, self.default) if not isinstance(value, self.instance_type): vollog.log( constants.LOGLEVEL_V, @@ -536,7 +535,7 @@ class ClassRequirement(RequirementInterface): """Checks to see if a class can be recovered.""" config_path = path_join(config_path, self.name) - value = self.config_value(context, config_path, None) + value = self.config_value(context, config_path, self.default) self._cls = None if value is not None and isinstance(value, str): if "." in value: From ba0c975e73207ee6555bd80067460ddbea6426a2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 13 Sep 2024 18:50:47 -0500 Subject: [PATCH 254/263] Address all feedback --- .../framework/plugins/windows/pe_symbols.py | 126 +++++++++--------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 30e9b49d1..85bfb572e 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -1,11 +1,12 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +import copy import io import logging import ntpath -from typing import Dict, Tuple, Optional, List, Generator, Union +from typing import Dict, Tuple, Optional, List, Generator, Union, Callable import pefile @@ -29,11 +30,13 @@ wanted_addresses_identifier = "addresses" # how wanted modules/symbols are specified, such as: # {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} # {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} -filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] +filter_module_info = Union[Dict[str, List[str]], Dict[str, List[int]]] +filter_modules_type = Dict[str, filter_module_info] # holds resolved symbols # {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} -found_symbols_type = Dict[str, List[Tuple[str, int]]] +found_symbols_module = List[Tuple[str, int]] +found_symbols_type = Dict[str, found_symbols_module] # used to hold informatin about a range (VAD or kernel module) # (start address, size, file path) @@ -61,7 +64,8 @@ class PESymbolFinder: cached_int_dict = Dict[str, Optional[int]] cached_value = Union[int, str, None] - cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + cached_module_lists = Union[Dict[str, List[str]], Dict[str, List[int]]] + cached_value_dict = Dict[str, cached_module_lists] def __init__( self, @@ -402,11 +406,11 @@ class PESymbols(interfaces.plugins.PluginInterface): context, layer_name, symbol_table_name, symbols ) - found_symbols = PESymbols.find_symbols( + found_symbols, missing_symbols = PESymbols.find_symbols( context, config_path, symbols, collected_modules ) - for mod_name, unresolved_symbols in symbols.items(): + for mod_name, unresolved_symbols in missing_symbols.items(): for symbol in unresolved_symbols: vollog.debug(f"Unable to resolve symbol {symbol} in module {mod_name}") @@ -449,7 +453,7 @@ class PESymbols(interfaces.plugins.PluginInterface): filename: {wanted_addresses_identifier: [address]} } - found_symbols = PESymbols.find_symbols( + found_symbols, _missing_msybols = PESymbols.find_symbols( context, config_path, filter_module, collected_modules ) @@ -624,61 +628,46 @@ class PESymbols(interfaces.plugins.PluginInterface): @staticmethod def _get_symbol_value( - wanted_modules: PESymbolFinder.cached_value_dict, - mod_name: str, + wanted_symbols: filter_module_info, symbol_resolver: PESymbolFinder, - ) -> Generator[Tuple[str, int], None, None]: + ) -> Generator[Tuple[str, int, str, int], None, None]: """ Enumerates the symbols specified as wanted by the calling plugin - removes entries from wanted_modules as they found to avoid PDB or export analysis after resolving all symbols - Args: - wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. - mod_name: the name of module to resolve symbols in + wanted_symbols: the set of symbols for a particular module + symbol_resolver: method in a layer to resolve the symbols Returns: - Tuple[str, int]: the name and address of resolved symbols + Tuple[str, int, str, int]: the index and value of the found symbol in the wanted list, and the name and address of resolved symbol """ - wanted_symbols = wanted_modules[mod_name] - if ( wanted_names_identifier not in wanted_symbols and wanted_addresses_identifier not in wanted_symbols ): vollog.warning( - f"Invalid `wanted_symbols` sent to `find_symbols` for module {mod_name}. addresses and names keys both misssing." + f"Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." ) return - symbol_keys = [ - (wanted_names_identifier, "get_address_for_name"), - (wanted_addresses_identifier, "get_name_for_address"), + symbol_keys: List[Tuple[str, Callable]] = [ + (wanted_names_identifier, symbol_resolver.get_address_for_name), + (wanted_addresses_identifier, symbol_resolver.get_name_for_address), ] for symbol_key, symbol_getter in symbol_keys: # address or name if symbol_key in wanted_symbols: # walk each wanted address or name - for wanted_value in wanted_symbols[symbol_key]: - symbol_value = symbol_resolver.__getattribute__(symbol_getter)( - wanted_value - ) + for value_index, wanted_value in enumerate(wanted_symbols[symbol_key]): + symbol_value = symbol_getter(wanted_value) + if symbol_value: - # yield out symbol name, symbol address + # yield out deleteion key, deletion index, symbol name, symbol address if symbol_key == wanted_names_identifier: - yield wanted_value, symbol_value # type: ignore + yield symbol_key, value_index, wanted_value, symbol_value # type: ignore else: - yield symbol_value, wanted_value # type: ignore - - index = wanted_modules[mod_name][symbol_key].index(wanted_value) # type: ignore - - del wanted_modules[mod_name][symbol_key][index] - - # if all names or addresses from a module are found, delete the key - if not wanted_modules[mod_name][symbol_key]: - del wanted_modules[mod_name][symbol_key] - break + yield symbol_key, value_index, symbol_value, wanted_value # type: ignore @staticmethod def _resolve_symbols_through_methods( @@ -687,7 +676,7 @@ class PESymbols(interfaces.plugins.PluginInterface): module_instances: collected_modules_info, wanted_modules: PESymbolFinder.cached_value_dict, mod_name: str, - ) -> Generator[Tuple[str, int], None, None]: + ) -> Tuple[found_symbols_module, PESymbolFinder.cached_module_lists]: """ Attempts to resolve every wanted symbol in `mod_name` Every layer is enumerated for maximum chance of recovery @@ -697,35 +686,53 @@ class PESymbols(interfaces.plugins.PluginInterface): wanted_modules: The symbols to resolve tied to their module names mod_name: name of the module to resolve symbols in Returns: - Generator[Tuple[str, int]]: resolved symbol names and addresses + Tuple[found_symbols_module, PESymbolFinder.cached_module_lists]: The set of found symbols and the ones that could not be resolved """ symbol_resolving_methods = [ PESymbols._find_symbols_through_pdb, PESymbols._find_symbols_through_exports, ] + found: found_symbols_module = [] + + # the symbols wanted from this module by the caller + wanted = wanted_modules[mod_name] + + # make a copy to remove from inside this function for returning to the caller + remaining = copy.deepcopy(wanted) + for method in symbol_resolving_methods: + # every layer where this module was found through the given method for symbol_resolver in method( context, config_path, module_instances, mod_name ): vollog.debug(f"Have resolver for method {method}") - yield from PESymbols._get_symbol_value( - wanted_modules, mod_name, symbol_resolver - ) + for ( + symbol_key, + value_index, + symbol_name, + symbol_address, + ) in PESymbols._get_symbol_value(remaining, symbol_resolver): + found.append((symbol_name, symbol_address)) + del remaining[symbol_key][value_index] - if not wanted_modules[mod_name]: + # everything was resolved, stop this resolver + if not remaining: break - if not wanted_modules[mod_name]: + # stop all resolving + if not remaining: break + return found, remaining + @staticmethod def find_symbols( context: interfaces.context.ContextInterface, config_path: str, wanted_modules: PESymbolFinder.cached_value_dict, collected_modules: collected_modules_type, - ) -> found_symbols_type: + ) -> Tuple[found_symbols_type, PESymbolFinder.cached_value_dict]: """ Loops through each method of symbol analysis until each wanted symbol is found Returns the resolved symbols as a dictionary that includes the name and runtime address @@ -734,9 +741,10 @@ class PESymbols(interfaces.plugins.PluginInterface): wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. collected_modules: return value from `get_kernel_modules` or `get_process_modules` Returns: - found_symbols_type: The set of symbols resolved to their name and/or address + Tuple[found_symbols_type, PESymbolFinder.cached_value_dict]: The set of found symbols but the ones that could not be resolved """ found_symbols: found_symbols_type = {} + missing_symbols: PESymbolFinder.cached_value_dict = {} for mod_name in wanted_modules: if mod_name not in collected_modules: @@ -745,24 +753,20 @@ class PESymbols(interfaces.plugins.PluginInterface): module_instances = collected_modules[mod_name] # try to resolve the symbols for `mod_name` through each method (PDB and export table currently) - for symbol_name, address in PESymbols._resolve_symbols_through_methods( + ( + found_in_module, + missing_in_module, + ) = PESymbols._resolve_symbols_through_methods( context, config_path, module_instances, wanted_modules, mod_name - ): - if mod_name not in found_symbols: - found_symbols[mod_name] = [] + ) - found_symbols[mod_name].append((symbol_name, address)) + if found_in_module: + found_symbols[mod_name] = found_in_module - # stop processing the layers (processes) if we found all the symbols for this module - if not wanted_modules[mod_name]: - break + if missing_in_module: + missing_symbols[mod_name] = missing_in_module - # stop processing this module if/when all symbols are found - if not wanted_modules[mod_name]: - del wanted_modules[mod_name] - break - - return found_symbols + return found_symbols, missing_symbols @staticmethod def get_kernel_modules( @@ -986,7 +990,7 @@ class PESymbols(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, filter_module ) - found_symbols = PESymbols.find_symbols( + found_symbols, _missing_symbols = PESymbols.find_symbols( self.context, self.config_path, filter_module, collected_modules ) From 21d21cf4b8ec3aafa5e98c563b207b21c41d5c3f Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 16:26:38 -0500 Subject: [PATCH 255/263] Break properly in all paths. Help callers to ensure always lower case module name matching. --- volatility3/framework/plugins/windows/pe_symbols.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 85bfb572e..d0ddcac57 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -380,7 +380,7 @@ class PESymbols(interfaces.plugins.PluginInterface): Returns: str: the bsae file name of the full path """ - return ntpath.basename(filepath) + return ntpath.basename(filepath).lower() @staticmethod def addresses_for_process_symbols( @@ -717,11 +717,12 @@ class PESymbols(interfaces.plugins.PluginInterface): del remaining[symbol_key][value_index] # everything was resolved, stop this resolver - if not remaining: + if not remaining[symbol_key]: break # stop all resolving - if not remaining: + if not remaining[symbol_key]: + del remaining[symbol_key] break return found, remaining From 291bc878ad771388f3514b2c66d31b7325b2bc01 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 16:31:33 -0500 Subject: [PATCH 256/263] Break in a cleaner flow --- volatility3/framework/plugins/windows/pe_symbols.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index d0ddcac57..44254513f 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -701,6 +701,8 @@ class PESymbols(interfaces.plugins.PluginInterface): # make a copy to remove from inside this function for returning to the caller remaining = copy.deepcopy(wanted) + done_processing = False + for method in symbol_resolving_methods: # every layer where this module was found through the given method for symbol_resolver in method( @@ -717,12 +719,14 @@ class PESymbols(interfaces.plugins.PluginInterface): del remaining[symbol_key][value_index] # everything was resolved, stop this resolver + # remove this key from the remaining symbols to resolve if not remaining[symbol_key]: + del remaining[symbol_key] + done_processing = True break # stop all resolving - if not remaining[symbol_key]: - del remaining[symbol_key] + if done_processing: break return found, remaining From 322f79fb5040e3e7ebdfa9de8c82a639729a39f2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 17:18:26 -0500 Subject: [PATCH 257/263] Bail as early as possible --- .../framework/plugins/windows/pe_symbols.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 44254513f..0bf03e7d6 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -437,7 +437,6 @@ class PESymbols(interfaces.plugins.PluginInterface): Returns: Tuple[str|renderers.NotApplicableValue|renderers.NotAvailableValue, str|renderers.NotApplicableValue|renderers.NotAvailableValue] """ - if not address: return renderers.NotApplicableValue(), renderers.NotApplicableValue() @@ -458,7 +457,7 @@ class PESymbols(interfaces.plugins.PluginInterface): ) if not found_symbols or filename not in found_symbols: - return renderers.NotAvailableValue(), renderers.NotAvailableValue() + return filepath, renderers.NotAvailableValue() return filepath, found_symbols[filename][0][0] @@ -718,11 +717,14 @@ class PESymbols(interfaces.plugins.PluginInterface): found.append((symbol_name, symbol_address)) del remaining[symbol_key][value_index] - # everything was resolved, stop this resolver - # remove this key from the remaining symbols to resolve - if not remaining[symbol_key]: - del remaining[symbol_key] - done_processing = True + # everything was resolved, stop this resolver + # remove this key from the remaining symbols to resolve + if not remaining[symbol_key]: + del remaining[symbol_key] + done_processing = True + break + + if done_processing: break # stop all resolving @@ -885,6 +887,7 @@ class PESymbols(interfaces.plugins.PluginInterface): for vad in vad_root.traverse(): filepath = vad.get_file_name() + if not isinstance(filepath, str) or filepath.count("\\") == 0: continue From 79b8ff7d05b56316d90be682f56db22401f2c265 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 15 Sep 2024 18:29:05 -0500 Subject: [PATCH 258/263] Address final feedback --- .../plugins/windows/debugregisters.py | 23 ++++++++----------- .../framework/plugins/windows/pe_symbols.py | 10 ++++---- .../plugins/windows/unhooked_system_calls.py | 8 +++++++ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 65b2e625b..945ba1df0 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -148,12 +148,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): file3, sym3 = path_and_symbol(vads, dr3) # if none map to an actual file VAD then bail - if not ( - isinstance(file0, str) - or isinstance(file1, str) - or isinstance(file2, str) - or isinstance(file3, str) - ): + if not (file0 or file1 or file2 or file3): continue process_name = owner_proc.ImageFileName.cast( @@ -173,17 +168,17 @@ class DebugRegisters(interfaces.plugins.PluginInterface): thread.Tcb.State, dr7, format_hints.Hex(dr0), - file0, - sym0, + file0 or renderers.NotApplicableValue(), + sym0 or renderers.NotApplicableValue(), format_hints.Hex(dr1), - file1, - sym1, + file1 or renderers.NotApplicableValue(), + sym1 or renderers.NotApplicableValue(), format_hints.Hex(dr2), - file2, - sym2, + file2 or renderers.NotApplicableValue(), + sym2 or renderers.NotApplicableValue(), format_hints.Hex(dr3), - file3, - sym3, + file3 or renderers.NotApplicableValue(), + sym3 or renderers.NotApplicableValue(), ), ) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 0bf03e7d6..955098d6b 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -423,7 +423,7 @@ class PESymbols(interfaces.plugins.PluginInterface): collected_modules: collected_modules_type, ranges: ranges_type, address: int, - ) -> Tuple[str, str]: + ) -> Tuple[Optional[str], Optional[str]]: """ Method for plugins to determine the file path and symbol name for a given address @@ -438,12 +438,12 @@ class PESymbols(interfaces.plugins.PluginInterface): Tuple[str|renderers.NotApplicableValue|renderers.NotAvailableValue, str|renderers.NotApplicableValue|renderers.NotAvailableValue] """ if not address: - return renderers.NotApplicableValue(), renderers.NotApplicableValue() + return None, None filepath = PESymbols.filepath_for_address(ranges, address) if not filepath: - return renderers.NotAvailableValue(), renderers.NotAvailableValue() + return None, None filename = PESymbols.filename_for_path(filepath).lower() @@ -452,12 +452,12 @@ class PESymbols(interfaces.plugins.PluginInterface): filename: {wanted_addresses_identifier: [address]} } - found_symbols, _missing_msybols = PESymbols.find_symbols( + found_symbols, _missing_symbols = PESymbols.find_symbols( context, config_path, filter_module, collected_modules ) if not found_symbols or filename not in found_symbols: - return filepath, renderers.NotAvailableValue() + return filepath, None return filepath, found_symbols[filename][0][0] diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index 68f4c4b80..c3d98254d 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -71,6 +71,13 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): } } + # This data structure is used to track unique implementations of functions across processes + # The outer dictionary holds the module name (e.g., ntdll.dll) + # The next dictionary holds the function names (NtTerminateProcess, NtSetValueKey, etc.) inside a module + # The innermost dictionary holds the unique implementation (bytes) of a function across processes + # Each implementation is tracked along with the process(es) that host it + # For systems without malware, all functions should have the same implementation + # When API hooking/module unhooking is done, the victim (infected) processes will have unique implementations _code_bytes_type = Dict[str, Dict[str, Dict[bytes, List[Tuple[int, str]]]]] @classmethod @@ -127,6 +134,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: continue + # see the definition of _code_bytes_type for details of this data structure if dll_name not in code_bytes: code_bytes[dll_name] = {} From b788733683256e2cfd436750f616054f707252e9 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 15 Sep 2024 19:48:55 -0500 Subject: [PATCH 259/263] More comments on unhooked system calls --- .../plugins/windows/debugregisters.py | 4 ++++ .../plugins/windows/unhooked_system_calls.py | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 945ba1df0..57dd1822c 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -1,6 +1,10 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# Full details on the techniques used in these plugins to detect EDR-evading malware +# can be found in our 20 page whitepaper submitted to DEFCON along with the presentation +# https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + import logging from typing import Tuple, Optional, Generator, List, Dict diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index c3d98254d..1a1e59940 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -1,6 +1,10 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# Full details on the techniques used in these plugins to detect EDR-evading malware +# can be found in our 20 page whitepaper submitted to DEFCON along with the presentation +# https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + import logging from typing import Dict, Tuple, List, Generator @@ -162,20 +166,30 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): # code_bytes[dll_name][func_name][func_bytes] code_bytes = self._gather_code_bytes(kernel, found_symbols) + # walk the functions that were evaluated for functions in code_bytes.values(): + # cbb is the distinct groups of bytes (instructions) + # for this function across processes for func_name, cbb in functions.items(): + # the dict key here is the raw instructions, which is not helpful to look at + # the values are the list of tuples for the (proc_id, proc_name) pairs for this set of bytes (instructions) cb = list(cbb.values()) - # same implementation in all + # if all processes map to the same implementation, then no malware is present if len(cb) == 1: yield 0, (func_name, "", len(cb[0])) else: - # find the processes that are hooked for reporting + # if there are differing implementations then it means + # that malware has overwritten system call(s) in infected processes + # max_idx and small_idx find which implementation of a system call has the least processes + # as all observed malware and open source projects only infected a few targets, leaving the + # rest with the original EDR hooks in place max_idx = 0 if len(cb[0]) > len(cb[1]) else 1 small_idx = (~max_idx) & 1 ps = [] + # gather processes on small_idx since these are the malware infected ones for pid, pname in cb[small_idx]: ps.append("{:d}:{}".format(pid, pname)) From 10ac21da2cbca02d47dc9aa938c2fe0d560af23d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 16 Sep 2024 12:16:43 +0100 Subject: [PATCH 260/263] Windows: Remove the unnecessary requirement on verinfo Fixes #1267 --- volatility3/framework/plugins/windows/verinfo.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 5b3c52bf6..57b8dcd3f 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -48,9 +48,6 @@ class VerInfo(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="modules", plugin=modules.Modules, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="dlllist", component=dlllist.DllList, version=(2, 0, 0) - ), requirements.BooleanRequirement( name="extensive", description="Search physical layer for version information", From f77003b670be71e012d0caa52f936a5adba8f162 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Sep 2024 12:50:58 +1000 Subject: [PATCH 261/263] Fix changes introduced to volatility3.framework.constants in PRs #838 and #1247 --- volatility3/framework/constants/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 8743a64b0..27fae4ba1 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -134,4 +134,5 @@ def __getattr__(name): ]: warnings.warn(f"{name} is deprecated", FutureWarning) return globals()[f"{deprecated_tag}{name}"] - return None + + return getattr(__import__(__name__), name) From 09fa859a92878b5d035e0dd5c44c0521661630fc Mon Sep 17 00:00:00 2001 From: eve Date: Wed, 25 Sep 2024 09:02:51 +0100 Subject: [PATCH 262/263] Windows: change warnings around large memory maps to debug level as per issue #1256 --- volatility3/framework/plugins/windows/vadyarascan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 7bc3377c3..efcc70d07 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -18,7 +18,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -68,7 +68,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): layer = self.context.layers[layer_name] for start, size in self.get_vad_maps(task): if size > sanity_check: - vollog.warn( + vollog.debug( f"VAD at 0x{start:x} over sanity-check size, not scanning" ) continue From 7f37135739c9ff951e73680f7cdf4333b47ee231 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 25 Sep 2024 13:13:33 -0500 Subject: [PATCH 263/263] Linux: Update sockstat to render process names Currently, process names are not displayed for sockets in the sockstat plugin, making analysis more painful than it needs to be. This updates the `list_sockets` classmethod and the `generator` method to return the process name in addition to the PID. Because this is changing the public interface, this commit includes a major version bump for `linux.sockstat.Sockstat`. --- volatility3/framework/plugins/linux/sockstat.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index b0503b105..d3efc78dd 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -22,7 +22,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) def __init__(self, vmlinux, task): self._vmlinux = vmlinux @@ -507,7 +507,7 @@ class Sockstat(plugins.PluginInterface): dfop_addr = vmlinux.object_from_symbol("sockfs_dentry_operations").vol.offset fd_generator = lsof.Lsof.list_fds(context, vmlinux.name, filter_func) - for _pid, _task_comm, task, fd_fields in fd_generator: + for _pid, task_comm, task, fd_fields in fd_generator: fd_num, filp, _full_path = fd_fields if filp.f_op not in (sfop_addr, dfop_addr): @@ -548,7 +548,7 @@ class Sockstat(plugins.PluginInterface): except AttributeError: netns_id = NotAvailableValue() - yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields + yield task_comm, task, netns_id, fd_num, family, sock_type, protocol, sock_fields def _format_fields(self, sock_stat, protocol): """Prepare the socket fields to be rendered @@ -595,6 +595,7 @@ class Sockstat(plugins.PluginInterface): ) for ( + task_comm, task, netns_id, fd_num, @@ -617,6 +618,7 @@ class Sockstat(plugins.PluginInterface): fields = ( netns_id, + task_comm, task.pid, fd_num, format_hints.Hex(sock.vol.offset), @@ -636,6 +638,7 @@ class Sockstat(plugins.PluginInterface): tree_grid_args = [ ("NetNS", int), + ("Process Name", str), ("Pid", int), ("FD", int), ("Sock Offset", format_hints.Hex),