From 02bf609fd14f64161ca5ad648860ed530b794d6b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 17:50:52 +1100 Subject: [PATCH 001/526] Prepare linux.lsof.Lsof plugin to work as a helper library for other plugin. --- volatility3/framework/plugins/linux/lsof.py | 51 ++++++++++++++------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index a074f5744..9ce7027f3 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -4,7 +4,7 @@ """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" import logging -from typing import List +from typing import List, Callable from volatility3.framework import renderers, interfaces, constants from volatility3.framework.configuration import requirements @@ -21,6 +21,8 @@ class Lsof(plugins.PluginInterface): _required_framework_version = (2, 0, 0) + _version = (2, 0, 0) + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ @@ -34,26 +36,43 @@ class Lsof(plugins.PluginInterface): optional = True) ] - def _generator(self, tasks): - symbol_table = None - for task in tasks: - if symbol_table is None: + @classmethod + def list_fds(cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + filter_func: Callable[[int], bool] = lambda _: False): + + linuxutils_symbol_table = None # type: ignore + for task in pslist.PsList.list_tasks(context, symbol_table, filter_func): + if linuxutils_symbol_table is None: if constants.BANG not in task.vol.type_name: raise ValueError("Task is not part of a symbol table") - symbol_table = task.vol.type_name.split(constants.BANG)[0] + linuxutils_symbol_table = task.vol.type_name.split(constants.BANG)[0] - name = utility.array_to_string(task.comm) + task_comm = utility.array_to_string(task.comm) pid = int(task.pid) - for fd_num, _, full_path in linux.LinuxUtilities.files_descriptors_for_process( - self.context, symbol_table, task): - yield (0, (pid, name, fd_num, full_path)) + fd_generator = linux.LinuxUtilities.files_descriptors_for_process( + context, + linuxutils_symbol_table, + task) - def run(self): + for fd_fields in fd_generator: + yield pid, task_comm, task, fd_fields + + def _generator(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - return renderers.TreeGrid([("PID", int), ("Process", str), ("FD", int), ("Path", str)], - self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + fds_generator = self.list_fds(self.context, + self.config['kernel'], + filter_func=filter_func) + + for pid, task_comm, _task, fd_fields in fds_generator: + fd_num, _filp, full_path = fd_fields + + fields = (pid, task_comm, fd_num, full_path) + yield (0, fields) + + def run(self): + tree_grid_args = [("PID", int), ("Process", str), ("FD", int), ("Path", str)] + return renderers.TreeGrid(tree_grid_args, self._generator()) From 6456e55ddcd121ba3e570a90c83c0138af5b532c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 17:51:27 +1100 Subject: [PATCH 002/526] Added Sockstat linux plugin to enumerate all processes sockets. The output format is based on the `ss` tools. It supports: * Unix socket * Inet/Inet6 sockets * Netlink sockets * VSock sockets * Packet sockets * XDP sockets (eBPF) * Bluetooth sockets (When the respective symbols are present) Changes to the linux Lsof plugin were required to be able to reuse its filedescriptor listing capability. --- .../framework/constants/linux/__init__.py | 216 ++++++++++ .../framework/plugins/linux/sockstat.py | 374 ++++++++++++++++++ .../framework/symbols/linux/__init__.py | 23 ++ .../symbols/linux/extensions/__init__.py | 251 ++++++++++++ 4 files changed, 864 insertions(+) create mode 100644 volatility3/framework/plugins/linux/sockstat.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index c25ea0e2f..6b63de6c5 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -11,3 +11,219 @@ KERNEL_NAME = "__kernel__" # arch/x86/include/asm/page_types.h PAGE_SHIFT = 12 """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" + +# Standard well-defined IP protocols. +# ref: include/uapi/linux/in.h +IP_PROTOCOLS = { + 0: "IP", + 1: "ICMP", + 2: "IGMP", + 4: "IPIP", + 6: "TCP", + 8: "EGP", + 12: "PUP", + 17: "UDP", + 22: "IDP", + 29: "TP", + 33: "DCCP", + 41: "IPV6", + 46: "RSVP", + 47: "GRE", + 50: "ESP", + 51: "AH", + 92: "MTP", + 94: "BEETPH", + 98: "ENCAP", + 103: "PIM", + 108: "COMP", + 132: "SCTP", + 136: "UDPLITE", + 137: "MPLS", + 143: "ETHERNET", + 255: "RAW", + 262: "MPTCP", +} + +# IPV6 extension headers +# ref: include/uapi/linux/in6.h +IPV6_PROTOCOLS = { + 0: "HOPBYHOP_OPTS", + 43: "ROUTING", + 44: "FRAGMENT", + 58: "ICMPv6", + 59: "NO_NEXT", + 60: "DESTINATION_OPTS", + 135: "MOBILITY", +} + +# ref: include/net/tcp_states.h +TCP_STATES = ( + "", + "ESTABLISHED", + "SYN_SENT", + "SYN_RECV", + "FIN_WAIT1", + "FIN_WAIT2", + "TIME_WAIT", + "CLOSE", + "CLOSE_WAIT", + "LAST_ACK", + "LISTEN", + "CLOSING", + "TCP_NEW_SYN_RECV", +) + +# ref: include/linux/net.h (socket_type enum) +SOCK_TYPES = { + 1: "STREAM", + 2: "DGRAM", + 3: "RAW", + 4: "RDM", + 5: "SEQPACKET", + 6: "DCCP", + 10: "PACKET", +} + +# Address families +# ref: include/linux/socket.h +SOCK_FAMILY = ( + "AF_UNSPEC", + "AF_UNIX", + "AF_INET", + "AF_AX25", + "AF_IPX", + "AF_APPLETALK", + "AF_NETROM", + "AF_BRIDGE", + "AF_ATMPVC", + "AF_X25", + "AF_INET6", + "AF_ROSE", + "AF_DECnet", + "AF_NETBEUI", + "AF_SECURITY", + "AF_KEY", + "AF_NETLINK", + "AF_PACKET", + "AF_ASH", + "AF_ECONET", + "AF_ATMSVC", + "AF_RDS", + "AF_SNA", + "AF_IRDA", + "AF_PPPOX", + "AF_WANPIPE", + "AF_LLC", + "AF_IB", + "AF_MPLS", + "AF_CAN", + "AF_TIPC", + "AF_BLUETOOTH", + "AF_IUCV", + "AF_RXRPC", + "AF_ISDN", + "AF_PHONET", + "AF_IEEE802154", + "AF_CAIF", + "AF_ALG", + "AF_NFC", + "AF_VSOCK", + "AF_KCM", + "AF_QIPCRTR", + "AF_SMC", + "AF_XDP", +) + +# Netlink protocols +# ref: include/uapi/linux/netlink.h +NETLINK_PROTOCOLS = ( + "NETLINK_ROUTE", + "NETLINK_UNUSED", + "NETLINK_USERSOCK", + "NETLINK_FIREWALL", + "NETLINK_SOCK_DIAG", + "NETLINK_NFLOG", + "NETLINK_XFRM", + "NETLINK_SELINUX", + "NETLINK_ISCSI", + "NETLINK_AUDIT", + "NETLINK_FIB_LOOKUP", + "NETLINK_CONNECTOR", + "NETLINK_NETFILTER", + "NETLINK_IP6_FW", + "NETLINK_DNRTMSG", + "NETLINK_KOBJECT_UEVENT", + "NETLINK_GENERIC", + "NETLINK_DM", + "NETLINK_SCSITRANSPORT", + "NETLINK_ECRYPTFS", + "NETLINK_RDMA", + "NETLINK_CRYPTO", + "NETLINK_SMC", +) + +# Short list of Ethernet Protocol ID's. +# ref: include/uapi/linux/if_ether.h +# Used in AF_PACKET socket family +ETH_PROTOCOLS = { + 0x0001: "ETH_P_802_3", + 0x0002: "ETH_P_AX25", + 0x0003: "ETH_P_ALL", + 0x0004: "ETH_P_802_2", + 0x0005: "ETH_P_SNAP", + 0x0006: "ETH_P_DDCMP", + 0x0007: "ETH_P_WAN_PPP", + 0x0008: "ETH_P_PPP_MP", + 0x0009: "ETH_P_LOCALTALK", + 0x000c: "ETH_P_CAN", + 0x000f: "ETH_P_CANFD", + 0x0010: "ETH_P_PPPTALK", + 0x0011: "ETH_P_TR_802_2", + 0x0016: "ETH_P_CONTROL", + 0x0017: "ETH_P_IRDA", + 0x0018: "ETH_P_ECONET", + 0x0019: "ETH_P_HDLC", + 0x001a: "ETH_P_ARCNET", + 0x001b: "ETH_P_DSA", + 0x001c: "ETH_P_TRAILER", + 0x0060: "ETH_P_LOOP", + 0x00F6: "ETH_P_IEEE802154", + 0x00F7: "ETH_P_CAIF", + 0x00F8: "ETH_P_XDSA", + 0x00F9: "ETH_P_MAP", + 0x0800: "ETH_P_IP", + 0x0805: "ETH_P_X25", + 0x0806: "ETH_P_ARP", + 0x8035: "ETH_P_RARP", + 0x809B: "ETH_P_ATALK", + 0x80F3: "ETH_P_AARP", + 0x8100: "ETH_P_8021Q", +} + +# Connection and socket states +# ref: include/net/bluetooth/bluetooth.h +BLUETOOTH_STATES = ( + "", + "CONNECTED", + "OPEN", + "BOUND", + "LISTEN", + "CONNECT", + "CONNECT2", + "CONFIG", + "DISCONN", + "CLOSED", +) + +# Bluetooth protocols +# ref: include/net/bluetooth/bluetooth.h +BLUETOOTH_PROTOCOLS = ( + "L2CAP", + "HCI", + "SCO", + "RFCOMM", + "BNEP", + "CMTP", + "HIDP", + "AVDTP", +) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py new file mode 100644 index 000000000..3be359463 --- /dev/null +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -0,0 +1,374 @@ +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +# Author: Gustavo Moreira + +import logging +from typing import Callable + +from volatility3.framework import renderers, interfaces, exceptions, constants +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility +from volatility3.framework.symbols import linux +from volatility3.plugins.linux import lsof + + +vollog = logging.getLogger(__name__) + +class SockHandlers(object): + def __init__(self, vmlinux, task): + self._vmlinux = vmlinux + self._task = task + + netns_id = task.nsproxy.net_ns.get_inode() + self._netdevices = self._build_network_devices_map(netns_id) + + self._sock_family_handlers = { + "AF_UNIX": self._unix_sock, + "AF_INET": self._inet_sock, + "AF_INET6": self._inet_sock, + "AF_NETLINK": self._netlink_sock, + "AF_VSOCK": self._vsock_sock, + "AF_PACKET": self._packet_sock, + "AF_XDP": self._xdp_sock, + "AF_BLUETOOTH": self._bluetooth_sock, + } + + def _build_network_devices_map(self, netns_id): + netdevices_map = {} + nethead = self._vmlinux.object_from_symbol(symbol_name="net_namespace_list") + net_symname = self._vmlinux.symbol_table_name + constants.BANG + "net" + for net in nethead.to_list(net_symname, "list"): + net_device_symname = self._vmlinux.symbol_table_name + constants.BANG + "net_device" + for net_dev in net.dev_base_head.to_list(net_device_symname, "dev_list"): + if net.get_inode() != netns_id: + continue + dev_name = str(utility.array_to_string(net_dev.name)) + netdevices_map[net_dev.ifindex] = dev_name + return netdevices_map + + def process_sock(self, sock): + family = sock.family + extended = {} + sock_handler = self._sock_family_handlers.get(family) + if sock_handler: + try: + sock_fields = sock_handler(sock, extended) + return *sock_fields, extended + except exceptions.SymbolError as e: + # Cannot finds the *_sock type in the symbols + vollog.warning("Error processing socket family '%s': %s", family, e) + else: + vollog.warning("Unsupported family '%s'", family) + + # Even if the sock family is not supported, or the required types + # are not present in the symbols, we can still show some general + # information about the socket that may be helpful. + saddr_tag = daddr_tag = state = "?" + + sock_stat = saddr_tag, daddr_tag, state + + return sock, sock_stat, extended + + def _unix_sock(self, sock, _extended): + unix_sock = sock.cast("unix_sock") + state = unix_sock.state + saddr = unix_sock.name + sinode = unix_sock.inode + if unix_sock.peer != 0: + peer = unix_sock.peer.dereference().cast("unix_sock") + daddr = peer.name + dinode = peer.inode + else: + daddr = dinode = "" + + saddr_tag = f"{saddr} {sinode}" + daddr_tag = f"{daddr} {dinode}" + sock_stat = saddr_tag, daddr_tag, state + return unix_sock, sock_stat + + def _inet_sock(self, sock, _extended): + inet_sock = sock.cast("inet_sock") + saddr = inet_sock.src_addr + sport = inet_sock.src_port + daddr = inet_sock.dst_addr + dport = inet_sock.dst_port + state = inet_sock.state + + if inet_sock.family == "AF_INET6": + saddr = f"[{saddr}]" + + saddr_tag = f"{saddr}:{sport}" + daddr_tag = f"{daddr}:{dport}" + sock_stat = saddr_tag, daddr_tag, state + return inet_sock, sock_stat + + def _netlink_sock(self, sock, _extended): + netlink_sock = sock.cast("netlink_sock") + + saddr_list = [] + src_portid = f"portid:{netlink_sock.portid}" + saddr_list.append(src_portid) + if netlink_sock.groups != 0: + groups_bitmap = netlink_sock.groups.dereference() + groups_str = f"groups:0x{groups_bitmap:08x}" + saddr_list.append(groups_str) + + daddr_list = [] + dst_portid = f"portid:{netlink_sock.dst_portid}" + daddr_list.append(dst_portid) + dst_group = f"group:0x{netlink_sock.dst_group:08x}" + daddr_list.append(dst_group) + module = netlink_sock.module + if module and netlink_sock.module.name: + module_name_str = utility.array_to_string(netlink_sock.module.name) + module_name = f"lkm:{module_name_str}" + daddr_list.append(module_name) + + saddr_tag = ",".join(saddr_list) + daddr_tag = ",".join(daddr_list) + state = netlink_sock.state + + sock_stat = saddr_tag, daddr_tag, state + return netlink_sock, sock_stat + + def _vsock_sock(self, sock, _extended): + vsock_sock = sock.cast("vsock_sock") + saddr = vsock_sock.local_addr.svm_cid + sport = vsock_sock.local_addr.svm_port + daddr = vsock_sock.remote_addr.svm_cid + dport = vsock_sock.remote_addr.svm_port + state = "" # Protocol is always 0 + + saddr_tag = f"{saddr}:{sport}" + daddr_tag = f"{daddr}:{dport}" + sock_stat = saddr_tag, daddr_tag, state + return vsock_sock, sock_stat + + def _packet_sock(self, sock, extended): + packet_sock = sock.cast("packet_sock") + ifindex = packet_sock.ifindex + dev_name = self._netdevices.get(ifindex, "") if ifindex > 0 else "ANY" + + if sock.has_member("sk_filter"): + sock_filter = sock.sk_filter + self.__update_extra_socket_bpf(sock_filter, extended) + + if sock.has_member("sk_reuseport_cb"): + sock_reuseport_cb = sock.sk_reuseport_cb + self.__update_extra_socket_bpf(sock_reuseport_cb, extended) + + saddr_tag = f"{dev_name}" + daddr_tag = "" + state = packet_sock.state + sock_stat = saddr_tag, daddr_tag, state + return packet_sock, sock_stat + + def __update_extra_socket_bpf(self, sock_filter, extended): + if not sock_filter: + return + + extended["bpf_filter_type"] = "cBPF" + + if not sock_filter.has_member("prog"): + return + + bpfprog = sock_filter.prog + if not bpfprog: + return + + BPF_PROG_TYPE_UNSPEC = 0 + if bpfprog.type > BPF_PROG_TYPE_UNSPEC: + extended["bpf_filter_type"] = "eBPF" + bpfprog_aux = bpfprog.aux + if bpfprog_aux: + extended["bpf_filter_id"] = str(bpfprog_aux.id) + bpfprog_name = str(utility.array_to_string(bpfprog.aux.name)) + if bpfprog_name: + extended["bpf_filter_name"] = bpfprog_name + + def _xdp_sock(self, sock, _extended): + xdp_sock = sock.cast("xdp_sock") + device = xdp_sock.dev + if not device: + return + + dev_name = utility.array_to_string(device.name) + saddr_tag = f"{dev_name}" + + bpfprog = device.xdp_prog + if not bpfprog: + return + + bpfprog_aux = bpfprog.aux + if bpfprog_aux: + bpfprog_id = bpfprog_aux.id + daddr_tag = f"ebpf_prog_id:{bpfprog_id}" + bpf_name = utility.array_to_string(bpfprog_aux.name) + if bpf_name: + daddr_tag += f",ebpf_prog_name:{bpf_name}" + else: + daddr_tag = "" + + # Hallelujah, xdp_sock.state is an enum + xsk_state = xdp_sock.state.lookup() + state = xsk_state.replace("XSK_", "") + + sock_stat = saddr_tag, daddr_tag, state + return xdp_sock, sock_stat + + def _bluetooth_sock(self, sock, _extended): + bt_sock = sock.cast("bt_sock") + + def bt_addr(addr): + return ":".join(reversed(["%02x" % x for x in addr.b])) + + saddr_tag = daddr_tag = "" + if bt_sock.protocol == "HCI": + pinfo = bt_sock.cast("hci_pinfo") + elif bt_sock.protocol == "L2CAP": + pinfo = bt_sock.cast("l2cap_pinfo") + src_addr = bt_addr(pinfo.chan.src) + dst_addr = bt_addr(pinfo.chan.dst) + saddr_tag = f"{src_addr}" + daddr_tag = f"{dst_addr}" + elif bt_sock.protocol == "RFCOMM": + pinfo = bt_sock.cast("rfcomm_pinfo") + src_addr = bt_addr(pinfo.src) + dst_addr = bt_addr(pinfo.dst) + channel = pinfo.channel + saddr_tag = f"[{src_addr}]:{channel}" + daddr_tag = f"{dst_addr}" + else: + vollog.warning("Unsupported bluetooth protocol '%s'", bt_sock.protocol) + + state = bt_sock.state + sock_stat = saddr_tag, daddr_tag, state + return bt_sock, sock_stat + +class Sockstat(plugins.PluginInterface): + """Lists all network connections for all processes.""" + + _required_framework_version = (2, 0, 0) + + _version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement(name="kernel", description="Linux kernel", + architectures=["Intel32", "Intel64"]), + requirements.PluginRequirement(name="lsof", plugin=lsof.Lsof, version=(2, 0, 0)), + requirements.VersionRequirement(name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0)), + requirements.BooleanRequirement(name="unix", + description=("Show UNIX domain Sockets only"), + default=False, + optional=True), + requirements.ListRequirement(name="pids", + description="Filter results by process IDs. " + "It takes the root PID namespace identifiers.", + element_type=int, + optional=True), + requirements.IntRequirement(name="netns", + description="Filter results by network namespace. " + "Otherwise, all of them are shown.", + optional=True), + ] + + @classmethod + def list_sockets(cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + filter_func: Callable[[int], bool] = lambda _: False): + """ + Returns every single socket descriptors + """ + vmlinux = context.modules[vmlinux_module_name] + + sfop_addr = vmlinux.object_from_symbol("socket_file_ops").vol.offset + dfop_addr = vmlinux.object_from_symbol("sockfs_dentry_operations").vol.offset + + fd_generator = lsof.Lsof.list_fds(context, vmlinux.name, filter_func) + for _pid, _task_comm, task, fd_fields in fd_generator: + fd_num, filp, _full_path = fd_fields + + if filp.f_op not in (sfop_addr, dfop_addr): + continue + + dentry = filp.get_dentry() + if not dentry: + continue + + d_inode = dentry.d_inode + if not d_inode: + continue + + socket_alloc = linux.LinuxUtilities.container_of(d_inode, "socket_alloc", "vfs_inode", vmlinux) + _socket = socket_alloc.socket + + vfs_inode = socket_alloc.vfs_inode + if not (_socket and vfs_inode): + continue + + sock = _socket.sk.dereference() + + sock_type = sock.type + family = sock.family + + sock_handler = SockHandlers(vmlinux, task) + sock_fields = sock_handler.process_sock(sock) + if not sock_fields: + continue + + child_sock = sock_fields[0] + protocol = child_sock.protocol if hasattr(child_sock, "protocol") else "" + + net = task.nsproxy.net_ns + netns_id = net.proc_inum if net.has_member("proc_inum") else net.ns.inum + yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields + + def _generator(self): + pids = self.config.get('pids') + filter_func = lsof.pslist.PsList.create_pid_filter(pids) + + tasks_per_sock = {} + socket_generator = self.list_sockets(self.context, self.config['kernel'], filter_func=filter_func) + for task, netns, fd_num, family, sock_type, protocol, sock_fields in socket_generator: + if self.config['netns'] and self.config['netns'] != netns: + continue + + sock, sock_stat, extended = sock_fields + + task_comm = utility.array_to_string(task.comm) + task_info = f"{task_comm},pid={task.pid},fd={fd_num}" + if extended: + extended_str = ",".join(f"{k}={v}" for k, v in extended.items()) + task_info = f"{task_info},{extended_str}" + + fields = netns, family, sock_type, protocol, *sock_stat + + sock_addr = sock.vol.offset + tasks_per_sock.setdefault(sock_addr, {}) + tasks_per_sock[sock_addr].setdefault('tasks', []) + tasks_per_sock[sock_addr]['tasks'].append(task_info) + tasks_per_sock[sock_addr]['fields'] = fields + + for data in tasks_per_sock.values(): + task_list = [f"({task})" for task in data['tasks']] + tasks = ",".join(task_list) + + fields = data['fields'] + (tasks,) + yield (0, fields) + + def run(self): + tree_grid_args = [("NetNS", int), + ("Family", str), + ("Type", str), + ("Proto", str), + ("Source Addr:Port", str), + ("Destination Addr:Port", str), + ("State", str), + ("Tasks", str)] + + return renderers.TreeGrid(tree_grid_args, self._generator()) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 36e23a35d..739ecbedb 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -30,6 +30,16 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('vfsmount', extensions.vfsmount) self.set_type_class('kobject', extensions.kobject) + # Network + self.set_type_class('net', extensions.net) + self.set_type_class('sock', extensions.sock) + self.set_type_class('inet_sock', extensions.inet_sock) + self.set_type_class('unix_sock', extensions.unix_sock) + self.set_type_class('netlink_sock', extensions.netlink_sock) + self.set_type_class('packet_sock', extensions.packet_sock) + if 'bt_sock' in self.types: + self.set_type_class('bt_sock', extensions.bt_sock) + if 'module' in self.types: self.set_type_class('module', extensions.module) @@ -183,6 +193,10 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): def files_descriptors_for_process(cls, context: interfaces.context.ContextInterface, symbol_table: str, task: interfaces.objects.ObjectInterface): + # task.files can be null + if not task.files: + return + fd_table = task.files.get_fds() if fd_table == 0: return @@ -267,3 +281,12 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): list_struct = vmlinux.object(object_type = struct_name, offset = list_start.vol.offset) yield list_struct list_start = getattr(list_struct, list_member) + + @classmethod + def container_of(cls, addr, type_name, member_name, vmlinux): + if not addr: + return + type_dec = vmlinux.get_type(type_name) + member_offset = type_dec.relative_child_offset(member_name) + container_addr = addr - member_offset + return vmlinux.object(object_type=type_name, offset=container_addr, absolute=True) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0edd60608..2af5f56b0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -4,9 +4,15 @@ import collections.abc import logging +import socket from typing import Generator, Iterable, Iterator, Optional, Tuple from volatility3.framework import constants +from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY +from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS +from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS +from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES +from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility @@ -539,3 +545,248 @@ class kobject(objects.StructType): ret = refcnt.refs.counter return ret + +class mnt_namespace(objects.StructType): + def get_inode(self): + if self.has_member("proc_inum"): + return self.proc_inum + elif self.ns.has_member("inum"): + return self.ns.inum + else: + raise AttributeError("Unable to find mnt_namespace inode") + +class net(objects.StructType): + def get_inode(self): + if self.has_member("proc_inum"): + return self.proc_inum + elif self.ns.has_member("inum"): + return self.ns.inum + else: + raise AttributeError("Unable to find net_namespace inode") + +class sock(objects.StructType): + def __get_vol_kernel_module_name(self): + symbol_table_arr = self.vol.type_name.split("!", 1) + symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None + + module_names = list(self._context.modules.get_modules_by_symbol_tables(symbol_table)) + if not module_names: + raise ValueError(f"No module using the symbol table {symbol_table}") + + return module_names[0] + + @property + def family(self): + family_idx = self.__sk_common.skc_family + if 0 <= family_idx < len(SOCK_FAMILY): + return SOCK_FAMILY[family_idx] + else: + return "UNKNOWN" + + @property + def type(self): + return SOCK_TYPES.get(self.sk_type, "") + + @property + def inode(self): + if not self.sk_socket: + return 0 + + kernel_module_name = self.__get_vol_kernel_module_name() + kernel = self._context.modules[kernel_module_name] + socket_alloc = linux.LinuxUtilities.container_of(self.sk_socket, "socket_alloc", "socket", kernel) + vfs_inode = socket_alloc.vfs_inode + + return vfs_inode.i_ino + +class unix_sock(objects.StructType): + @property + def name(self): + if self.addr: + sockaddr_un = self.addr.name.cast("sockaddr_un") + saddr = str(utility.array_to_string(sockaddr_un.sun_path)) + else: + saddr = "" + return saddr + + @property + def protocol(self): + return "" + + @property + def state(self): + """Return a string representing the sock state.""" + + # Unix socket states reuse (a subset) of the inet_sock states contants + if self.sk.type == "STREAM": + state_idx = self.sk.__sk_common.skc_state + if 0 <= state_idx < len(TCP_STATES): + state = TCP_STATES[state_idx] + else: + state = "UNKNOWN" + else: + state = "UNCONNECTED" + + return state + + @property + def inode(self): + return self.sk.inode + +class inet_sock(objects.StructType): + @property + def family(self): + family_idx = self.sk.__sk_common.skc_family + if 0 <= family_idx < len(SOCK_FAMILY): + return SOCK_FAMILY[family_idx] + else: + return "UNKNOWN" + + @property + def protocol(self): + # If INET6 family and a proto is defined, we use that specific IPv6 protocol. + # Otherwise, we use the standard IP protocol. + protocol = IP_PROTOCOLS.get(self.sk.sk_protocol, "UNKNOWN") + if self.family == "AF_INET6": + protocol = IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) + return protocol + + @property + def state(self): + """Return a string representing the sock state.""" + + if self.sk.type == "STREAM": + state_idx = self.sk.__sk_common.skc_state + if 0 <= state_idx < len(TCP_STATES): + state = TCP_STATES[state_idx] + else: + state = "UNKNOWN" + else: + state = "UNCONNECTED" + + return state + + @property + def src_port(self): + sport_le = getattr(self, "sport", getattr(self, "inet_sport", None)) + if sport_le is not None: + return socket.htons(sport_le) + + @property + def dst_port(self): + sk_common = self.sk.__sk_common + if hasattr(sk_common, "skc_portpair"): + dport_le = sk_common.skc_portpair & 0xffff + elif hasattr(self, "dport"): + dport_le = self.dport + elif hasattr(self, "inet_dport"): + dport_le = self.inet_dport + elif hasattr(sk_common, "skc_dport"): + dport_le = sk_common.skc_dport + else: + return + + return socket.htons(dport_le) + + @property + def src_addr(self): + sk_common = self.sk.__sk_common + family = sk_common.skc_family + if family == socket.AF_INET: + addr_size = 4 + if hasattr(self, "rcv_saddr"): + saddr = self.rcv_saddr + elif hasattr(self, "inet_rcv_saddr"): + saddr = self.inet_rcv_saddr + else: + saddr = sk_common.skc_rcv_saddr + elif family == socket.AF_INET6: + addr_size = 16 + saddr = self.pinet6.saddr + else: + return + + parent_layer = self._context.layers[self.vol.layer_name] + addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) + return socket.inet_ntop(family, addr_bytes) + + @property + def dst_addr(self): + sk_common = self.sk.__sk_common + family = sk_common.skc_family + if family == socket.AF_INET: + if hasattr(self, "daddr") and self.daddr: + daddr = self.daddr + elif hasattr(self, "inet_daddr") and self.inet_daddr: + daddr = self.inet_daddr + else: + daddr = sk_common.skc_daddr + addr_size = 4 + elif family == socket.AF_INET6: + if hasattr(self.pinet6, "daddr"): + daddr = self.pinet6.daddr + else: + daddr = sk_common.skc_v6_daddr + addr_size = 16 + else: + return + + parent_layer = self._context.layers[self.vol.layer_name] + addr_bytes = parent_layer.read(daddr.vol.offset, addr_size) + return socket.inet_ntop(family, addr_bytes) + +class netlink_sock(objects.StructType): + @property + def protocol(self): + protocol_idx = self.sk.sk_protocol + if 0 <= protocol_idx < len(NETLINK_PROTOCOLS): + return NETLINK_PROTOCOLS[protocol_idx] + else: + return "UNKNOWN" + + @property + def state(self): + # Netlink is a datagram-oriented service. We can only have + # SOCK_RAW or SOCK_DGRAM socket types. + # NOTE: We are overridden the netlink_sock.state member here + return "UNCONNECTED" + + +class packet_sock(objects.StructType): + @property + def protocol(self): + eth_proto = socket.htons(self.num) + if eth_proto == 0: + return "" + elif eth_proto in ETH_PROTOCOLS: + return ETH_PROTOCOLS[eth_proto] + else: + return f"0x{eth_proto:x}" + + @property + def state(self): + # Packet socket types are either SOCK_RAW or SOCK_DGRAM. + # NOTE: We are overriding netlink_sock.state here + return "UNCONNECTED" + + +class bt_sock(objects.StructType): + @property + def protocol(self): + type_idx = self.sk.sk_protocol + if 0 <= type_idx < len(BLUETOOTH_PROTOCOLS): + state = BLUETOOTH_PROTOCOLS[type_idx] + else: + state = "UNKNOWN" + + return state + + @property + def state(self): + state_idx = self.sk.__sk_common.skc_state + if 0 <= state_idx < len(BLUETOOTH_STATES): + state = BLUETOOTH_STATES[state_idx] + else: + state = "UNKNOWN" + + return state From 5c507f5ae8de542f1bf530b3762eeaafa90226da Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 12:44:50 +1100 Subject: [PATCH 003/526] Plugins versioning fixes --- volatility3/framework/plugins/linux/lsof.py | 2 +- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 9ce7027f3..5ebd8e5c9 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -21,7 +21,7 @@ class Lsof(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 3be359463..4611d52e0 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -252,14 +252,14 @@ class Sockstat(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls): return [ requirements.ModuleRequirement(name="kernel", description="Linux kernel", architectures=["Intel32", "Intel64"]), - requirements.PluginRequirement(name="lsof", plugin=lsof.Lsof, version=(2, 0, 0)), + requirements.PluginRequirement(name="lsof", plugin=lsof.Lsof, version=(1, 1, 0)), requirements.VersionRequirement(name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0)), requirements.BooleanRequirement(name="unix", description=("Show UNIX domain Sockets only"), From 9766327433de338e377340ba5f3565b85869d3c0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 12:55:08 +1100 Subject: [PATCH 004/526] Parameterized generator --- volatility3/framework/plugins/linux/lsof.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 5ebd8e5c9..30ceafdab 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -60,13 +60,7 @@ class Lsof(plugins.PluginInterface): for fd_fields in fd_generator: yield pid, task_comm, task, fd_fields - def _generator(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - - fds_generator = self.list_fds(self.context, - self.config['kernel'], - filter_func=filter_func) - + def _generator(self, fds_generator): for pid, task_comm, _task, fd_fields in fds_generator: fd_num, _filp, full_path = fd_fields @@ -74,5 +68,10 @@ class Lsof(plugins.PluginInterface): yield (0, fields) def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + fds_generator = self.list_fds(self.context, + self.config['kernel'], + filter_func=filter_func) + tree_grid_args = [("PID", int), ("Process", str), ("FD", int), ("Path", str)] - return renderers.TreeGrid(tree_grid_args, self._generator()) + return renderers.TreeGrid(tree_grid_args, self._generator(fds_generator)) From 2e93db9b57eb9ba8d33fa70209e77d881b686d16 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 13:04:29 +1100 Subject: [PATCH 005/526] Adding versioning to SockHandlers --- volatility3/framework/plugins/linux/sockstat.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 4611d52e0..c53091d86 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -16,7 +16,12 @@ from volatility3.plugins.linux import lsof vollog = logging.getLogger(__name__) -class SockHandlers(object): +class SockHandlers(interfaces.configuration.VersionableInterface): + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + def __init__(self, vmlinux, task): self._vmlinux = vmlinux self._task = task @@ -259,6 +264,7 @@ class Sockstat(plugins.PluginInterface): return [ requirements.ModuleRequirement(name="kernel", description="Linux kernel", architectures=["Intel32", "Intel64"]), + requirements.VersionRequirement(name="SockHandlers", component=SockHandlers, version=(1, 0, 0)), requirements.PluginRequirement(name="lsof", plugin=lsof.Lsof, version=(1, 1, 0)), requirements.VersionRequirement(name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0)), requirements.BooleanRequirement(name="unix", From 6970776c4c330e1bc96e77df6c13d823503d1295 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 13:07:50 +1100 Subject: [PATCH 006/526] Renaming function to use single underscore and name from `extra` to `extended` --- volatility3/framework/plugins/linux/sockstat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index c53091d86..eab8993ba 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -158,11 +158,11 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if sock.has_member("sk_filter"): sock_filter = sock.sk_filter - self.__update_extra_socket_bpf(sock_filter, extended) + self._update_extended_socket_bpf(sock_filter, extended) if sock.has_member("sk_reuseport_cb"): sock_reuseport_cb = sock.sk_reuseport_cb - self.__update_extra_socket_bpf(sock_reuseport_cb, extended) + self._update_extended_socket_bpf(sock_reuseport_cb, extended) saddr_tag = f"{dev_name}" daddr_tag = "" @@ -170,7 +170,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return packet_sock, sock_stat - def __update_extra_socket_bpf(self, sock_filter, extended): + def _update_extended_socket_bpf(self, sock_filter, extended): if not sock_filter: return From 86676cf4e87bbfa99cf4165af39ba71d7e8f9481 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 13:20:31 +1100 Subject: [PATCH 007/526] Changing BPF_PROG_TYPE_UNSPEC constant in favor of a literal 0 and a comment. --- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index eab8993ba..943c497b1 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -183,8 +183,8 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if not bpfprog: return - BPF_PROG_TYPE_UNSPEC = 0 - if bpfprog.type > BPF_PROG_TYPE_UNSPEC: + # BPF_PROG_TYPE_UNSPEC = 0 + if bpfprog.type > 0: extended["bpf_filter_type"] = "eBPF" bpfprog_aux = bpfprog.aux if bpfprog_aux: From a386a7a9482edbb4ef0cb011cf89f681a4d0042e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 15:40:51 +1100 Subject: [PATCH 008/526] Removed underscore from unused arguments --- volatility3/framework/plugins/linux/sockstat.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 943c497b1..d147f3e5e 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -76,7 +76,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return sock, sock_stat, extended - def _unix_sock(self, sock, _extended): + def _unix_sock(self, sock, extended): unix_sock = sock.cast("unix_sock") state = unix_sock.state saddr = unix_sock.name @@ -93,7 +93,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return unix_sock, sock_stat - def _inet_sock(self, sock, _extended): + def _inet_sock(self, sock, extended): inet_sock = sock.cast("inet_sock") saddr = inet_sock.src_addr sport = inet_sock.src_port @@ -109,7 +109,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return inet_sock, sock_stat - def _netlink_sock(self, sock, _extended): + def _netlink_sock(self, sock, extended): netlink_sock = sock.cast("netlink_sock") saddr_list = [] @@ -138,7 +138,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return netlink_sock, sock_stat - def _vsock_sock(self, sock, _extended): + def _vsock_sock(self, sock, extended): vsock_sock = sock.cast("vsock_sock") saddr = vsock_sock.local_addr.svm_cid sport = vsock_sock.local_addr.svm_port @@ -193,7 +193,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if bpfprog_name: extended["bpf_filter_name"] = bpfprog_name - def _xdp_sock(self, sock, _extended): + def _xdp_sock(self, sock, extended): xdp_sock = sock.cast("xdp_sock") device = xdp_sock.dev if not device: @@ -223,7 +223,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return xdp_sock, sock_stat - def _bluetooth_sock(self, sock, _extended): + def _bluetooth_sock(self, sock, extended): bt_sock = sock.cast("bt_sock") def bt_addr(addr): From c54818e6309d6bd4a83de14dfdb6a3c8fddd6d22 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 16:00:11 +1100 Subject: [PATCH 009/526] Remove redundancies around array_to_string(). It returns a str() already. --- volatility3/framework/plugins/linux/sockstat.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index d147f3e5e..f00169cf6 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -49,7 +49,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): for net_dev in net.dev_base_head.to_list(net_device_symname, "dev_list"): if net.get_inode() != netns_id: continue - dev_name = str(utility.array_to_string(net_dev.name)) + dev_name = utility.array_to_string(net_dev.name) netdevices_map[net_dev.ifindex] = dev_name return netdevices_map @@ -189,7 +189,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): bpfprog_aux = bpfprog.aux if bpfprog_aux: extended["bpf_filter_id"] = str(bpfprog_aux.id) - bpfprog_name = str(utility.array_to_string(bpfprog.aux.name)) + bpfprog_name = utility.array_to_string(bpfprog.aux.name) if bpfprog_name: extended["bpf_filter_name"] = bpfprog_name @@ -199,8 +199,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if not device: return - dev_name = utility.array_to_string(device.name) - saddr_tag = f"{dev_name}" + saddr_tag = utility.array_to_string(device.name) bpfprog = device.xdp_prog if not bpfprog: From 52181c7e8fd50757e549121cb00e81deaaabf6c6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 16:22:30 +1100 Subject: [PATCH 010/526] Improvements to the parameterized generator changes --- volatility3/framework/plugins/linux/lsof.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 30ceafdab..983f62562 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -60,7 +60,12 @@ class Lsof(plugins.PluginInterface): for fd_fields in fd_generator: yield pid, task_comm, task, fd_fields - def _generator(self, fds_generator): + def _generator(self, pids, symbol_table): + filter_func = pslist.PsList.create_pid_filter(pids) + fds_generator = self.list_fds(self.context, + symbol_table, + filter_func=filter_func) + for pid, task_comm, _task, fd_fields in fds_generator: fd_num, _filp, full_path = fd_fields @@ -68,10 +73,8 @@ class Lsof(plugins.PluginInterface): yield (0, fields) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - fds_generator = self.list_fds(self.context, - self.config['kernel'], - filter_func=filter_func) + pids = self.config.get('pid', None) + symbol_table = self.config['kernel'] tree_grid_args = [("PID", int), ("Process", str), ("FD", int), ("Path", str)] - return renderers.TreeGrid(tree_grid_args, self._generator(fds_generator)) + return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) From 7ed5739e24d5f5c836a1cad2d2b0ddf73000f9a7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 16:24:57 +1100 Subject: [PATCH 011/526] socket is no longer imported in this file. Remove the underscore --- volatility3/framework/plugins/linux/sockstat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index f00169cf6..1af232b07 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -310,13 +310,13 @@ class Sockstat(plugins.PluginInterface): continue socket_alloc = linux.LinuxUtilities.container_of(d_inode, "socket_alloc", "vfs_inode", vmlinux) - _socket = socket_alloc.socket + socket = socket_alloc.socket vfs_inode = socket_alloc.vfs_inode - if not (_socket and vfs_inode): + if not (socket and vfs_inode): continue - sock = _socket.sk.dereference() + sock = socket.sk.dereference() sock_type = sock.type family = sock.family From b4bcdd0856c8dd85a393647d02b95553c82bf855 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 16:51:02 +1100 Subject: [PATCH 012/526] Minor changes --- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 1af232b07..2e71dcd61 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -189,7 +189,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): bpfprog_aux = bpfprog.aux if bpfprog_aux: extended["bpf_filter_id"] = str(bpfprog_aux.id) - bpfprog_name = utility.array_to_string(bpfprog.aux.name) + bpfprog_name = utility.array_to_string(bpfprog_aux.name) if bpfprog_name: extended["bpf_filter_name"] = bpfprog_name @@ -287,7 +287,7 @@ class Sockstat(plugins.PluginInterface): vmlinux_module_name: str, filter_func: Callable[[int], bool] = lambda _: False): """ - Returns every single socket descriptors + Returns every single socket descriptor """ vmlinux = context.modules[vmlinux_module_name] From 8af74229288cd30e7a662ad7025b4e4093c19c61 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 16:52:47 +1100 Subject: [PATCH 013/526] Parameterized generator --- volatility3/framework/plugins/linux/sockstat.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 2e71dcd61..c133a400d 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -333,14 +333,13 @@ class Sockstat(plugins.PluginInterface): netns_id = net.proc_inum if net.has_member("proc_inum") else net.ns.inum yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields - def _generator(self): - pids = self.config.get('pids') + def _generator(self, pids, netns_arg, symbol_table): filter_func = lsof.pslist.PsList.create_pid_filter(pids) + socket_generator = self.list_sockets(self.context, symbol_table, filter_func=filter_func) tasks_per_sock = {} - socket_generator = self.list_sockets(self.context, self.config['kernel'], filter_func=filter_func) for task, netns, fd_num, family, sock_type, protocol, sock_fields in socket_generator: - if self.config['netns'] and self.config['netns'] != netns: + if netns_arg and netns_arg != netns: continue sock, sock_stat, extended = sock_fields @@ -367,6 +366,10 @@ class Sockstat(plugins.PluginInterface): yield (0, fields) def run(self): + pids = self.config.get('pids') + netns = self.config['netns'] + symbol_table = self.config['kernel'] + tree_grid_args = [("NetNS", int), ("Family", str), ("Type", str), @@ -376,4 +379,4 @@ class Sockstat(plugins.PluginInterface): ("State", str), ("Tasks", str)] - return renderers.TreeGrid(tree_grid_args, self._generator()) + return renderers.TreeGrid(tree_grid_args, self._generator(pids, netns, symbol_table)) From f4e1f4729f5e1f049f69dd6a1d7e144ffa93b9fb Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 16:56:27 +1100 Subject: [PATCH 014/526] Added type annotations --- .../framework/plugins/linux/sockstat.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index c133a400d..e1fbfeddb 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -6,7 +6,7 @@ import logging from typing import Callable -from volatility3.framework import renderers, interfaces, exceptions, constants +from volatility3.framework import renderers, interfaces, exceptions, constants, objects from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -40,7 +40,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): "AF_BLUETOOTH": self._bluetooth_sock, } - def _build_network_devices_map(self, netns_id): + def _build_network_devices_map(self, netns_id: int): netdevices_map = {} nethead = self._vmlinux.object_from_symbol(symbol_name="net_namespace_list") net_symname = self._vmlinux.symbol_table_name + constants.BANG + "net" @@ -53,7 +53,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): netdevices_map[net_dev.ifindex] = dev_name return netdevices_map - def process_sock(self, sock): + def process_sock(self, sock: objects.StructType): family = sock.family extended = {} sock_handler = self._sock_family_handlers.get(family) @@ -76,7 +76,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return sock, sock_stat, extended - def _unix_sock(self, sock, extended): + def _unix_sock(self, sock: objects.StructType, extended: dict): unix_sock = sock.cast("unix_sock") state = unix_sock.state saddr = unix_sock.name @@ -93,7 +93,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return unix_sock, sock_stat - def _inet_sock(self, sock, extended): + def _inet_sock(self, sock: objects.StructType, extended: dict): inet_sock = sock.cast("inet_sock") saddr = inet_sock.src_addr sport = inet_sock.src_port @@ -109,7 +109,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return inet_sock, sock_stat - def _netlink_sock(self, sock, extended): + def _netlink_sock(self, sock: objects.StructType, extended: dict): netlink_sock = sock.cast("netlink_sock") saddr_list = [] @@ -138,7 +138,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return netlink_sock, sock_stat - def _vsock_sock(self, sock, extended): + def _vsock_sock(self, sock: objects.StructType, extended: dict): vsock_sock = sock.cast("vsock_sock") saddr = vsock_sock.local_addr.svm_cid sport = vsock_sock.local_addr.svm_port @@ -151,7 +151,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return vsock_sock, sock_stat - def _packet_sock(self, sock, extended): + def _packet_sock(self, sock: objects.StructType, extended: dict): packet_sock = sock.cast("packet_sock") ifindex = packet_sock.ifindex dev_name = self._netdevices.get(ifindex, "") if ifindex > 0 else "ANY" @@ -170,7 +170,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return packet_sock, sock_stat - def _update_extended_socket_bpf(self, sock_filter, extended): + def _update_extended_socket_bpf(self, sock_filter: objects.Pointer, extended: dict): if not sock_filter: return @@ -193,7 +193,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if bpfprog_name: extended["bpf_filter_name"] = bpfprog_name - def _xdp_sock(self, sock, extended): + def _xdp_sock(self, sock: objects.StructType, extended: dict): xdp_sock = sock.cast("xdp_sock") device = xdp_sock.dev if not device: @@ -222,7 +222,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return xdp_sock, sock_stat - def _bluetooth_sock(self, sock, extended): + def _bluetooth_sock(self, sock: objects.StructType, extended: dict): bt_sock = sock.cast("bt_sock") def bt_addr(addr): From 34176a80665dfb5af34955b1079af2f3b6c63b1f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 17:10:45 +1100 Subject: [PATCH 015/526] Moving log lines from warning to LOGLEVEL_V --- volatility3/framework/plugins/linux/sockstat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index e1fbfeddb..5f9b6fb07 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -63,9 +63,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return *sock_fields, extended except exceptions.SymbolError as e: # Cannot finds the *_sock type in the symbols - vollog.warning("Error processing socket family '%s': %s", family, e) + vollog.log(constants.LOGLEVEL_V, "Error processing socket family '%s': %s", family, e) else: - vollog.warning("Unsupported family '%s'", family) + vollog.log(constants.LOGLEVEL_V, "Unsupported family '%s'", family) # Even if the sock family is not supported, or the required types # are not present in the symbols, we can still show some general @@ -245,7 +245,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): saddr_tag = f"[{src_addr}]:{channel}" daddr_tag = f"{dst_addr}" else: - vollog.warning("Unsupported bluetooth protocol '%s'", bt_sock.protocol) + vollog.log(constants.LOGLEVEL_V, "Unsupported bluetooth protocol '%s'", bt_sock.protocol) state = bt_sock.state sock_stat = saddr_tag, daddr_tag, state From a1ff8d7809545a69e8f780913d05da3d7e886d9b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 17:25:58 +1100 Subject: [PATCH 016/526] Fix comment. This was related to netlink_sock not packer_sock --- volatility3/framework/symbols/linux/extensions/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 2af5f56b0..d551f3b02 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -766,7 +766,6 @@ class packet_sock(objects.StructType): @property def state(self): # Packet socket types are either SOCK_RAW or SOCK_DGRAM. - # NOTE: We are overriding netlink_sock.state here return "UNCONNECTED" From 7dc33b83145e1f0b243a6ab3f5261c62dee22966 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 17:28:38 +1100 Subject: [PATCH 017/526] fix typo --- volatility3/framework/symbols/linux/extensions/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d551f3b02..3f286e1a3 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -748,7 +748,8 @@ class netlink_sock(objects.StructType): def state(self): # Netlink is a datagram-oriented service. We can only have # SOCK_RAW or SOCK_DGRAM socket types. - # NOTE: We are overridden the netlink_sock.state member here + # NOTE: We are overriding the netlink_sock.state member here + return "UNCONNECTED" From a7520a377dfa1e4aaea3c82f683adcbea032f5e0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 21 Dec 2021 14:53:13 +1100 Subject: [PATCH 018/526] Supporting socket and reuseport filters in all the socket families. --- .../framework/plugins/linux/sockstat.py | 84 ++++++++++--------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 5f9b6fb07..323f02213 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -59,7 +59,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_handler = self._sock_family_handlers.get(family) if sock_handler: try: - sock_fields = sock_handler(sock, extended) + sock_fields = sock_handler(sock) + self._update_extended_socket_filters_info(sock, extended) + return *sock_fields, extended except exceptions.SymbolError as e: # Cannot finds the *_sock type in the symbols @@ -76,7 +78,42 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return sock, sock_stat, extended - def _unix_sock(self, sock: objects.StructType, extended: dict): + def _update_extended_socket_filters_info(self, sock: objects.Pointer, extended: dict) -> None: + """Get infomation from the socket and reuseport filters + + Args: + sock: The kernel sock (sk) struct + extended: Dictionary to store extended information + """ + if sock.has_member("sk_filter") and sock.sk_filter: + sock_filter = sock.sk_filter + extended["filter_type"] = "socket_filter" + self._extract_socket_filter_info(sock_filter, extended) + + if sock.has_member("sk_reuseport_cb") and sock.sk_reuseport_cb: + sock_reuseport_cb = sock.sk_reuseport_cb + extended["filter_type"] = "reuseport_filter" + self._extract_socket_filter_info(sock_reuseport_cb, extended) + + def _extract_socket_filter_info(self, sock_filter: objects.Pointer, extended: dict): + extended["bpf_filter_type"] = "cBPF" + + if not sock_filter.has_member("prog") or not sock_filter.prog: + return + + bpfprog = sock_filter.prog + + # BPF_PROG_TYPE_UNSPEC = 0 + if bpfprog.type > 0: + extended["bpf_filter_type"] = "eBPF" + bpfprog_aux = bpfprog.aux + if bpfprog_aux: + extended["bpf_filter_id"] = str(bpfprog_aux.id) + bpfprog_name = utility.array_to_string(bpfprog_aux.name) + if bpfprog_name: + extended["bpf_filter_name"] = bpfprog_name + + def _unix_sock(self, sock: objects.StructType): unix_sock = sock.cast("unix_sock") state = unix_sock.state saddr = unix_sock.name @@ -93,7 +130,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return unix_sock, sock_stat - def _inet_sock(self, sock: objects.StructType, extended: dict): + def _inet_sock(self, sock: objects.StructType): inet_sock = sock.cast("inet_sock") saddr = inet_sock.src_addr sport = inet_sock.src_port @@ -109,7 +146,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return inet_sock, sock_stat - def _netlink_sock(self, sock: objects.StructType, extended: dict): + def _netlink_sock(self, sock: objects.StructType): netlink_sock = sock.cast("netlink_sock") saddr_list = [] @@ -138,7 +175,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return netlink_sock, sock_stat - def _vsock_sock(self, sock: objects.StructType, extended: dict): + def _vsock_sock(self, sock: objects.StructType): vsock_sock = sock.cast("vsock_sock") saddr = vsock_sock.local_addr.svm_cid sport = vsock_sock.local_addr.svm_port @@ -151,49 +188,18 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return vsock_sock, sock_stat - def _packet_sock(self, sock: objects.StructType, extended: dict): + def _packet_sock(self, sock: objects.StructType): packet_sock = sock.cast("packet_sock") ifindex = packet_sock.ifindex dev_name = self._netdevices.get(ifindex, "") if ifindex > 0 else "ANY" - if sock.has_member("sk_filter"): - sock_filter = sock.sk_filter - self._update_extended_socket_bpf(sock_filter, extended) - - if sock.has_member("sk_reuseport_cb"): - sock_reuseport_cb = sock.sk_reuseport_cb - self._update_extended_socket_bpf(sock_reuseport_cb, extended) - saddr_tag = f"{dev_name}" daddr_tag = "" state = packet_sock.state sock_stat = saddr_tag, daddr_tag, state return packet_sock, sock_stat - def _update_extended_socket_bpf(self, sock_filter: objects.Pointer, extended: dict): - if not sock_filter: - return - - extended["bpf_filter_type"] = "cBPF" - - if not sock_filter.has_member("prog"): - return - - bpfprog = sock_filter.prog - if not bpfprog: - return - - # BPF_PROG_TYPE_UNSPEC = 0 - if bpfprog.type > 0: - extended["bpf_filter_type"] = "eBPF" - bpfprog_aux = bpfprog.aux - if bpfprog_aux: - extended["bpf_filter_id"] = str(bpfprog_aux.id) - bpfprog_name = utility.array_to_string(bpfprog_aux.name) - if bpfprog_name: - extended["bpf_filter_name"] = bpfprog_name - - def _xdp_sock(self, sock: objects.StructType, extended: dict): + def _xdp_sock(self, sock: objects.StructType): xdp_sock = sock.cast("xdp_sock") device = xdp_sock.dev if not device: @@ -222,7 +228,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return xdp_sock, sock_stat - def _bluetooth_sock(self, sock: objects.StructType, extended: dict): + def _bluetooth_sock(self, sock: objects.StructType): bt_sock = sock.cast("bt_sock") def bt_addr(addr): From 7099a7a52ec3d4bab96bcd3e3f70d72abdb54221 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 21 Dec 2021 14:54:38 +1100 Subject: [PATCH 019/526] Fix. We should call the `net` type method here. --- volatility3/framework/plugins/linux/sockstat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 323f02213..4f9ac64de 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -336,7 +336,7 @@ class Sockstat(plugins.PluginInterface): protocol = child_sock.protocol if hasattr(child_sock, "protocol") else "" net = task.nsproxy.net_ns - netns_id = net.proc_inum if net.has_member("proc_inum") else net.ns.inum + netns_id = net.get_inode() yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields def _generator(self, pids, netns_arg, symbol_table): From fe105dc4a707ae5631322fc75bfb2c441d8ae034 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 21 Dec 2021 15:01:29 +1100 Subject: [PATCH 020/526] Add doc strings and typing info everywhere. Improve some variable names --- .../framework/plugins/linux/sockstat.py | 160 +++++++++++++++--- 1 file changed, 139 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 4f9ac64de..a73d5581a 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -4,7 +4,7 @@ # Author: Gustavo Moreira import logging -from typing import Callable +from typing import Callable, Tuple, List, Dict from volatility3.framework import renderers, interfaces, exceptions, constants, objects from volatility3.framework.configuration import requirements @@ -17,6 +17,7 @@ from volatility3.plugins.linux import lsof vollog = logging.getLogger(__name__) class SockHandlers(interfaces.configuration.VersionableInterface): + """Handles several socket families extracting the sockets information.""" _required_framework_version = (2, 0, 0) @@ -40,7 +41,17 @@ class SockHandlers(interfaces.configuration.VersionableInterface): "AF_BLUETOOTH": self._bluetooth_sock, } - def _build_network_devices_map(self, netns_id: int): + def _build_network_devices_map(self, netns_id: int) -> Dict: + """Given a namespace ID it returns a dictionary mapping each network + interface index (ifindex) to its network interface name: + + Args: + netns_id: The network namespace ID + + Returns: + netdevices_map: Mapping network interface index (ifindex) to network + interface name + """ netdevices_map = {} nethead = self._vmlinux.object_from_symbol(symbol_name="net_namespace_list") net_symname = self._vmlinux.symbol_table_name + constants.BANG + "net" @@ -53,7 +64,17 @@ class SockHandlers(interfaces.configuration.VersionableInterface): netdevices_map[net_dev.ifindex] = dev_name return netdevices_map - def process_sock(self, sock: objects.StructType): + def process_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str], Dict]: + """Takes a kernel generic `sock` object and processes it with its respective socket family + + Args: + sock: Kernel generic `sock` object + + Returns a tuple with: + sock: The respective kernel's *_sock object for that socket family + sock_stat: A tuple with the source, destination and state strings. + extended: A dictionary with key/value extended information. + """ family = sock.family extended = {} sock_handler = self._sock_family_handlers.get(family) @@ -95,7 +116,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): extended["filter_type"] = "reuseport_filter" self._extract_socket_filter_info(sock_reuseport_cb, extended) - def _extract_socket_filter_info(self, sock_filter: objects.Pointer, extended: dict): + def _extract_socket_filter_info(self, sock_filter: objects.Pointer, extended: dict) -> None: extended["bpf_filter_type"] = "cBPF" if not sock_filter.has_member("prog") or not sock_filter.prog: @@ -113,7 +134,16 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if bpfprog_name: extended["bpf_filter_name"] = bpfprog_name - def _unix_sock(self, sock: objects.StructType): + def _unix_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + """Handles the AF_UNIX socket family + + Args: + sock: Kernel generic `sock` object + + Returns: + unix_sock: The kernel's `unix_sock` object + sock_stat: A tuple with the source, destination and state strings. + """ unix_sock = sock.cast("unix_sock") state = unix_sock.state saddr = unix_sock.name @@ -130,7 +160,16 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return unix_sock, sock_stat - def _inet_sock(self, sock: objects.StructType): + def _inet_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + """Handles the AF_INET/6 socket families + + Args: + sock: Kernel generic `sock` object + + Returns: + inet_sock: The kernel's `inet_sock` object + sock_stat: A tuple with the source, destination and state strings. + """ inet_sock = sock.cast("inet_sock") saddr = inet_sock.src_addr sport = inet_sock.src_port @@ -146,7 +185,16 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return inet_sock, sock_stat - def _netlink_sock(self, sock: objects.StructType): + def _netlink_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + """Handles the AF_NETLINK socket family + + Args: + sock: Kernel generic `sock` object + + Returns: + netlink_sock: The kernel's `netlink_sock` object + sock_stat: A tuple with the source, destination and state strings. + """ netlink_sock = sock.cast("netlink_sock") saddr_list = [] @@ -175,7 +223,16 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return netlink_sock, sock_stat - def _vsock_sock(self, sock: objects.StructType): + def _vsock_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + """Handles the AF_VSOCK socket family + + Args: + sock: Kernel generic `sock` object + + Returns: + vsock_sock: The kernel `vsock_sock` object + sock_stat: A tuple with the source, destination and state strings. + """ vsock_sock = sock.cast("vsock_sock") saddr = vsock_sock.local_addr.svm_cid sport = vsock_sock.local_addr.svm_port @@ -188,7 +245,16 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return vsock_sock, sock_stat - def _packet_sock(self, sock: objects.StructType): + def _packet_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + """Handles the AF_PACKET socket family + + Args: + sock: Kernel generic `sock` object + + Returns: + packet_sock: The kernel's `packet_sock` object + sock_stat: A tuple with the source, destination and state strings. + """ packet_sock = sock.cast("packet_sock") ifindex = packet_sock.ifindex dev_name = self._netdevices.get(ifindex, "") if ifindex > 0 else "ANY" @@ -199,7 +265,16 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return packet_sock, sock_stat - def _xdp_sock(self, sock: objects.StructType): + def _xdp_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + """Handles the AF_XDP socket family + + Args: + sock: Kernel generic `sock` object + + Returns: + xdp_sock: The kernel's `xdp_sock` object + sock_stat: A tuple with the source, destination and state strings. + """ xdp_sock = sock.cast("xdp_sock") device = xdp_sock.dev if not device: @@ -228,7 +303,16 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = saddr_tag, daddr_tag, state return xdp_sock, sock_stat - def _bluetooth_sock(self, sock: objects.StructType): + def _bluetooth_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + """Handles the AF_BLUETOOTH socket family + + Args: + sock: Kernel generic `sock` object + + Returns: + bt_sock: The kernel's `bt_sock` object + sock_stat: A tuple with the source, destination and state strings. + """ bt_sock = sock.cast("bt_sock") def bt_addr(addr): @@ -290,12 +374,26 @@ class Sockstat(plugins.PluginInterface): @classmethod def list_sockets(cls, context: interfaces.context.ContextInterface, - vmlinux_module_name: str, + symbol_table: str, filter_func: Callable[[int], bool] = lambda _: False): + """Returns every single socket descriptor + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of the kernel module on which to operate + filter_func: A function which takes a task object and returns True if the task should be ignored/filtered + + Yields: + task: Kernel's task object + netns_id: Network namespace ID + fd_num: File descriptor number + family: Socket family string (AF_UNIX, AF_INET, etc) + sock_type: Socket type string (STREAM, DGRAM, etc) + protocol: Protocol string (UDP, TCP, etc) + sock_fields: A tuple with the *_sock object, the sock stats and the + extended info dictionary """ - Returns every single socket descriptor - """ - vmlinux = context.modules[vmlinux_module_name] + vmlinux = context.modules[symbol_table] sfop_addr = vmlinux.object_from_symbol("socket_file_ops").vol.offset dfop_addr = vmlinux.object_from_symbol("sockfs_dentry_operations").vol.offset @@ -339,13 +437,31 @@ class Sockstat(plugins.PluginInterface): netns_id = net.get_inode() yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields - def _generator(self, pids, netns_arg, symbol_table): + def _generator(self, pids: List[int], netns_id_arg: int, symbol_table: str): + """Enumerate tasks sockets. Each row represents a kernel socket. + + Args: + pids: List of PIDs to filter. If a empty list or + netns_id_arg: If a network namespace ID is set, it will only show this namespace. + symbol_table: The name of the kernel module on which to operate + + Yields: + netns_id: Network namespace ID + family: Socket family string (AF_UNIX, AF_INET, etc) + sock_type: Socket type string (STREAM, DGRAM, etc) + protocol: Protocol string (UDP, TCP, etc) + source: Source address string + destination: Destination address string + state: State strings (LISTEN, CONNECTED, etc) + tasks: String with a list of tasks and FDs using a socket. It can also have + exteded information such as socket filters, bpf info, etc. + """ filter_func = lsof.pslist.PsList.create_pid_filter(pids) socket_generator = self.list_sockets(self.context, symbol_table, filter_func=filter_func) tasks_per_sock = {} - for task, netns, fd_num, family, sock_type, protocol, sock_fields in socket_generator: - if netns_arg and netns_arg != netns: + for task, netns_id, fd_num, family, sock_type, protocol, sock_fields in socket_generator: + if netns_id_arg and netns_id_arg != netns_id: continue sock, sock_stat, extended = sock_fields @@ -356,8 +472,10 @@ class Sockstat(plugins.PluginInterface): extended_str = ",".join(f"{k}={v}" for k, v in extended.items()) task_info = f"{task_info},{extended_str}" - fields = netns, family, sock_type, protocol, *sock_stat + fields = netns_id, family, sock_type, protocol, *sock_stat + # Each row represents a kernel socket, so let's group the task FDs + # by socket using the socket address sock_addr = sock.vol.offset tasks_per_sock.setdefault(sock_addr, {}) tasks_per_sock[sock_addr].setdefault('tasks', []) @@ -373,7 +491,7 @@ class Sockstat(plugins.PluginInterface): def run(self): pids = self.config.get('pids') - netns = self.config['netns'] + netns_id = self.config['netns'] symbol_table = self.config['kernel'] tree_grid_args = [("NetNS", int), @@ -385,4 +503,4 @@ class Sockstat(plugins.PluginInterface): ("State", str), ("Tasks", str)] - return renderers.TreeGrid(tree_grid_args, self._generator(pids, netns, symbol_table)) + return renderers.TreeGrid(tree_grid_args, self._generator(pids, netns_id, symbol_table)) From 8f8e04da97816a2f3236fdbd7782ad7f0c5b6020 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 21 Dec 2021 15:52:22 +1100 Subject: [PATCH 021/526] vfs_inode is not being used nor required --- volatility3/framework/plugins/linux/sockstat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index a73d5581a..f526d614e 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -416,8 +416,7 @@ class Sockstat(plugins.PluginInterface): socket_alloc = linux.LinuxUtilities.container_of(d_inode, "socket_alloc", "vfs_inode", vmlinux) socket = socket_alloc.socket - vfs_inode = socket_alloc.vfs_inode - if not (socket and vfs_inode): + if not (socket and socket.sk): continue sock = socket.sk.dereference() From b40391ee1f8e5130f52c3716e8e181eedb912cc7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 17 Feb 2022 11:19:58 +1100 Subject: [PATCH 022/526] Returning a starred expression is not yet supported in python 3.6. Fixed --- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index f526d614e..0346cf7e4 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -80,10 +80,10 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_handler = self._sock_family_handlers.get(family) if sock_handler: try: - sock_fields = sock_handler(sock) + unix_sock, sock_stat = sock_handler(sock) self._update_extended_socket_filters_info(sock, extended) - return *sock_fields, extended + return unix_sock, sock_stat, extended except exceptions.SymbolError as e: # Cannot finds the *_sock type in the symbols vollog.log(constants.LOGLEVEL_V, "Error processing socket family '%s': %s", family, e) From 643a8cc74cada83ec5d6341298c31481a0e40ec4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 26 Feb 2022 10:14:23 +1100 Subject: [PATCH 023/526] Make this method private using just a single leading underscore --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3f286e1a3..c9ad48044 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -565,7 +565,7 @@ class net(objects.StructType): raise AttributeError("Unable to find net_namespace inode") class sock(objects.StructType): - def __get_vol_kernel_module_name(self): + def _get_vol_kernel_module_name(self): symbol_table_arr = self.vol.type_name.split("!", 1) symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None @@ -592,7 +592,7 @@ class sock(objects.StructType): if not self.sk_socket: return 0 - kernel_module_name = self.__get_vol_kernel_module_name() + kernel_module_name = self._get_vol_kernel_module_name() kernel = self._context.modules[kernel_module_name] socket_alloc = linux.LinuxUtilities.container_of(self.sk_socket, "socket_alloc", "socket", kernel) vfs_inode = socket_alloc.vfs_inode From 8801a8974b72c19527d85587e312dbf3433d2dc9 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 27 Mar 2022 21:19:04 +0530 Subject: [PATCH 024/526] Added Caption To make it look organized in the left side of the readthedocs. --- doc/source/index.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index 3b5a5d2a8..50eaab694 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -10,6 +10,7 @@ Volatility 3 is Open Source. Here are some guidelines for using Volatility 3 effectively: .. toctree:: + :caption: Documentation basics development @@ -18,10 +19,10 @@ Here are some guidelines for using Volatility 3 effectively: volshell glossary -Python Packages -=============== .. toctree:: + :caption: Python Packages + volatility3 Indices and tables From 26251f28c23a7b1ef361ef4d083ce1c2df95e4e5 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 27 Mar 2022 21:33:35 +0530 Subject: [PATCH 025/526] Structure for Getting started added --- doc/source/index.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index 50eaab694..0d35b02ba 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -25,6 +25,15 @@ Here are some guidelines for using Volatility 3 effectively: volatility3 + +.. toctree:: + :caption: Getting Started + + FAQ + Installation + Linux + Windows + Indices and tables ================== From 0f6bb99d115fd4b629bbdd5085c354cea98d1d72 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 29 Mar 2022 17:41:37 +0530 Subject: [PATCH 026/526] Cross document linked for symbol table Received help from my friend to resolve issues with it Co-authored-by: Abhinandhan S Signed-off-by: Tejas <47889755+tejas15802@users.noreply.github.com> --- doc/source/Linux.rst | 6 ++++++ doc/source/conf.py | 4 +++- doc/source/symbol-tables.rst | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 doc/source/Linux.rst diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst new file mode 100644 index 000000000..280126cb0 --- /dev/null +++ b/doc/source/Linux.rst @@ -0,0 +1,6 @@ +Linux +===== + +How to create symbol tables + +- :ref:`symbol-tables:Mac or Linux symbol tables`. diff --git a/doc/source/conf.py b/doc/source/conf.py index 731a73d56..cadf6d3f2 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -111,9 +111,11 @@ needs_sphinx = '2.0' # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'sphinx.ext.viewcode' + 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.autosectionlabel' ] +autosectionlabel_prefix_document = True + try: import sphinx_autodoc_typehints diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 245dd9c67..36b283fff 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -38,7 +38,7 @@ following command: The :envvar:`PYTHONPATH` environment variable is not required if the Volatility library is installed in the system's library path or a virtual environment. -Mac/Linux symbol tables +Mac or Linux symbol tables ----------------------- For Mac/Linux systems, both use the same mechanism for identification. JSON files live under the symbol directories, From 7ac6a60ff9546e89919e33598142d19b30f8082c Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Wed, 30 Mar 2022 00:05:08 +0530 Subject: [PATCH 027/526] Updated linux page similar to vol2 wiki --- doc/source/Linux.rst | 73 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 280126cb0..decc0c246 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -1,6 +1,71 @@ -Linux -===== +Linux Tutorial +============== + +This guide gives you a brief introduction to how volatility3 works and some demonstration on suite of plugins available from + +Procedure to create symbol tables for linux +-------------------------------------------- + +To create symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. +You can also find some ISF files from this website `Linux ISF Server `_ Which is built and maintained by `kevthehermit `_. + +Using plugins +------------- + +The following is the syntax to run volatility tool. + +.. code-block:: shell-session + + $ python3 vol.py -f plugin_name plugin_option + +List of Plugins +---------------- + +Following are the list of linux plugins available for volatility3. More plugins will be available on future releases. +For plugin requests, Please create an issue with description of the plugin. + +.. code-block:: shell-session + + $ vol3 --help | grep -i linux + + banners.Banners Attempts to identify potential linux banners in an + linux.bash.Bash Recovers bash command history from memory. + linux.check_afinfo.Check_afinfo + linux.check_creds.Check_creds + linux.check_idt.Check_idt + linux.check_modules.Check_modules + linux.check_syscall.Check_syscall + linux.elfs.Elfs Lists all memory mapped ELF files for all processes. + linux.keyboard_notifiers.Keyboard_notifiers + linux.kmsg.Kmsg Kernel log buffer reader + linux.lsmod.Lsmod Lists loaded kernel modules. + linux.lsof.Lsof Lists all memory maps for all processes. + linux.malfind.Malfind + linux.proc.Maps Lists all memory maps for all processes. + linux.pslist.PsList + Lists the processes present in a particular linux + linux.pstree.PsTree + linux.tty_check.tty_check + + +Acquiring memory +---------------- + +Volatility does not provide the ability to acquire memory. We recommend using `Lime `_ for this purpose. +It supports 32 and 64 bit captures from native Intel hardware as well as virtual machine guests. +It also supports capture from Android devices. See below for example commands building and running LiME: + +.. code-block:: shell-session + + $ tar -xvzf lime-forensics-1.1-r14.tar.gz + $ cd lime-forensics-1.1-r14/src + $ make + .... + CC [M] /home/mhl/Downloads/src/tcp.o + CC [M] /home/mhl/Downloads/src/disk.o + .... + $ sudo insmod lime-3.2.0-23-generic.ko "path=/home/mhl/ubuntu.lime format=lime" + $ ls -alh /home/mhl/ubuntu.lime + -r--r--r-- 1 root root 2.0G Aug 17 19:37 /home/mhl/ubuntu.lime -How to create symbol tables -- :ref:`symbol-tables:Mac or Linux symbol tables`. From 0d4c4b58081b808f248f5fe51516a7b5a7081cf2 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Wed, 30 Mar 2022 08:38:06 +0530 Subject: [PATCH 028/526] Making changes as per review --- doc/source/Linux.rst | 82 +++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 46 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index decc0c246..6ee797c76 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -3,55 +3,10 @@ Linux Tutorial This guide gives you a brief introduction to how volatility3 works and some demonstration on suite of plugins available from -Procedure to create symbol tables for linux --------------------------------------------- - -To create symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. -You can also find some ISF files from this website `Linux ISF Server `_ Which is built and maintained by `kevthehermit `_. - -Using plugins -------------- - -The following is the syntax to run volatility tool. - -.. code-block:: shell-session - - $ python3 vol.py -f plugin_name plugin_option - -List of Plugins ----------------- - -Following are the list of linux plugins available for volatility3. More plugins will be available on future releases. -For plugin requests, Please create an issue with description of the plugin. - -.. code-block:: shell-session - - $ vol3 --help | grep -i linux - - banners.Banners Attempts to identify potential linux banners in an - linux.bash.Bash Recovers bash command history from memory. - linux.check_afinfo.Check_afinfo - linux.check_creds.Check_creds - linux.check_idt.Check_idt - linux.check_modules.Check_modules - linux.check_syscall.Check_syscall - linux.elfs.Elfs Lists all memory mapped ELF files for all processes. - linux.keyboard_notifiers.Keyboard_notifiers - linux.kmsg.Kmsg Kernel log buffer reader - linux.lsmod.Lsmod Lists loaded kernel modules. - linux.lsof.Lsof Lists all memory maps for all processes. - linux.malfind.Malfind - linux.proc.Maps Lists all memory maps for all processes. - linux.pslist.PsList - Lists the processes present in a particular linux - linux.pstree.PsTree - linux.tty_check.tty_check - - Acquiring memory ---------------- -Volatility does not provide the ability to acquire memory. We recommend using `Lime `_ for this purpose. +Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `Lime `_ for this purpose. It supports 32 and 64 bit captures from native Intel hardware as well as virtual machine guests. It also supports capture from Android devices. See below for example commands building and running LiME: @@ -68,4 +23,39 @@ It also supports capture from Android devices. See below for example commands bu $ ls -alh /home/mhl/ubuntu.lime -r--r--r-- 1 root root 2.0G Aug 17 19:37 /home/mhl/ubuntu.lime +Procedure to create symbol tables for linux +-------------------------------------------- + +To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. +We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. + + +Using plugins +------------- + +The following is the syntax to run volatility tool. + +.. code-block:: shell-session + + $ python3 vol.py -f plugin_name plugin_option + +Listing plugins +--------------- + +Following are the list of linux plugins available for volatility3. More plugins will be available on future releases. +For plugin requests, Please create an issue with description of the plugin. + +.. code-block:: shell-session + + $ vol3 --help | grep -i linux. | head -n 5 + banners.Banners Attempts to identify potential linux banners in an + linux.bash.Bash Recovers bash command history from memory. + linux.check_afinfo.Check_afinfo + linux.check_creds.Check_creds + linux.check_idt.Check_idt + + + + + From f4e1533628dcfe5bf45e775259afa68af3e3983e Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sat, 2 Apr 2022 17:28:36 +0530 Subject: [PATCH 029/526] Order changed --- doc/source/Linux.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 6ee797c76..e7f81ff75 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -30,15 +30,6 @@ To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symb We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. -Using plugins -------------- - -The following is the syntax to run volatility tool. - -.. code-block:: shell-session - - $ python3 vol.py -f plugin_name plugin_option - Listing plugins --------------- @@ -55,6 +46,15 @@ For plugin requests, Please create an issue with description of the plugin. linux.check_idt.Check_idt +Using plugins +------------- + +The following is the syntax to run volatility tool. + +.. code-block:: shell-session + + $ python3 vol.py -f plugin_name plugin_option + From b6830a1f3172588e310b149da1bad28377903ad3 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sat, 2 Apr 2022 17:38:46 +0530 Subject: [PATCH 030/526] Additional context for proceudre to create symbol --- doc/source/Linux.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index e7f81ff75..afbccec49 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -28,6 +28,7 @@ Procedure to create symbol tables for linux To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. +After creating the file or downloading the file from the ISF server, please place the file under the directory ``volatility3/symbols/linux``. Make a directory linux under symbols. Listing plugins From 20f1fc39f243e61d9e6fba9c1472a671791462a8 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sat, 2 Apr 2022 18:46:16 +0530 Subject: [PATCH 031/526] Example 1 Added --- doc/source/Linux.rst | 128 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index afbccec49..2913a93b2 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -57,6 +57,134 @@ The following is the syntax to run volatility tool. $ python3 vol.py -f plugin_name plugin_option +Example +------- + +Example 1 +~~~~~~~~~ + +In this example we will be using memory dump from Insomni'hack teaser 2020 CTF. Challenge name Getdents, you can find the memory dump +in the link `here `_ . We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. +I'd like to say thanks to `stuxnet `_ for providing this memory dump and `writeup `_. +.. code-block:: shell-session + $ python3 vol.py -f memory.vmem banners + + Volatility 3 Framework 2.0.3 + + Progress: 100.00 PDB scanning finished + Offset Banner + + 0x141c1390 Linux version 4.15.0-42-generic (buildd@lgw01-amd64-023) (gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)) #45-Ubuntu SMP Thu Nov 15 19:32:57 UTC 2018 (Ubuntu 4.15.0-42.45-generic 4.15.18) + 0x63a00160 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x6455c4d4 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x6e1e055f Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x7fde0010 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + + +This above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for ISF file from the ISF server. +If you do not find the ISF file then, please follow the instructions on :ref:`Linux:Procedure to create symbol tables for linux`. After that place the ISF file under ``volatility3/symbols/linux`` directory. + +.. tip:: Use the banner text which is most repeated to search from ISF Server. + + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.pslist + + Volatility 3 Framework 2.0.3 Stacking attempts finished + + PID PPID COMM + + 1 0 systemd + 2 0 kthreadd + 3 2 kworker/0:0 + 4 2 kworker/0:0H + 5 2 kworker/u256:0 + 6 2 mm_percpu_wq + 7 2 ksoftirqd/0 + 8 2 rcu_sched + 9 2 rcu_bh + 10 2 migration/0 + 11 2 watchdog/0 + 12 2 cpuhp/0 + 13 2 kdevtmpfs + 14 2 netns + 15 2 rcu_tasks_kthre + 16 2 kauditd + ..... + +``linux.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.pstree + Volatility 3 Framework 2.0.3 + Progress: 100.00 Stacking attempts finished + PID PPID COMM + + 1 0 systemd + * 636 1 polkitd + * 514 1 acpid + * 1411 1 pulseaudio + * 517 1 rsyslogd + * 637 1 cups-browsed + * 903 1 whoopsie + * 522 1 ModemManager + * 525 1 cron + * 526 1 avahi-daemon + ** 542 526 avahi-daemon + * 657 1 unattended-upgr + * 914 1 kerneloops + * 532 1 dbus-daemon + * 1429 1 ibus-x11 + * 929 1 kerneloops + * 1572 1 gsd-printer + * 933 1 upowerd + * 1071 1 rtkit-daemon + * 692 1 gdm3 + ** 1234 692 gdm-session-wor + *** 1255 1234 gdm-x-session + **** 1257 1255 Xorg + **** 1266 1255 gnome-session-b + ***** 1537 1266 gsd-clipboard + ***** 1539 1266 gsd-color + ***** 1542 1266 gsd-datetime + ***** 2950 1266 deja-dup-monito + ***** 1546 1266 gsd-housekeepin + ***** 1548 1266 gsd-keyboard + ***** 1550 1266 gsd-media-keys + +``linux.pstree`` helps us to display the parent child relation of processes. + +Now to find the commands ran in bash shell. Lets use ``linux.bash``. + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.bash + + Volatility 3 Framework 2.0.3 + Progress: 100.00 Stacking attempts finished + PID Process CommandTime Command + + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 AWAVH�� + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 rub + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 uname -a + 1733 bash 2020-01-16 14:00:36.000000 uname -a + 1733 bash 2020-01-16 14:00:36.000000 sudo apt autoclean + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:41.000000 chmod +x meterpreter + 1733 bash 2020-01-16 14:00:42.000000 sudo ./meterpreter From ada212da9232a0fff1cdc21bfc2ed7c1793aa20f Mon Sep 17 00:00:00 2001 From: TEJENDRA SARADHI <47889755+tejas15802@users.noreply.github.com> Date: Sun, 3 Apr 2022 06:13:53 +0530 Subject: [PATCH 032/526] lime to LiME Fix inconsistency --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 2913a93b2..294f8ff63 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -6,7 +6,7 @@ This guide gives you a brief introduction to how volatility3 works and some demo Acquiring memory ---------------- -Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `Lime `_ for this purpose. +Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `LiME `_ for this purpose. It supports 32 and 64 bit captures from native Intel hardware as well as virtual machine guests. It also supports capture from Android devices. See below for example commands building and running LiME: From bbffe9f620924c10de7a05efe1a41692aad8d26d Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Wed, 6 Apr 2022 21:08:07 +0530 Subject: [PATCH 033/526] Windows page added and few commands in example1 --- doc/source/Windows.rst | 88 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 doc/source/Windows.rst diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst new file mode 100644 index 000000000..0d8c92c41 --- /dev/null +++ b/doc/source/Windows.rst @@ -0,0 +1,88 @@ +Windows Tutorial +================ + +This guide gives you a brief introduction to how volatility3 works and some demonstration on suite of plugins available from + +Acquiring memory +---------------- + +Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `WinPmem `_ for this purpose. + +Listing Plugins +--------------- + + $ vol3 --help | grep windows | head -n 5 + windows.bigpools.BigPools + windows.cmdline.CmdLine + windows.crashinfo.Crashinfo + windows.dlllist.DllList + Lists the loaded modules in a particular windows + +Using plugins +------------- + +The following is the syntax to run volatility tool. + +.. code-block:: shell-session + + $ python3 vol.py -f plugin_name plugin_option + + +Example +------- + +Example 1 +~~~~~~~~~ + +In this example we will be using memory dump from PragyanCTF'22. The dump is available `here `_. +We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. + +In windows memory forensics using volatility3, most of the times we do not require creating a ISF file. + +.. code-block:: shell-session + + $ vol3 -f MemDump.DMP windows.pslist | head -n 10 + + Volatility 3 Framework 2.0.2 PDB scanning finished + + PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output + + 4 0 System 0xfa8000cbc040 85 492 N/A False 2022-02-07 16:30:12.000000 N/A Disabled + 276 4 smss.exe 0xfa8001e04040 2 29 N/A False 2022-02-07 16:30:12.000000 N/A Disabled + 352 336 csrss.exe 0xfa8002110b30 9 375 0 False 2022-02-07 16:30:13.000000 N/A Disabled + 404 336 wininit.exe 0xfa800219f060 3 74 0 False 2022-02-07 16:30:13.000000 N/A Disabled + 412 396 csrss.exe 0xfa80021c5b30 9 224 1 False 2022-02-07 16:30:13.000000 N/A Disabled + 468 396 winlogon.exe 0xfa8002284060 5 113 1 False 2022-02-07 16:30:14.000000 N/A Disabled + +``windows.pslist`` helps us list the processes running while the memory dump was taken. + +.. code-block:: shell-session + + $ vol3 -f MemDump.DMP windows.pstree | head -n 20 + Volatility 3 Framework 2.0.2 PDB scanning finished + + PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime + + 4 0 System 0xfa8000cbc040 85 492 N/A False 2022-02-07 16:30:12.000000 N/A + * 276 4 smss.exe 0xfa8001e04040 2 29 N/A False 2022-02-07 16:30:12.000000 N/A + 352 336 csrss.exe 0xfa8002110b30 9 375 0 False 2022-02-07 16:30:13.000000 N/A + 404 336 wininit.exe 0xfa800219f060 3 74 0 False 2022-02-07 16:30:13.000000 N/A + * 504 404 services.exe 0xfa80022ccb30 7 190 0 False 2022-02-07 16:30:14.000000 N/A + ** 960 504 svchost.exe 0xfa8001c17b30 39 1003 0 False 2022-02-07 16:30:14.000000 N/A + ** 1216 504 svchost.exe 0xfa80026e0b30 18 311 0 False 2022-02-07 16:30:15.000000 N/A + ** 1312 504 svchost.exe 0xfa8002740380 19 287 0 False 2022-02-07 16:30:15.000000 N/A + ** 1984 504 taskhost.exe 0xfa8002eb1b30 8 129 1 False 2022-02-07 16:30:27.000000 N/A + ** 804 504 svchost.exe 0xfa80024ca5f0 20 450 0 False 2022-02-07 16:30:14.000000 N/A + *** 100 804 audiodg.exe 0xfa80025b4b30 6 131 0 False 2022-02-07 16:30:14.000000 N/A + ** 1568 504 SearchIndexer. 0xfa800254b480 12 616 0 False 2022-02-07 16:30:32.000000 N/A + ** 744 504 svchost.exe 0xfa8002477b30 8 265 0 False 2022-02-07 16:30:14.000000 N/A + ** 1096 504 svchost.exe 0xfa800260db30 14 357 0 False 2022-02-07 16:30:14.000000 N/A + ** 616 504 svchost.exe 0xfa8002b86ab0 13 314 0 False 2022-02-07 16:32:16.000000 N/A + ** 624 504 svchost.exe 0xfa8002410630 10 350 0 False 2022-02-07 16:30:14.000000 N/A + +``windows.pstree`` helps us to display the parent child relation of processes. + + + + + From 39db890ebd7dd0ef191e96831133e46812ed112b Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Wed, 6 Apr 2022 21:10:41 +0530 Subject: [PATCH 034/526] Fix code block syntax highlight --- doc/source/Windows.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index 0d8c92c41..5a197a73e 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -11,6 +11,8 @@ Volatility does not provide the ability to acquire memory. In this tutorial we w Listing Plugins --------------- +.. code-block:: shell-session + $ vol3 --help | grep windows | head -n 5 windows.bigpools.BigPools windows.cmdline.CmdLine From 215a43478932c9fa2f6db52487aab4f3fc672576 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Wed, 6 Apr 2022 21:13:16 +0530 Subject: [PATCH 035/526] Update alias vol3 to python3 vol.py --- doc/source/Windows.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index 5a197a73e..b086cc57b 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -12,8 +12,8 @@ Listing Plugins --------------- .. code-block:: shell-session - - $ vol3 --help | grep windows | head -n 5 + + $ python3 vol.py --help | grep windows | head -n 5 windows.bigpools.BigPools windows.cmdline.CmdLine windows.crashinfo.Crashinfo @@ -43,7 +43,7 @@ In windows memory forensics using volatility3, most of the times we do not requi .. code-block:: shell-session - $ vol3 -f MemDump.DMP windows.pslist | head -n 10 + $ python3 vol.py -f MemDump.DMP windows.pslist | head -n 10 Volatility 3 Framework 2.0.2 PDB scanning finished @@ -60,7 +60,7 @@ In windows memory forensics using volatility3, most of the times we do not requi .. code-block:: shell-session - $ vol3 -f MemDump.DMP windows.pstree | head -n 20 + $ python3 vol.py -f MemDump.DMP windows.pstree | head -n 20 Volatility 3 Framework 2.0.2 PDB scanning finished PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime From 333bb090cd9700f346ea10ddc4635cc87732e0b3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 29 Apr 2022 10:21:28 +1000 Subject: [PATCH 036/526] Minor. Constants regrouped --- volatility3/framework/constants/linux/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index a35b6f02e..550690b90 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -12,6 +12,9 @@ KERNEL_NAME = "__kernel__" PAGE_SHIFT = 12 """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" +# include/linux/sched.h +PF_KTHREAD = 0x00200000 # I'm a kernel thread + # Standard well-defined IP protocols. # ref: include/uapi/linux/in.h IP_PROTOCOLS = { @@ -227,6 +230,3 @@ BLUETOOTH_PROTOCOLS = ( "HIDP", "AVDTP", ) - -# include/linux/sched.h -PF_KTHREAD = 0x00200000 # I'm a kernel thread From a17d1617f4b0185edca858f5defa43027a945c62 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 29 Apr 2022 12:08:04 +1000 Subject: [PATCH 037/526] Remove author --- volatility3/framework/plugins/linux/sockstat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 0346cf7e4..fd172fc43 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -1,7 +1,6 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -# Author: Gustavo Moreira import logging from typing import Callable, Tuple, List, Dict From 37d8328a18c6d7ddec5bfa74d1ba436351b8df35 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 29 Apr 2022 12:41:11 +1000 Subject: [PATCH 038/526] Adding typing information to container_of() --- .../framework/symbols/linux/__init__.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 739ecbedb..347d0b366 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import List, Tuple, Iterator +from typing import List, Tuple, Iterator, Optional from volatility3 import framework from volatility3.framework import exceptions, constants, interfaces, objects @@ -283,9 +283,25 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): list_start = getattr(list_struct, list_member) @classmethod - def container_of(cls, addr, type_name, member_name, vmlinux): + def container_of( + cls, addr: int, type_name: str, member_name: str, vmlinux: interfaces.context.ModuleInterface + ) -> Optional[interfaces.objects.ObjectInterface]: + """Cast a member of a structure out to the containing structure. + It mimicks the Linux kernel macro container_of() see include/linux.kernel.h + + Args: + addr: The pointer to the member. + type_name: The type of the container struct this is embedded in. + member_name: The name of the member within the struct. + vmlinux: The kernel symbols object + + Returns: + The constructed object or None + """ + if not addr: return + type_dec = vmlinux.get_type(type_name) member_offset = type_dec.relative_child_offset(member_name) container_addr = addr - member_offset From 1679769da74163a6821c58b7352f5c0420578773 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 29 Apr 2022 12:58:32 +1000 Subject: [PATCH 039/526] Catch exception when it is unable to get the kernel module name --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 05a88ba5a..fe495f88d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -634,7 +634,11 @@ class sock(objects.StructType): if not self.sk_socket: return 0 - kernel_module_name = self._get_vol_kernel_module_name() + try: + kernel_module_name = self._get_vol_kernel_module_name() + except ValueError: + return 0 + kernel = self._context.modules[kernel_module_name] socket_alloc = linux.LinuxUtilities.container_of(self.sk_socket, "socket_alloc", "socket", kernel) vfs_inode = socket_alloc.vfs_inode From f919d29c50a374836a2e934f1efdf2f4b119cc0d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 30 Apr 2022 14:14:18 +1000 Subject: [PATCH 040/526] Changing properties for getters --- .../framework/plugins/linux/sockstat.py | 43 ++++++------ .../symbols/linux/extensions/__init__.py | 68 +++++++------------ 2 files changed, 46 insertions(+), 65 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index fd172fc43..7d4b20da0 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -74,7 +74,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat: A tuple with the source, destination and state strings. extended: A dictionary with key/value extended information. """ - family = sock.family + family = sock.get_family() extended = {} sock_handler = self._sock_family_handlers.get(family) if sock_handler: @@ -144,13 +144,13 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat: A tuple with the source, destination and state strings. """ unix_sock = sock.cast("unix_sock") - state = unix_sock.state - saddr = unix_sock.name - sinode = unix_sock.inode + state = unix_sock.get_state() + saddr = unix_sock.get_name() + sinode = unix_sock.get_inode() if unix_sock.peer != 0: peer = unix_sock.peer.dereference().cast("unix_sock") - daddr = peer.name - dinode = peer.inode + daddr = peer.get_name() + dinode = peer.get_inode() else: daddr = dinode = "" @@ -170,13 +170,13 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat: A tuple with the source, destination and state strings. """ inet_sock = sock.cast("inet_sock") - saddr = inet_sock.src_addr - sport = inet_sock.src_port - daddr = inet_sock.dst_addr - dport = inet_sock.dst_port - state = inet_sock.state + saddr = inet_sock.get_src_addr() + sport = inet_sock.get_src_port() + daddr = inet_sock.get_dst_addr() + dport = inet_sock.get_dst_port() + state = inet_sock.get_state() - if inet_sock.family == "AF_INET6": + if inet_sock.get_family() == "AF_INET6": saddr = f"[{saddr}]" saddr_tag = f"{saddr}:{sport}" @@ -217,7 +217,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): saddr_tag = ",".join(saddr_list) daddr_tag = ",".join(daddr_list) - state = netlink_sock.state + state = netlink_sock.get_state() sock_stat = saddr_tag, daddr_tag, state return netlink_sock, sock_stat @@ -260,7 +260,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): saddr_tag = f"{dev_name}" daddr_tag = "" - state = packet_sock.state + state = packet_sock.get_state() sock_stat = saddr_tag, daddr_tag, state return packet_sock, sock_stat @@ -318,15 +318,16 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return ":".join(reversed(["%02x" % x for x in addr.b])) saddr_tag = daddr_tag = "" - if bt_sock.protocol == "HCI": + bt_protocol = bt_sock.get_protocol() + if bt_protocol == "HCI": pinfo = bt_sock.cast("hci_pinfo") - elif bt_sock.protocol == "L2CAP": + elif bt_protocol == "L2CAP": pinfo = bt_sock.cast("l2cap_pinfo") src_addr = bt_addr(pinfo.chan.src) dst_addr = bt_addr(pinfo.chan.dst) saddr_tag = f"{src_addr}" daddr_tag = f"{dst_addr}" - elif bt_sock.protocol == "RFCOMM": + elif bt_protocol == "RFCOMM": pinfo = bt_sock.cast("rfcomm_pinfo") src_addr = bt_addr(pinfo.src) dst_addr = bt_addr(pinfo.dst) @@ -334,7 +335,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): saddr_tag = f"[{src_addr}]:{channel}" daddr_tag = f"{dst_addr}" else: - vollog.log(constants.LOGLEVEL_V, "Unsupported bluetooth protocol '%s'", bt_sock.protocol) + vollog.log(constants.LOGLEVEL_V, "Unsupported bluetooth protocol '%s'", bt_protocol) state = bt_sock.state sock_stat = saddr_tag, daddr_tag, state @@ -420,8 +421,8 @@ class Sockstat(plugins.PluginInterface): sock = socket.sk.dereference() - sock_type = sock.type - family = sock.family + sock_type = sock.get_type() + family = sock.get_family() sock_handler = SockHandlers(vmlinux, task) sock_fields = sock_handler.process_sock(sock) @@ -429,7 +430,7 @@ class Sockstat(plugins.PluginInterface): continue child_sock = sock_fields[0] - protocol = child_sock.protocol if hasattr(child_sock, "protocol") else "" + protocol = child_sock.get_protocol() net = task.nsproxy.net_ns netns_id = net.get_inode() diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fe495f88d..80180ad1b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -617,20 +617,17 @@ class sock(objects.StructType): return module_names[0] - @property - def family(self): + def get_family(self): family_idx = self.__sk_common.skc_family if 0 <= family_idx < len(SOCK_FAMILY): return SOCK_FAMILY[family_idx] else: return "UNKNOWN" - @property - def type(self): + def get_type(self): return SOCK_TYPES.get(self.sk_type, "") - @property - def inode(self): + def get_inode(self): if not self.sk_socket: return 0 @@ -646,8 +643,7 @@ class sock(objects.StructType): return vfs_inode.i_ino class unix_sock(objects.StructType): - @property - def name(self): + def get_name(self): if self.addr: sockaddr_un = self.addr.name.cast("sockaddr_un") saddr = str(utility.array_to_string(sockaddr_un.sun_path)) @@ -655,16 +651,14 @@ class unix_sock(objects.StructType): saddr = "" return saddr - @property - def protocol(self): + def get_protocol(self): return "" - @property - def state(self): + def get_state(self): """Return a string representing the sock state.""" # Unix socket states reuse (a subset) of the inet_sock states contants - if self.sk.type == "STREAM": + if self.sk.get_type() == "STREAM": state_idx = self.sk.__sk_common.skc_state if 0 <= state_idx < len(TCP_STATES): state = TCP_STATES[state_idx] @@ -675,33 +669,29 @@ class unix_sock(objects.StructType): return state - @property - def inode(self): - return self.sk.inode + def get_inode(self): + return self.sk.get_inode() class inet_sock(objects.StructType): - @property - def family(self): + def get_family(self): family_idx = self.sk.__sk_common.skc_family if 0 <= family_idx < len(SOCK_FAMILY): return SOCK_FAMILY[family_idx] else: return "UNKNOWN" - @property - def protocol(self): + def get_protocol(self): # If INET6 family and a proto is defined, we use that specific IPv6 protocol. # Otherwise, we use the standard IP protocol. protocol = IP_PROTOCOLS.get(self.sk.sk_protocol, "UNKNOWN") - if self.family == "AF_INET6": + if self.get_family() == "AF_INET6": protocol = IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) return protocol - @property - def state(self): + def get_state(self): """Return a string representing the sock state.""" - if self.sk.type == "STREAM": + if self.sk.get_type() == "STREAM": state_idx = self.sk.__sk_common.skc_state if 0 <= state_idx < len(TCP_STATES): state = TCP_STATES[state_idx] @@ -712,14 +702,12 @@ class inet_sock(objects.StructType): return state - @property - def src_port(self): + def get_src_port(self): sport_le = getattr(self, "sport", getattr(self, "inet_sport", None)) if sport_le is not None: return socket.htons(sport_le) - @property - def dst_port(self): + def get_dst_port(self): sk_common = self.sk.__sk_common if hasattr(sk_common, "skc_portpair"): dport_le = sk_common.skc_portpair & 0xffff @@ -734,8 +722,7 @@ class inet_sock(objects.StructType): return socket.htons(dport_le) - @property - def src_addr(self): + def get_src_addr(self): sk_common = self.sk.__sk_common family = sk_common.skc_family if family == socket.AF_INET: @@ -756,8 +743,7 @@ class inet_sock(objects.StructType): addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) return socket.inet_ntop(family, addr_bytes) - @property - def dst_addr(self): + def get_dst_addr(self): sk_common = self.sk.__sk_common family = sk_common.skc_family if family == socket.AF_INET: @@ -782,16 +768,14 @@ class inet_sock(objects.StructType): return socket.inet_ntop(family, addr_bytes) class netlink_sock(objects.StructType): - @property - def protocol(self): + def get_protocol(self): protocol_idx = self.sk.sk_protocol if 0 <= protocol_idx < len(NETLINK_PROTOCOLS): return NETLINK_PROTOCOLS[protocol_idx] else: return "UNKNOWN" - @property - def state(self): + def get_state(self): # Netlink is a datagram-oriented service. We can only have # SOCK_RAW or SOCK_DGRAM socket types. # NOTE: We are overriding the netlink_sock.state member here @@ -800,8 +784,7 @@ class netlink_sock(objects.StructType): class packet_sock(objects.StructType): - @property - def protocol(self): + def get_protocol(self): eth_proto = socket.htons(self.num) if eth_proto == 0: return "" @@ -810,15 +793,13 @@ class packet_sock(objects.StructType): else: return f"0x{eth_proto:x}" - @property - def state(self): + def get_state(self): # Packet socket types are either SOCK_RAW or SOCK_DGRAM. return "UNCONNECTED" class bt_sock(objects.StructType): - @property - def protocol(self): + def get_protocol(self): type_idx = self.sk.sk_protocol if 0 <= type_idx < len(BLUETOOTH_PROTOCOLS): state = BLUETOOTH_PROTOCOLS[type_idx] @@ -827,8 +808,7 @@ class bt_sock(objects.StructType): return state - @property - def state(self): + def get_state(self): state_idx = self.sk.__sk_common.skc_state if 0 <= state_idx < len(BLUETOOTH_STATES): state = BLUETOOTH_STATES[state_idx] From 5f70c7031c107645be93a71d392177c123c60307 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 30 Apr 2022 14:29:52 +1000 Subject: [PATCH 041/526] Added kernel 'socket' and 'vsock_sock' struct extensions. This removes hardcoding state in same socket families, taking the information directly from the generic 'socket'. --- .../framework/constants/linux/__init__.py | 10 ++ .../framework/plugins/linux/sockstat.py | 7 +- .../framework/symbols/linux/__init__.py | 2 + .../symbols/linux/extensions/__init__.py | 114 ++++++++++-------- 4 files changed, 83 insertions(+), 50 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 550690b90..0c4d3c376 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -137,6 +137,16 @@ SOCK_FAMILY = ( "AF_XDP", ) +# Socket states +# ref: include/uapi/linux/net.h +SOCKET_STATES = ( + "FREE", + "UNCONNECTED", + "CONNECTING", + "CONNECTED", + "DISCONNECTING" +) + # Netlink protocols # ref: include/uapi/linux/netlink.h NETLINK_PROTOCOLS = ( diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 7d4b20da0..0306bec02 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -92,7 +92,8 @@ class SockHandlers(interfaces.configuration.VersionableInterface): # Even if the sock family is not supported, or the required types # are not present in the symbols, we can still show some general # information about the socket that may be helpful. - saddr_tag = daddr_tag = state = "?" + saddr_tag = daddr_tag = "?" + state = sock.get_state() sock_stat = saddr_tag, daddr_tag, state @@ -237,7 +238,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sport = vsock_sock.local_addr.svm_port daddr = vsock_sock.remote_addr.svm_cid dport = vsock_sock.remote_addr.svm_port - state = "" # Protocol is always 0 + state = vsock_sock.get_state() saddr_tag = f"{saddr}:{sport}" daddr_tag = f"{daddr}:{dport}" @@ -337,7 +338,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): else: vollog.log(constants.LOGLEVEL_V, "Unsupported bluetooth protocol '%s'", bt_protocol) - state = bt_sock.state + state = bt_sock.get_state() sock_stat = saddr_tag, daddr_tag, state return bt_sock, sock_stat diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 347d0b366..977066eac 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -32,10 +32,12 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): # Network self.set_type_class('net', extensions.net) + self.set_type_class('socket', extensions.socket) self.set_type_class('sock', extensions.sock) self.set_type_class('inet_sock', extensions.inet_sock) self.set_type_class('unix_sock', extensions.unix_sock) self.set_type_class('netlink_sock', extensions.netlink_sock) + self.set_type_class('vsock_sock', extensions.vsock_sock) self.set_type_class('packet_sock', extensions.packet_sock) if 'bt_sock' in self.types: self.set_type_class('bt_sock', extensions.bt_sock) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 80180ad1b..644a88632 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -4,7 +4,7 @@ import collections.abc import logging -import socket +import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple from volatility3.framework import constants @@ -12,7 +12,7 @@ from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES -from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS +from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility @@ -606,8 +606,8 @@ class net(objects.StructType): else: raise AttributeError("Unable to find net_namespace inode") -class sock(objects.StructType): - def _get_vol_kernel_module_name(self): +class socket(objects.StructType): + def _get_vol_kernel(self): symbol_table_arr = self.vol.type_name.split("!", 1) symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None @@ -615,8 +615,29 @@ class sock(objects.StructType): if not module_names: raise ValueError(f"No module using the symbol table {symbol_table}") - return module_names[0] + kernel_module_name = module_names[0] + kernel = self._context.modules[kernel_module_name] + return kernel + def get_inode(self): + try: + kernel = self._get_vol_kernel() + except ValueError: + return 0 + + socket_alloc = linux.LinuxUtilities.container_of(self.vol.offset, "socket_alloc", "socket", kernel) + vfs_inode = socket_alloc.vfs_inode + + return vfs_inode.i_ino + + def get_state(self): + socket_state_idx = self.state + if 0 <= socket_state_idx < len(SOCKET_STATES): + return SOCKET_STATES[socket_state_idx] + else: + return "UNKNOWN" + +class sock(objects.StructType): def get_family(self): family_idx = self.__sk_common.skc_family if 0 <= family_idx < len(SOCK_FAMILY): @@ -631,16 +652,11 @@ class sock(objects.StructType): if not self.sk_socket: return 0 - try: - kernel_module_name = self._get_vol_kernel_module_name() - except ValueError: - return 0 + return self.sk_socket.get_inode() - kernel = self._context.modules[kernel_module_name] - socket_alloc = linux.LinuxUtilities.container_of(self.sk_socket, "socket_alloc", "socket", kernel) - vfs_inode = socket_alloc.vfs_inode - - return vfs_inode.i_ino + def get_state(self): + # Return the generic socket state + return self.sk.sk_socket.get_state() class unix_sock(objects.StructType): def get_name(self): @@ -661,13 +677,12 @@ class unix_sock(objects.StructType): if self.sk.get_type() == "STREAM": state_idx = self.sk.__sk_common.skc_state if 0 <= state_idx < len(TCP_STATES): - state = TCP_STATES[state_idx] + return TCP_STATES[state_idx] else: - state = "UNKNOWN" + return "UNKNOWN" else: - state = "UNCONNECTED" - - return state + # Return the generic socket state + return self.sk.sk_socket.get_state() def get_inode(self): return self.sk.get_inode() @@ -694,18 +709,17 @@ class inet_sock(objects.StructType): if self.sk.get_type() == "STREAM": state_idx = self.sk.__sk_common.skc_state if 0 <= state_idx < len(TCP_STATES): - state = TCP_STATES[state_idx] + return TCP_STATES[state_idx] else: - state = "UNKNOWN" + return "UNKNOWN" else: - state = "UNCONNECTED" - - return state + # Return the generic socket state + return self.sk.sk_socket.get_state() def get_src_port(self): sport_le = getattr(self, "sport", getattr(self, "inet_sport", None)) if sport_le is not None: - return socket.htons(sport_le) + return socket_module.htons(sport_le) def get_dst_port(self): sk_common = self.sk.__sk_common @@ -720,12 +734,12 @@ class inet_sock(objects.StructType): else: return - return socket.htons(dport_le) + return socket_module.htons(dport_le) def get_src_addr(self): sk_common = self.sk.__sk_common family = sk_common.skc_family - if family == socket.AF_INET: + if family == socket_module.AF_INET: addr_size = 4 if hasattr(self, "rcv_saddr"): saddr = self.rcv_saddr @@ -733,7 +747,7 @@ class inet_sock(objects.StructType): saddr = self.inet_rcv_saddr else: saddr = sk_common.skc_rcv_saddr - elif family == socket.AF_INET6: + elif family == socket_module.AF_INET6: addr_size = 16 saddr = self.pinet6.saddr else: @@ -741,12 +755,12 @@ class inet_sock(objects.StructType): parent_layer = self._context.layers[self.vol.layer_name] addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) - return socket.inet_ntop(family, addr_bytes) + return socket_module.inet_ntop(family, addr_bytes) def get_dst_addr(self): sk_common = self.sk.__sk_common family = sk_common.skc_family - if family == socket.AF_INET: + if family == socket_module.AF_INET: if hasattr(self, "daddr") and self.daddr: daddr = self.daddr elif hasattr(self, "inet_daddr") and self.inet_daddr: @@ -754,7 +768,7 @@ class inet_sock(objects.StructType): else: daddr = sk_common.skc_daddr addr_size = 4 - elif family == socket.AF_INET6: + elif family == socket_module.AF_INET6: if hasattr(self.pinet6, "daddr"): daddr = self.pinet6.daddr else: @@ -765,7 +779,7 @@ class inet_sock(objects.StructType): parent_layer = self._context.layers[self.vol.layer_name] addr_bytes = parent_layer.read(daddr.vol.offset, addr_size) - return socket.inet_ntop(family, addr_bytes) + return socket_module.inet_ntop(family, addr_bytes) class netlink_sock(objects.StructType): def get_protocol(self): @@ -776,16 +790,26 @@ class netlink_sock(objects.StructType): return "UNKNOWN" def get_state(self): - # Netlink is a datagram-oriented service. We can only have - # SOCK_RAW or SOCK_DGRAM socket types. - # NOTE: We are overriding the netlink_sock.state member here + # Return the generic socket state + return self.sk.sk_socket.get_state() - return "UNCONNECTED" + +class vsock_sock(objects.StructType): + def get_protocol(self): + # The protocol should always be 0 for vsocks + if self.sk.sk_protocol == 0: + return "" + else: + return "UNKNOWN" + + def get_state(self): + # Return the generic socket state + return self.sk.sk_socket.get_state() class packet_sock(objects.StructType): def get_protocol(self): - eth_proto = socket.htons(self.num) + eth_proto = socket_module.htons(self.num) if eth_proto == 0: return "" elif eth_proto in ETH_PROTOCOLS: @@ -794,25 +818,21 @@ class packet_sock(objects.StructType): return f"0x{eth_proto:x}" def get_state(self): - # Packet socket types are either SOCK_RAW or SOCK_DGRAM. - return "UNCONNECTED" + # Return the generic socket state + return self.sk.sk_socket.get_state() class bt_sock(objects.StructType): def get_protocol(self): type_idx = self.sk.sk_protocol if 0 <= type_idx < len(BLUETOOTH_PROTOCOLS): - state = BLUETOOTH_PROTOCOLS[type_idx] + return BLUETOOTH_PROTOCOLS[type_idx] else: - state = "UNKNOWN" - - return state + return "UNKNOWN" def get_state(self): state_idx = self.sk.__sk_common.skc_state if 0 <= state_idx < len(BLUETOOTH_STATES): - state = BLUETOOTH_STATES[state_idx] + return BLUETOOTH_STATES[state_idx] else: - state = "UNKNOWN" - - return state + return "UNKNOWN" From 9b0b2547d5a69b5ba78416fa3c858a41489e4dde Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 30 Apr 2022 14:37:51 +1000 Subject: [PATCH 042/526] Manage invalid address exception when reading src and dst addresses --- .../framework/symbols/linux/extensions/__init__.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 644a88632..53ad48942 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -754,7 +754,12 @@ class inet_sock(objects.StructType): return parent_layer = self._context.layers[self.vol.layer_name] - addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) + try: + addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) + except exceptions.InvalidAddressException: + vollog.debug(f"Unable to read socket src address from {saddr.vol.offset:#x}") + return "?" + return socket_module.inet_ntop(family, addr_bytes) def get_dst_addr(self): @@ -778,7 +783,12 @@ class inet_sock(objects.StructType): return parent_layer = self._context.layers[self.vol.layer_name] - addr_bytes = parent_layer.read(daddr.vol.offset, addr_size) + try: + addr_bytes = parent_layer.read(daddr.vol.offset, addr_size) + except exceptions.InvalidAddressException: + vollog.debug(f"Unable to read socket dst address from {daddr.vol.offset:#x}") + return "?" + return socket_module.inet_ntop(family, addr_bytes) class netlink_sock(objects.StructType): From db9f287cb146052d28ad75f1b9787b0131b46e8a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 30 Apr 2022 15:25:55 +1000 Subject: [PATCH 043/526] Unrelated to this PR. Removed unused import. --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index e0f927ec9..e91b0cd4e 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -9,7 +9,7 @@ import struct from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload from volatility3.framework import constants, interfaces -from volatility3.framework.objects import templates, utility +from volatility3.framework.objects import templates vollog = logging.getLogger(__name__) From 78eb31014b0a97145582d8ae4beef598c2659b1f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 2 May 2022 00:00:33 +0900 Subject: [PATCH 044/526] Fix: get owning process method from _ETHREAD --- volatility3/framework/symbols/windows/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 7d083fbba..fd5e075da 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -448,9 +448,9 @@ class KMUTANT(objects.StructType, pool.ExecutiveObject): class ETHREAD(objects.StructType): """A class for executive thread objects.""" - def owning_process(self, kernel_layer: str = None) -> interfaces.objects.ObjectInterface: + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" - return self.ThreadsProcess.dereference(kernel_layer) + return self.Tcb.Process.dereference().cast("_EPROCESS") def get_cross_thread_flags(self) -> str: dictCrossThreadFlags = { From 51b901960169b5681f4a1cbebf03c19997685232 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 2 May 2022 10:34:04 +0900 Subject: [PATCH 045/526] Bump: patch version 2.1.1 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 5060906d5..e08bc42bc 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 1 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 0edee3fe257dd39fec3a9d41f132fbb4da7f102d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 8 May 2022 00:35:53 +0900 Subject: [PATCH 046/526] Add: vadwalk plugin initialize version --- .../framework/plugins/windows/vadwalk.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 volatility3/framework/plugins/windows/vadwalk.py diff --git a/volatility3/framework/plugins/windows/vadwalk.py b/volatility3/framework/plugins/windows/vadwalk.py new file mode 100644 index 000000000..37367aa5d --- /dev/null +++ b/volatility3/framework/plugins/windows/vadwalk.py @@ -0,0 +1,68 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import Iterator, List, Tuple + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist, vadinfo + +vollog = logging.getLogger(__name__) + + +class VadWalk(interfaces.plugins.PluginInterface): + """Walk the VAD tree""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.PluginRequirement(name = 'vadinfo', plugin = vadinfo.VadInfo, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True) + ] + + def _generator(self, procs) -> Iterator[Tuple]: + for proc in procs: + for vad in vadinfo.VadInfo.list_vads(proc): + if(vad): + yield(0, (proc.UniqueProcessId, + utility.array_to_string(proc.ImageFileName), + format_hints.Hex(vad.vol.offset), + format_hints.Hex(vad.get_parent()), + format_hints.Hex(vad.get_right_child()), + format_hints.Hex(vad.get_left_child()), + format_hints.Hex(vad.get_start()), + format_hints.Hex(vad.get_end()), + vad.get_tag())) + + def run(self) -> renderers.TreeGrid: + kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + + return renderers.TreeGrid([('PID', int), + ('Process', str), + ('Offset', format_hints.Hex), + ('Parent', format_hints.Hex), + ('Right', format_hints.Hex), + ('Left', format_hints.Hex), + ('Start', format_hints.Hex), + ('End', format_hints.Hex), + ('Tag', str)], + self._generator( + pslist.PsList.list_processes( + context = self.context, + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, + filter_func = filter_func))) From 000fdd74c818b6c22344bfe7c58d2c96474521e4 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 8 May 2022 00:44:49 +0900 Subject: [PATCH 047/526] Fix: change left and right value --- volatility3/framework/plugins/windows/vadwalk.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadwalk.py b/volatility3/framework/plugins/windows/vadwalk.py index 37367aa5d..37f2ced88 100644 --- a/volatility3/framework/plugins/windows/vadwalk.py +++ b/volatility3/framework/plugins/windows/vadwalk.py @@ -15,7 +15,7 @@ vollog = logging.getLogger(__name__) class VadWalk(interfaces.plugins.PluginInterface): - """Walk the VAD tree""" + """Walk the VAD tree.""" _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @@ -41,8 +41,8 @@ class VadWalk(interfaces.plugins.PluginInterface): utility.array_to_string(proc.ImageFileName), format_hints.Hex(vad.vol.offset), format_hints.Hex(vad.get_parent()), - format_hints.Hex(vad.get_right_child()), format_hints.Hex(vad.get_left_child()), + format_hints.Hex(vad.get_right_child()), format_hints.Hex(vad.get_start()), format_hints.Hex(vad.get_end()), vad.get_tag())) @@ -55,8 +55,8 @@ class VadWalk(interfaces.plugins.PluginInterface): ('Process', str), ('Offset', format_hints.Hex), ('Parent', format_hints.Hex), - ('Right', format_hints.Hex), ('Left', format_hints.Hex), + ('Right', format_hints.Hex), ('Start', format_hints.Hex), ('End', format_hints.Hex), ('Tag', str)], From bbbecd6692f424104ee709152eb483757891d0e8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 9 May 2022 11:37:05 +0900 Subject: [PATCH 048/526] Add: argument type for _generator --- volatility3/framework/plugins/windows/vadwalk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadwalk.py b/volatility3/framework/plugins/windows/vadwalk.py index 37f2ced88..942cef357 100644 --- a/volatility3/framework/plugins/windows/vadwalk.py +++ b/volatility3/framework/plugins/windows/vadwalk.py @@ -3,7 +3,7 @@ # import logging -from typing import Iterator, List, Tuple +from typing import Generator, Iterator, List, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -33,7 +33,7 @@ class VadWalk(interfaces.plugins.PluginInterface): optional = True) ] - def _generator(self, procs) -> Iterator[Tuple]: + def _generator(self, procs: Generator[interfaces.objects.ObjectInterface, None, None]) -> Iterator[Tuple]: for proc in procs: for vad in vadinfo.VadInfo.list_vads(proc): if(vad): From f4dd582f158e8024e3fc5b4fba21a727f0913bfb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 12:29:36 +0900 Subject: [PATCH 049/526] Fix: ThreadsProcess for windows older version --- .../framework/symbols/windows/extensions/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b317f7693..ccfcb4290 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -450,7 +450,12 @@ class ETHREAD(objects.StructType): def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" - return self.Tcb.Process.dereference().cast("_EPROCESS") + if(self.has_member("ThreadsProcess")): + return self.ThreadsProcess.dereference().cast("_EPROCESS") + elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): + return self.Tcb.Process.dereference().cast("_EPROCESS") + else: + raise AttributeError("Unable to find the owning process of ethread") def get_cross_thread_flags(self) -> str: dictCrossThreadFlags = { From 1125be122e8d330cad156ba67a279bf83f22c2f0 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 12:31:56 +0900 Subject: [PATCH 050/526] Add: code comment for windows version --- volatility3/framework/symbols/windows/extensions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ccfcb4290..a1c347ed2 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -450,6 +450,7 @@ class ETHREAD(objects.StructType): def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" + if(self.has_member("ThreadsProcess")): return self.ThreadsProcess.dereference().cast("_EPROCESS") elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): From 3956f0ecc0406f32482123c0c1866755b8fd7cf7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 12:36:56 +0900 Subject: [PATCH 051/526] Add: code comment for windows version --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index a1c347ed2..805f8c26b 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -451,8 +451,10 @@ class ETHREAD(objects.StructType): def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" + # For Windows XPs if(self.has_member("ThreadsProcess")): return self.ThreadsProcess.dereference().cast("_EPROCESS") + # For Windows Vista and later versions elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): return self.Tcb.Process.dereference().cast("_EPROCESS") else: From 690d8e3efe8ec08d4500b75ba60f14a281a52673 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 19:07:04 +0900 Subject: [PATCH 052/526] Fix: sync bump version --- volatility3/framework/constants/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index e08bc42bc..472a743e6 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,8 +39,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 1 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 2 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From a5bf5548b8d7e83ffd3b9065e968e321f6fcc964 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 19:08:29 +0900 Subject: [PATCH 053/526] Bump: patch version 2.2.1 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 472a743e6..44b98b95f 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 2 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From e5dfc47cc419d4d1ac929782008bc24765f46428 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 19:08:56 +0900 Subject: [PATCH 054/526] Fix: required framework version of psscan by bump --- volatility3/framework/plugins/windows/psscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index a0601aef1..cc030b4bf 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -22,7 +22,7 @@ vollog = logging.getLogger(__name__) class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for processes present in a particular windows memory image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 2, 1) _version = (1, 1, 0) @classmethod From baaedf21e51b33c9c3eee772296ebfd3b9dd9869 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 19:37:30 +0900 Subject: [PATCH 055/526] Add: plugin version, logger, dump options --- .../plugins/windows/registry/certificates.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 91f17fb2d..81b7d766f 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,16 +1,19 @@ +import logging import struct from typing import List, Iterator, Tuple -from volatility3.framework import interfaces, renderers +from volatility3.framework import constants, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey +vollog = logging.getLogger(__name__) class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -20,7 +23,11 @@ class Certificates(interfaces.plugins.PluginInterface): architectures = ["Intel32", "Intel64"]), requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)) + requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)), + requirements.BooleanRequirement(name = 'dump', + description = "Extract listed certificates", + default = False, + optional = True) ] def parse_data(self, data: bytes) -> Tuple[str, bytes]: @@ -48,21 +55,22 @@ class Certificates(interfaces.plugins.PluginInterface): try: # Walk it node_path = hive.get_key(top_key, return_list = True) - for (depth, is_key, last_write_time, key_path, volatility, - node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): + for (_, is_key, _, key_path, _, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = key_path.casefold().index(top_key.casefold()) + len(top_key) + 1 reg_section = key_path[unique_key_offset:key_path.index("\\", unique_key_offset)] key_hash = key_path[key_path.rindex("\\") + 1:] - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - with self.open("{} - {} - {}.crt".format(hex(hive.hive_offset), reg_section, - key_hash)) as file_data: - file_data.write(certificate_data) + if self.config['dump']: + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + with self.open("{} - {} - {}.crt".format(hex(hive.hive_offset), reg_section, + key_hash)) as file_data: + file_data.write(certificate_data) yield (0, (top_key, reg_section, key_hash, name)) except KeyError: # Key wasn't found in this hive, carry on + vollog.log(constants.LOGLEVEL_VVVV, "Key wasn't found in this hive") pass def run(self) -> renderers.TreeGrid: From dafd62d6cee5f8366e3071adfc80e1c910d301fa Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 19:52:57 +0900 Subject: [PATCH 056/526] Fix: dump options for depreated step --- .../plugins/windows/registry/certificates.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 81b7d766f..f261b5e36 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -41,6 +41,12 @@ class Certificates(interfaces.plugins.PluginInterface): elif ctype == 0x100000020: certificate_data = cvalue return (name, certificate_data) + + def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str) -> str: + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) + with self.open(dump_name) as file_data: + file_data.write(certificate_data) def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: for hive in hivelist.HiveList.list_hives(self.context, @@ -63,10 +69,11 @@ class Certificates(interfaces.plugins.PluginInterface): key_hash = key_path[key_path.rindex("\\") + 1:] if self.config['dump']: - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - with self.open("{} - {} - {}.crt".format(hex(hive.hive_offset), reg_section, - key_hash)) as file_data: - file_data.write(certificate_data) + self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) + else: + vollog.warning("Certificates plugin is no longer support automatically dumped, please use the dump option.") + self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) + yield (0, (top_key, reg_section, key_hash, name)) except KeyError: # Key wasn't found in this hive, carry on From f63f869506186d0396625d6232f9657ca1dad717 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 20:23:42 +0900 Subject: [PATCH 057/526] Remove: return type of dump method --- volatility3/plugins/windows/registry/certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index f261b5e36..ee9cba6d3 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -42,7 +42,7 @@ class Certificates(interfaces.plugins.PluginInterface): certificate_data = cvalue return (name, certificate_data) - def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str) -> str: + def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str): if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) with self.open(dump_name) as file_data: From 0e8958b8416350ac9c22961674532e62b72050ec Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 16 May 2022 15:11:07 +0900 Subject: [PATCH 058/526] Fix: classmethod, variable name, exceptions, etc --- .../plugins/windows/registry/certificates.py | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index ee9cba6d3..a27b3545a 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,8 +1,8 @@ import logging import struct -from typing import List, Iterator, Tuple +from typing import List, Iterator, Tuple, Type -from volatility3.framework import constants, interfaces, renderers +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey @@ -13,7 +13,6 @@ class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -42,11 +41,20 @@ class Certificates(interfaces.plugins.PluginInterface): certificate_data = cvalue return (name, certificate_data) - def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str): - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) - with self.open(dump_name) as file_data: - file_data.write(certificate_data) + @classmethod + def dump_certificate(cls, certificate_data: bytes, hive_offset: int, + reg_section: str, key_hash: str, + open_method: Type[interfaces.plugins.FileHandlerInterface]) -> \ + interfaces.plugins.FileHandlerInterface: + try: + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) + with open_method(dump_name) as file_data: + file_data.write(certificate_data) + except exceptions.InvalidAddressException: + vollog.debug(f"Unable to certificate file at {hive_offset:#x}") + return None + def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: for hive in hivelist.HiveList.list_hives(self.context, @@ -61,7 +69,7 @@ class Certificates(interfaces.plugins.PluginInterface): try: # Walk it node_path = hive.get_key(top_key, return_list = True) - for (_, is_key, _, key_path, _, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): + for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = key_path.casefold().index(top_key.casefold()) + len(top_key) + 1 @@ -69,10 +77,9 @@ class Certificates(interfaces.plugins.PluginInterface): key_hash = key_path[key_path.rindex("\\") + 1:] if self.config['dump']: - self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) - else: - vollog.warning("Certificates plugin is no longer support automatically dumped, please use the dump option.") - self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) + file_handle = self.dump_certificate(certificate_data, hive.hive_offset, reg_section, key_hash, self.open) + if file_handle: + file_handle.close() yield (0, (top_key, reg_section, key_hash, name)) except KeyError: From 5770d35a4afd718760ddeeb1c21606e6b5bd1e2e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 17 May 2022 00:18:06 +0900 Subject: [PATCH 059/526] Add: return file handle --- volatility3/plugins/windows/registry/certificates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index a27b3545a..b079844e7 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -49,8 +49,9 @@ class Certificates(interfaces.plugins.PluginInterface): try: if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) - with open_method(dump_name) as file_data: - file_data.write(certificate_data) + file_handle = open_method(dump_name) + file_handle.write(certificate_data) + return file_handle except exceptions.InvalidAddressException: vollog.debug(f"Unable to certificate file at {hive_offset:#x}") return None From 3846268bf7a8b3731a80a61959ca1aee227a112d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 17 May 2022 00:19:39 +0900 Subject: [PATCH 060/526] Add: optional return type --- volatility3/plugins/windows/registry/certificates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index b079844e7..d2fb61f02 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,6 +1,6 @@ import logging import struct -from typing import List, Iterator, Tuple, Type +from typing import List, Iterator, Optional, Tuple, Type from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -45,7 +45,7 @@ class Certificates(interfaces.plugins.PluginInterface): def dump_certificate(cls, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str, open_method: Type[interfaces.plugins.FileHandlerInterface]) -> \ - interfaces.plugins.FileHandlerInterface: + Optional[interfaces.plugins.FileHandlerInterface]: try: if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) From bd332261dede65118114e6484c4d8ce446d3b165 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 05:39:37 +0900 Subject: [PATCH 061/526] Fix: __del__ to __exit__ --- volatility3/framework/layers/physical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 5cf0b776d..0633637ca 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -191,7 +191,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): """Closes the file handle.""" self._file.close() - def __del__(self) -> None: + def __exit__(self) -> None: self.destroy() @classmethod From a4e162c7f1dd8597cd0b9a0a8e175573c7fb8c62 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 07:24:45 +0900 Subject: [PATCH 062/526] Fix: minor for better code --- volatility3/framework/plugins/mac/kauth_listeners.py | 2 +- volatility3/framework/plugins/windows/skeleton_key_check.py | 2 +- volatility3/framework/symbols/metadata.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/mac/kauth_listeners.py b/volatility3/framework/plugins/mac/kauth_listeners.py index 7002d88e2..fba6a8e0a 100644 --- a/volatility3/framework/plugins/mac/kauth_listeners.py +++ b/volatility3/framework/plugins/mac/kauth_listeners.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index cd4a5baec..4a1b48c9a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -9,7 +9,7 @@ # For a thorough walkthrough on how the R&D was performed to develop this plugin, # please see our blogpost here: # -# +# https://volatility-labs.blogspot.com/2021/10/memory-forensics-r-illustrated.html import io import logging diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 7cde686ee..350bb0a53 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -38,4 +38,4 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): class LinuxMetadata(interfaces.symbols.MetadataInterface): - """Class to handle the etadata from a Linux symbol table.""" + """Class to handle the metadata from a Linux symbol table.""" From 1ef8c5167722aaed65e163be3ab1d1f06c6117bb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 08:29:13 +0900 Subject: [PATCH 063/526] Fix: minor code for improve --- volatility3/framework/plugins/linux/check_syscall.py | 2 +- volatility3/framework/plugins/linux/mountinfo.py | 1 - volatility3/framework/plugins/windows/ssdt.py | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 87d252cd5..50fd05fa5 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -80,7 +80,7 @@ class Check_syscall(plugins.PluginInterface): def _get_table_info_disassembly(self, ptr_sz, vmlinux): """Find the size of the system call table by disassembling functions - that immediately reference it in their first isntruction This is in the + that immediately reference it in their first instruction This is in the form 'cmp reg,NR_syscalls'.""" table_size = 0 diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 6f3cb712d..551d128ad 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -1,7 +1,6 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -# Author: Gustavo Moreira import logging from collections import namedtuple diff --git a/volatility3/framework/plugins/windows/ssdt.py b/volatility3/framework/plugins/windows/ssdt.py index 0d921535d..78fd72630 100644 --- a/volatility3/framework/plugins/windows/ssdt.py +++ b/volatility3/framework/plugins/windows/ssdt.py @@ -95,10 +95,10 @@ class SSDT(plugins.PluginInterface): if is_kernel_64: array_subtype = "long" - def kvo_calulator(func: int) -> int: + def kvo_calculator(func: int) -> int: return kvo + service_table_address + (func >> 4) - find_address = kvo_calulator + find_address = kvo_calculator else: array_subtype = "unsigned long" From b312a5780ccc19f7c672d3b08d10e64da59cf0ab Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 08:56:27 +0900 Subject: [PATCH 064/526] Fix: address mask from @ikelos help --- volatility3/framework/plugins/windows/vadwalk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadwalk.py b/volatility3/framework/plugins/windows/vadwalk.py index 942cef357..1090ed554 100644 --- a/volatility3/framework/plugins/windows/vadwalk.py +++ b/volatility3/framework/plugins/windows/vadwalk.py @@ -40,7 +40,7 @@ class VadWalk(interfaces.plugins.PluginInterface): yield(0, (proc.UniqueProcessId, utility.array_to_string(proc.ImageFileName), format_hints.Hex(vad.vol.offset), - format_hints.Hex(vad.get_parent()), + format_hints.Hex(vad.get_parent() & self.context.layers[vad.vol.layer_name].address_mask), format_hints.Hex(vad.get_left_child()), format_hints.Hex(vad.get_right_child()), format_hints.Hex(vad.get_start()), From 8b128b05f834c210ce607ab40d386718b5b363b5 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 22:17:33 +0900 Subject: [PATCH 065/526] Fix: typo for code comments --- volatility3/framework/objects/templates.py | 2 +- volatility3/framework/plugins/linux/psaux.py | 2 +- volatility3/framework/plugins/linux/pstree.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/objects/templates.py b/volatility3/framework/objects/templates.py index b544d117f..56754d255 100644 --- a/volatility3/framework/objects/templates.py +++ b/volatility3/framework/objects/templates.py @@ -63,7 +63,7 @@ class ObjectTemplate(interfaces.objects.Template): object_info: interfaces.objects.ObjectInformation) -> interfaces.objects.ObjectInterface: """Constructs the object. - Returns: an object adhereing to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface` + Returns: an object adhering to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface` """ arguments: Dict[str, Any] = {} for arg in self.vol: diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index c62712907..ed91c66f2 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -40,7 +40,7 @@ class PsAux(plugins.PluginInterface): name: string name of the process (from task.comm) """ - # kernel theads never have an mm as they do not have userland mappings + # kernel threads never have an mm as they do not have userland mappings try: mm = task.mm except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index a44310147..3ad5f3e19 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -19,7 +19,7 @@ class PsTree(pslist.PsList): """Finds how deep the PID is in the tasks hierarchy. Args: - pid: PID to find the level in the hierachy + pid: PID to find the level in the hierarchy """ seen = set([pid]) level = 0 From bb80d7067e99d55748e1067e6d11e22c2ffe5e4d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 31 May 2022 23:24:49 +0900 Subject: [PATCH 066/526] Fix: typo of timeliner parameter --- volatility3/framework/plugins/timeliner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 8785f62e1..c1d29062d 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -101,7 +101,7 @@ class Timeliner(interfaces.plugins.PluginInterface): return [sortable(timestamp) for timestamp in data[2:]] - def _generator(self, runable_plugins: List[TimeLinerInterface]) -> Optional[Iterable[Tuple[int, Tuple]]]: + def _generator(self, runnable_plugins: List[TimeLinerInterface]) -> Optional[Iterable[Tuple[int, Tuple]]]: """Takes a timeline, sorts it and output the data from each relevant row from each plugin.""" # Generate the results for each plugin @@ -115,9 +115,9 @@ class Timeliner(interfaces.plugins.PluginInterface): file_data = None fp = None - for plugin in runable_plugins: + for plugin in runnable_plugins: plugin_name = plugin.__class__.__name__ - self._progress_callback((runable_plugins.index(plugin) * 100) // len(runable_plugins), + self._progress_callback((runnable_plugins.index(plugin) * 100) // len(runnable_plugins), f"Running plugin {plugin_name}...") try: vollog.log(logging.INFO, f"Running {plugin_name}") From 0abd2e53abefd0856c83bfad2cf61ab500868d38 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 5 Jun 2022 10:56:42 +0100 Subject: [PATCH 067/526] Pyinstaller: Fix path need to current directory to be correct --- vol.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vol.spec b/vol.spec index 42b69af3f..666526dde 100644 --- a/vol.spec +++ b/vol.spec @@ -26,7 +26,7 @@ except ImportError: # Volatility must be findable in sys.path in order for collect_submodules to work # This adds the current working directory, which should usually do the trick -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.abspath(SPEC))) vol_analysis = Analysis(['vol.py'], pathex = [], From db3408bdaa978de5b23eecd2d4411df8a6de1f16 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 5 Jun 2022 22:23:30 +0100 Subject: [PATCH 068/526] Windows: Extend the pdb support to modules --- .../framework/symbols/windows/pdbutil.py | 75 +++++++++++++++---- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 585e96b6d..41037d464 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -10,10 +10,10 @@ import os import re import struct from typing import Any, Dict, Generator, List, Optional, Tuple, Union -from urllib import request, parse +from urllib import parse, request from volatility3 import symbols -from volatility3.framework import constants, interfaces, exceptions +from volatility3.framework import constants, contexts, exceptions, interfaces from volatility3.framework.configuration.requirements import SymbolTableRequirement from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbconv @@ -24,7 +24,7 @@ vollog = logging.getLogger(__name__) class PDBUtility(interfaces.configuration.VersionableInterface): """Class to handle and manage all getting symbols based on MZ header""" - _version = (1, 0, 0) + _version = (1, 0, 1) _required_framework_version = (2, 0, 0) @classmethod @@ -131,14 +131,14 @@ class PDBUtility(interfaces.configuration.VersionableInterface): # Check it is actually the MZ header if mz_sig != b"MZ": return None - + nt_header_start, = struct.unpack(" str: + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: """Creates symbol table for a module in the specified layer_name. Searches the memory section of the loaded module for its PDB GUID @@ -307,6 +307,19 @@ class PDBUtility(interfaces.configuration.VersionableInterface): Returns: The name of the constructed and loaded symbol table """ + _, symbol_table_name = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size) + return symbol_table_name + + @classmethod + def _modtable_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None, + create_module: bool = False) -> Tuple[Optional[str], Optional[str]]: + + if module_offset is None: + module_offset = context.layers[layer_name].minimum_address + if module_size is None: + module_size = context.layers[layer_name].maximum_address - module_offset guids = list( cls.pdbname_scan(context, @@ -323,12 +336,46 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - return cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) + module_name = guid["pdb_name"].strip('.pdb') + + symbol_table_name = cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) + + new_module_name = None + if create_module: + new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], + symbol_table_name = symbol_table_name) + new_module_name = new_module.name + + return new_module_name, symbol_table_name + + @classmethod + def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: + """Creates a module in the specified layer_name based on a pdb name. + + Searches the memory section of the loaded module for its PDB GUID + and loads the associated symbol table into the symbol space. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + config_path: The config path where to find symbol files + layer_name: The name of the layer on which to operate + module_offset: This memory dump's module image offset + module_size: The size of the module for this dump + + Returns: + The name of the constructed and loaded symbol table + """ + + module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size, create_module = True) + + return module_name class PdbSignatureScanner(interfaces.layers.ScannerInterface): From 21d916be0a08eccc91bbd4884f458ae6ff489b95 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 6 Jun 2022 14:53:52 +0100 Subject: [PATCH 069/526] Pyinstaller: Support pyinstaller 5 and later --- volatility3/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/__init__.py b/volatility3/__init__.py index db52aa9b0..b6da6e01e 100644 --- a/volatility3/__init__.py +++ b/volatility3/__init__.py @@ -37,9 +37,9 @@ class WarningFindSpec(abc.MetaPathFinder): first.""" if fullname.startswith("volatility3.framework.plugins."): warning = "Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins" - # Pyinstaller uses walk_packages to import, but needs to read the modules to figure out dependencies - # As such, we only print the warning when directly imported rather than from within walk_packages - if inspect.stack()[-2].function != 'walk_packages': + # Pyinstaller uses walk_packages/_collect_submodules to import, but needs to read the modules to figure out dependencies + # As such, we only print the warning when directly imported rather than from within walk_packages/_collect_submodules + if inspect.stack()[-2].function in ['walk_packages', '_collect_submodules']: raise Warning(warning) From aa06ed6e674761c8ec1238daeff9a64603aff392 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 12 Jun 2022 20:32:06 +0900 Subject: [PATCH 070/526] Add: new options for vol-cli.rst --- doc/source/vol-cli.rst | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 9db29c818..902787c9c 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -9,7 +9,11 @@ Synopsis **volatility** [-h] [-c CONFIG] [--parallelism [{processes,threads,off}]] [-e EXTEND] [-p PLUGIN_DIRS] [-s SYMBOL_DIRS] [-v] [-l LOG] [-o OUTPUT_DIR] [-q] [-r RENDERER] [-f FILE] - [--write-config] [--single-location SINGLE_LOCATION] + [--write-config] [--save-config SAVE_CONFIG] + [--clear-cache] [--cache-path CACHE_PATH] + [--offline] + [--single-location SINGLE_LOCATION] + [--stackers [STACKERS ...]] [--single-swap-locations SINGLE_SWAP_LOCATIONS] ... @@ -105,11 +109,31 @@ Options other plugins, but there's no guarantee that plugins use the same configuration options. +--save-config + This flag specifies that volatility should write or overwrite a file + called config.json in the current directory. The file will contain + the necessary JSON configuration to recreate the environment that the + plugin was previously run in. This configuration *may* be accepted by + other plugins, but there's no guarantee that plugins use the same + configuration options. + +--clear-cache + Clears out all short-term cached items. + +--cache-path + Change the default path ({constants.CACHE_PATH}) used to store the cache. + +--offline + Do not search online for additional JSON files. + --single-location SINGLE_LOCATION This specifies a URL which will be downloaded if necessary, and built upon by the automagic and, since most plugins require a single memory image, can be considered the input for the program. +--stackers STACKERS + + --single-swap-locations SINGLE_SWAP_LOCATIONS A comma-separated list of swap files to be considered as part of the memory image specified by the single-location or file parameters. From 2d14e4e012d6745862fafa1a857414102a412eb1 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 12 Jun 2022 20:50:03 +0900 Subject: [PATCH 071/526] Add: descriptions of new options --- doc/source/vol-cli.rst | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 902787c9c..b9e16623d 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -102,12 +102,8 @@ Options attempt to build upon, and can be considered the input for the program. --write-config - This flag specifies that volatility should write or overwrite a file - called config.json in the current directory. The file will contain - the necessary JSON configuration to recreate the environment that the - plugin was previously run in. This configuration *may* be accepted by - other plugins, but there's no guarantee that plugins use the same - configuration options. + *Deprecated* + Use of `--write-config` has been deprecated, replaced by `--save-config` --save-config This flag specifies that volatility should write or overwrite a file @@ -121,19 +117,18 @@ Options Clears out all short-term cached items. --cache-path - Change the default path ({constants.CACHE_PATH}) used to store the cache. + Change the default path used to store the cache. --offline Do not search online for additional JSON files. + Run offline mode (defaults to false) and for + remote windows symbol tables, linux/mac banner repositories. --single-location SINGLE_LOCATION This specifies a URL which will be downloaded if necessary, and built upon by the automagic and, since most plugins require a single memory image, can be considered the input for the program. ---stackers STACKERS - - --single-swap-locations SINGLE_SWAP_LOCATIONS A comma-separated list of swap files to be considered as part of the memory image specified by the single-location or file parameters. From 3ef505641eb2f7d3d76174effb18cba69434298f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 12 Jun 2022 20:55:15 +0900 Subject: [PATCH 072/526] Add: stacker descriptions --- doc/source/vol-cli.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index b9e16623d..cc6f7fe6a 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -129,6 +129,9 @@ Options upon by the automagic and, since most plugins require a single memory image, can be considered the input for the program. +--stackers STACKERS + Creates the list of stackers to use based on the config option. + --single-swap-locations SINGLE_SWAP_LOCATIONS A comma-separated list of swap files to be considered as part of the memory image specified by the single-location or file parameters. From e1f3f65202d7eb23901a4c9639ad1523f4429369 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 14 Jun 2022 21:03:33 +0900 Subject: [PATCH 073/526] Fix: typo for code comment, requirements name --- volatility3/framework/interfaces/configuration.py | 2 +- volatility3/framework/interfaces/layers.py | 4 ++-- volatility3/framework/plugins/mac/kevents.py | 2 +- volatility3/framework/plugins/windows/modscan.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index c39dba680..e271ef6d4 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -523,7 +523,7 @@ class ConstructableRequirementInterface(RequirementInterface): must happen after the class configuration value has been provided). These values are then provided to the object's constructor by name as arguments (as well as the standard `context` and `config_path` - arguments. + arguments). """ def __init__(self, *args, **kwargs) -> None: diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index a42282c39..7ff110c6e 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -307,7 +307,7 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla while length > 0: chunk_size = min(length, scanner.chunk_size + scanner.overlap) yield [(layer_name, mapped_offset, chunk_size)], offset + chunk_size - # It we've got more than the scanner's chunk_size, only move up by the chunk_size + # If we've got more than the scanner's chunk_size, only move up by the chunk_size if chunk_size > scanner.chunk_size: chunk_size -= scanner.overlap length -= chunk_size @@ -517,7 +517,7 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): yield output, chunk_position output = [] chunk_position = chunk_start - # Take from chunk_position as far as far as the block can go, + # Take from chunk_position as far as the block can go, # or as much left of a scanner chunk as we can chunk_size = min(block_end - chunk_position, scanner.chunk_size + scanner.overlap - (chunk_position - chunk_start)) diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 6f82c75cd..4a82d81cd 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -74,7 +74,7 @@ class Kevents(interfaces.plugins.PluginInterface): @classmethod def _walk_klist_array(cls, kernel, fdp, array_pointer_member, array_size_member): """ - Convience wrapper for walking an array of lists of kernel events + Convenience wrapper for walking an array of lists of kernel events Handles invalid address references """ try: diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index b661d71d7..e352c21fe 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -25,7 +25,7 @@ class ModScan(interfaces.plugins.PluginInterface): return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'poolerscanner', + requirements.VersionRequirement(name = 'poolscanner', component = poolscanner.PoolScanner, version = (1, 0, 0)), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), From dd92955a99249fe9e8863cb2754229e01a917d73 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 16 Jun 2022 05:40:26 +0900 Subject: [PATCH 074/526] Remove: unreachable code --- volatility3/cli/volshell/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 769e958fd..5eeef77cf 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -257,7 +257,6 @@ class VolShell(cli.CommandLine): constructed.run() except exceptions.VolatilityException as excp: self.process_exceptions(excp) - parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") def main(): From 9f525dfa733dd65769458540d3996917a1daaa96 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 22 Jun 2022 19:57:47 +0900 Subject: [PATCH 075/526] Refactor: simplify comparision --- volatility3/framework/automagic/pdbscan.py | 2 +- volatility3/framework/plugins/windows/ldrmodules.py | 12 ++++++------ volatility3/framework/plugins/windows/vadinfo.py | 2 +- .../framework/symbols/linux/extensions/__init__.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 5db66a3d0..cedbc4919 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -148,7 +148,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vollog.debug("Kernel base determination - optimized scan virtual layer") valid_kernel = self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, True, False, progress_callback) - if valid_kernel != None: + if valid_kernel is not None: return valid_kernel vollog.debug("Kernel base determination - slow scan virtual layer") diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index e7c96e946..284d1afc2 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -71,14 +71,14 @@ class LdrModules(interfaces.plugins.PluginInterface): mem_mod = mem_order_mod.get(base, None) yield (0, [int(proc.UniqueProcessId), - str(proc.ImageFileName.cast("string", + str(proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, errors = 'replace')), - format_hints.Hex(base), - load_mod != None, - init_mod != None, - mem_mod != None, - mapped_files[base]]) + format_hints.Hex(base), + load_mod is not None, + init_mod is not None, + mem_mod is not None, + mapped_files[base]]) def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 9fa1458d1..e357b150a 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -132,7 +132,7 @@ class VadInfo(interfaces.plugins.PluginInterface): vollog.debug("Unable to find the starting/ending VPN member") return None - if maxsize > 0 and (vad_end - vad_start) > maxsize: + if 0 < maxsize < (vad_end - vad_start): vollog.debug(f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit") return None diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6792ab19c..73f31115a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -409,7 +409,7 @@ class vm_area_struct(objects.StructType): fname = linux.LinuxUtilities.path_for_file(context, task, self.vm_file) elif self.vm_start <= task.mm.start_brk and self.vm_end >= task.mm.brk: fname = "[heap]" - elif self.vm_start <= task.mm.start_stack and self.vm_end >= task.mm.start_stack: + elif self.vm_start <= task.mm.start_stack <= self.vm_end: fname = "[stack]" elif self.vm_mm.context.has_member("vdso") and self.vm_start == self.vm_mm.context.vdso: fname = "[vdso]" From 7ba27a75ca9cbecbe796f6475340718e6bce0dd0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 22 Jun 2022 14:49:57 +0100 Subject: [PATCH 076/526] Documentation: Improve the simple-plugin example --- doc/source/simple-plugin.rst | 54 ++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 8446b0ef5..904c586c7 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -30,6 +30,9 @@ to be able to run properly. Any that are defined as optional need not necessari :: + _version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + @classmethod def get_requirements(cls): return [requirements.TranslationLayerRequirement(name = 'primary', @@ -37,13 +40,13 @@ to be able to run properly. Any that are defined as optional need not necessari architectures = ["Intel32", "Intel64"]), requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (1, 0, 0)), requirements.ListRequirement(name = 'pid', element_type = int, description = "Process IDs to include (all other processes are excluded)", - optional = True)] + optional = True), + requirements.PluginRequirement(name = 'pslist', + plugin = pslist.PsList, + version = (1, 0, 0))] This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how @@ -91,29 +94,44 @@ name of the :py:class:`SymbolTable Date: Wed, 22 Jun 2022 15:12:40 +0100 Subject: [PATCH 077/526] Documentation: Update the documentation to the latest framework --- doc/source/simple-plugin.rst | 103 ++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 904c586c7..543451b88 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -35,11 +35,8 @@ to be able to run properly. Any that are defined as optional need not necessari @classmethod def get_requirements(cls): - return [requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.ListRequirement(name = 'pid', element_type = int, description = "Process IDs to include (all other processes are excluded)", @@ -54,45 +51,73 @@ to instantiate the plugin). At the moment these requirements are fairly straigh :: - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), -This requirement indicates that the plugin will operate on a single -:py:class:`TranslationLayer `. The name of the -loaded layer will appear in the plugin's configuration under the name ``primary``. Requirement values can be -accessed within the plugin through the plugin's `config` attribute (for example ``self.config['pid']``). +This requirement specifies the need for a particular submodule. Each module requires a +:py:class:`TranslationLayer ` and a +:py:class:`SymbolTable `, which are fulfilled by two +subrequirements: a +:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` and a +:py:class:`~volatility3.framework.configuration.requirements.SymbolTableRequirement`. At the moment, the automagic +only fills `ModuleRequirements` with kernels, and so has relatively few parameters. It requires the architecture for +the underlying TranslationLayer, and the offset of the module within that layer. -.. note:: The name itself is dynamic depending on the other layers already present in the Context. Always use the value - from the configuration rather than attempting to guess what the layer will be called. +The name of the module will be stored in the ``kernel`` configuration option, and the module object itself +can be accessed from the ``context.modules`` collection. This requirement is a Complex Requirement and therefore will +not be requested directly from the user. -Finally, this defines that the translation layer must be on the Intel Architecture. At the moment, this acts as a filter, -failing to be satisfied by memory images that do not match the architecture required. -Most plugins will only operate on a single layer, but it is entirely possible for a plugin to request two different -layers, for example a plugin that carries out some form of difference or statistics against multiple memory images. +.. note:: -This requirement (and the next two) are known as Complex Requirements, and user interfaces will likely not directly -request a value for this from a user. The value stored in the configuration tree for a -:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` is -the string name of a layer present in the context's memory that satisfies the requirement. + In previous versions of volatility 3, there was no `ModuleRequirement`, and instead two requirements were defined + a :py:class:`TranslationLayer ` and a `SymbolTableRequirement`. These still exist, and can be used, most plugins just + define a single `ModuleRequirement` for the kernel, which the automagic will populate. The `ModuleRequirement` has + two automatic sub-requirements, a `TranslationLayerRequirement` and a `SymbolTableRequirement`, but the module also + includes the offset of the module, and will allow future expansion to specify specific modules when application + level plugins become more common. Below are how the requirements would be specified: -:: + :: - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), -This requirement specifies the need for a particular -:py:class:`SymbolTable ` -to be loaded. This gets populated by various -:py:class:`Automagic ` as the nearest sibling to a particular -:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`. -This means that if the :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` -is satisfied and the :py:class:`Automagic ` can determine -the appropriate :py:class:`SymbolTable `, the -name of the :py:class:`SymbolTable ` will be stored in the configuration. + This requirement indicates that the plugin will operate on a single + :py:class:`TranslationLayer `. The name of the + loaded layer will appear in the plugin's configuration under the name ``primary``. Requirement values can be + accessed within the plugin through the plugin's `config` attribute (for example ``self.config['pid']``). -This requirement is also a Complex Requirement and therefore will not be requested directly from the user. + .. note:: The name itself is dynamic depending on the other layers already present in the Context. Always use the value + from the configuration rather than attempting to guess what the layer will be called. + + Finally, this defines that the translation layer must be on the Intel Architecture. At the moment, this acts as a filter, + failing to be satisfied by memory images that do not match the architecture required. + + Most plugins will only operate on a single layer, but it is entirely possible for a plugin to request two different + layers, for example a plugin that carries out some form of difference or statistics against multiple memory images. + + This requirement (and the next two) are known as Complex Requirements, and user interfaces will likely not directly + request a value for this from a user. The value stored in the configuration tree for a + :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` is + the string name of a layer present in the context's memory that satisfies the requirement. + + :: + + requirements.SymbolTableRequirement(name = "nt_symbols", + description = "Windows kernel symbols"), + + This requirement specifies the need for a particular + :py:class:`SymbolTable ` + to be loaded. This gets populated by various + :py:class:`Automagic ` as the nearest sibling to a particular + :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`. + This means that if the :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` + is satisfied and the :py:class:`Automagic ` can determine + the appropriate :py:class:`SymbolTable `, the + name of the :py:class:`SymbolTable ` will be stored in the configuration. + + This requirement is also a Complex Requirement and therefore will not be requested directly from the user. :: @@ -147,6 +172,7 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] return renderers.TreeGrid([("PID", int), ("Process", str), @@ -155,8 +181,8 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. ("Name", str), ("Path", str)], self._generator(pslist.PsList.list_processes(self.context, - self.config['primary'], - self.config['nt_symbols'], + kernel.layer_name, + kernel.symbol_table_name, filter_func = filter_func))) In this instance, the plugin constructs a filter (using the PsList plugin's *classmethod* for creating filters). @@ -175,7 +201,8 @@ the :py:class:`~volatility3.plugins.windows.pslist.PsList` plugin. That plugin so that other plugins can call it. As such, it takes all the necessary parameters rather than accessing them from a configuration. Since it must be portable code, it takes a context, as well as the layer name, symbol table and optionally a filter. In this instance we unconditionally -pass it the values from the configuration for the ``primary`` and ``nt_symbols`` requirements. This will generate a list +pass it the values from the configuration for the layer and symbol table from the kernel module object, constructed from +the ``kernel`` configuration requirement. This will generate a list of :py:class:`~volatility3.framework.symbols.windows.extensions.EPROCESS` objects, as provided by the :py:class:`~volatility.plugins.windows.pslist.PsList` plugin, and is not covered here but is used as an example for how to share code across plugins (both as the provider and the consumer of the shared code). From fd524a6b314750bd07779f65257d010ed635b1f6 Mon Sep 17 00:00:00 2001 From: ikelos Date: Wed, 22 Jun 2022 17:08:18 +0100 Subject: [PATCH 078/526] Update doc/source/simple-plugin.rst Yep, that seems fine. Co-authored-by: Donghyun Kim --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 543451b88..d03f7c7d6 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -43,7 +43,7 @@ to be able to run properly. Any that are defined as optional need not necessari optional = True), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, - version = (1, 0, 0))] + version = (2, 0, 0))] This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how From a386de72f5a22d176ecad730e83f804e2f62c633 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 22 Jun 2022 17:12:24 +0100 Subject: [PATCH 079/526] Documentation: Fix pslist plugin requirement --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index d03f7c7d6..1c7b91205 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -134,7 +134,7 @@ being defined within the configuration tree at all. requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, - version = (1, 0, 0)) + version = (2, 0, 0))] This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements on that plugin. The version is specified in terms of Semantic Versioning meaning that, to be compatible, the major From aed87346cdd362fb59fce772cbd62b0dded51bf5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 23 Jun 2022 09:23:31 +0100 Subject: [PATCH 080/526] Core: Add support to templates to get child templates --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/interfaces/objects.py | 11 +++++++++++ volatility3/framework/objects/__init__.py | 17 +++++++++++++++++ volatility3/framework/objects/templates.py | 7 +++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 472a743e6..f08819f29 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 2 # Number of changes that only add to the interface +VERSION_MINOR = 3 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 2240c58c9..3cc23e759 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -241,6 +241,13 @@ class ObjectInterface(metaclass = abc.ABCMeta): the child member.""" raise KeyError(f"Template does not contain any children: {template.vol.type_name}") + @classmethod + @abc.abstractmethod + def child_template(cls, template: 'Template', child: str) -> interfaces.objects.Template: + """Returns the template of the child member from the parent.""" + raise KeyError(f"Template does not contain any children: {template.vol.type_name}") + + @classmethod @abc.abstractmethod def has_member(cls, template: 'Template', member_name: str) -> bool: @@ -305,6 +312,10 @@ class Template: """Returns the relative offset of the `child` member from its parent offset.""" + @abc.abstractmethod + def child_template(self, child: str) -> interfaces.objects.Template: + """Returns the `child` member template from its parent.""" + @abc.abstractmethod def replace_child(self, old_child: 'Template', new_child: 'Template') -> None: """Replaces `old_child` with `new_child` in the list of children.""" diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index e0f927ec9..feb49a089 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -602,6 +602,14 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): return 0 raise IndexError(f"Member not present in array template: {child}") + @classmethod + def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: + """Returns the template of the child member.""" + if 'subtype' in template.vol and child == 'subtype'@ + return template.vol.subtype + raise IndexError(f"Member not present in array template: {child}") + + @overload def __getitem__(self, i: int) -> interfaces.objects.Template: ... @@ -715,6 +723,15 @@ class AggregateType(interfaces.objects.ObjectInterface): raise IndexError(f"Member not present in template: {child}") return retlist[0] + @classmethod + def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: + """Returns the template of a child to its parent.""" + retlist = template.vol.members.get(child, None) + if retlist is None: + raise IndexError(f"Member not present in template: {child}") + return retlist[1] + + @classmethod def has_member(cls, template: interfaces.objects.Template, member_name: str) -> bool: """Returns whether the object would contain a member called diff --git a/volatility3/framework/objects/templates.py b/volatility3/framework/objects/templates.py index 56754d255..e8b523373 100644 --- a/volatility3/framework/objects/templates.py +++ b/volatility3/framework/objects/templates.py @@ -48,6 +48,12 @@ class ObjectTemplate(interfaces.objects.Template): plateProxy`)""" return self.vol.object_class.VolTemplateProxy.relative_child_offset(self, child) + def child_template(self, child: str) -> interfaces.objects.Template: + """Returns the template of a child of the templated object (see + :class:`~volatility3.framework.interfaces.objects.ObjectInterface.VolTem + plateProxy`)""" + return self.vol.object_class.VolTemplateProxy.child_template(self, child) + def replace_child(self, old_child: interfaces.objects.Template, new_child: interfaces.objects.Template) -> None: """Replaces `old_child` for `new_child` in the templated object's child list (see :class:`~volatility3.framework.interfaces.objects.ObjectInterf @@ -99,6 +105,7 @@ class ReferenceTemplate(interfaces.objects.Template): size: ClassVar[Any] = property(_unresolved) replace_child: ClassVar[Any] = _unresolved relative_child_offset: ClassVar[Any] = _unresolved + child_template: ClassVar[Any] = _unresolved has_member: ClassVar[Any] = _unresolved def __call__(self, context: interfaces.context.ContextInterface, object_info: interfaces.objects.ObjectInformation): From 6982650c188a7c8fccbbee3c1d7d1f47f309df28 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Jun 2022 15:38:48 +0100 Subject: [PATCH 081/526] Volshell: Fixes use of old config variables Closes #780 --- volatility3/cli/volshell/linux.py | 4 ++-- volatility3/cli/volshell/mac.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 97a488743..0f2a90c7e 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -56,13 +56,13 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['vmlinux'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: - symbol_table = self.config['vmlinux'] + symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) @property diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 305f80505..6744f3394 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -56,7 +56,7 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['darwin'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): From e8e6bacb194933de3402a182ffc3dd070256e32b Mon Sep 17 00:00:00 2001 From: ikelos Date: Thu, 30 Jun 2022 09:57:09 +0100 Subject: [PATCH 082/526] Update volatility3/framework/objects/__init__.py Fix typo Co-authored-by: Donghyun Kim --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index feb49a089..62e6de553 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -605,7 +605,7 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): @classmethod def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: """Returns the template of the child member.""" - if 'subtype' in template.vol and child == 'subtype'@ + if 'subtype' in template.vol and child == 'subtype': return template.vol.subtype raise IndexError(f"Member not present in array template: {child}") From 951a0f5d508b8db4985d54751ea93ca78e57b191 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 30 Jun 2022 11:43:38 +0100 Subject: [PATCH 083/526] Documentation: Clarify that the code is just an example Clarifies for #773 and #776 --- doc/source/simple-plugin.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 1c7b91205..e2143f1b7 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -6,6 +6,12 @@ This guide will step through how to construct a simple plugin using Volatility 3 The example plugin we'll use is :py:class:`~volatility3.plugins.windows.dlllist.DllList`, which features the main traits of a normal plugin, and reuses other plugins appropriately. +.. note:: + + This document will not include the complete code necessary for a + working plugin (such as imports, etc) since it's designed to focus on the necessary componets for writing a plugin. + For complete and functioning plugins, the ``framework/plugins`` directory should be consulted. + Inherit from PluginInterface ---------------------------- From 6d7095fa3bf01aa4f2a9fceb1887cf28ed463e58 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 2 Jul 2022 19:30:00 +0900 Subject: [PATCH 084/526] Add: exceptions code --- .../plugins/windows/registry/certificates.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index d2fb61f02..e2fe662fc 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -17,10 +17,8 @@ class Certificates(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)), requirements.BooleanRequirement(name = 'dump', @@ -58,10 +56,12 @@ class Certificates(interfaces.plugins.PluginInterface): def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: + kernel = self.context.modules[self.config['kernel']] + for hive in hivelist.HiveList.list_hives(self.context, base_config_path = self.config_path, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols']): + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name): for top_key in [ "Microsoft\\SystemCertificates", @@ -87,6 +87,12 @@ class Certificates(interfaces.plugins.PluginInterface): # Key wasn't found in this hive, carry on vollog.log(constants.LOGLEVEL_VVVV, "Key wasn't found in this hive") pass + except exceptions.SwappedInvalidAddressException as exp: + vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)") + pass + except exceptions.PagedInvalidAddressException as exp: + vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)") + pass def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), From c40aecdfdacfde5ac17b658b581aa85595272b85 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 2 Jul 2022 19:38:12 +0900 Subject: [PATCH 085/526] Remove: invalid exceptions code --- volatility3/plugins/windows/registry/certificates.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index e2fe662fc..e873fd1d6 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -90,9 +90,6 @@ class Certificates(interfaces.plugins.PluginInterface): except exceptions.SwappedInvalidAddressException as exp: vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)") pass - except exceptions.PagedInvalidAddressException as exp: - vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)") - pass def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), From 69c50e3c6511bf8dc5dbbd80d908ab82f1b926ff Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 3 Jul 2022 19:57:16 +0530 Subject: [PATCH 086/526] last command added to example 1 --- doc/source/Windows.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index b086cc57b..3e6844f22 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -84,6 +84,22 @@ In windows memory forensics using volatility3, most of the times we do not requi ``windows.pstree`` helps us to display the parent child relation of processes. +.. code-block:: shell-session + + $ python3 vol.py -f MemDump.DMP windows.hashdump + Volatility 3 Framework 2.0.3 + Progress: 100.00 PDB scanning finished + User rid lmhash nthash + + Administrator 500 aad3b435b51404eeaad3b435b51404ee 31d6cfe0d16ae931b73c59d7e0c089c0 + Guest 501 aad3b435b51404eeaad3b435b51404ee 31d6cfe0d16ae931b73c59d7e0c089c0 + Frank Reynolds 1000 aad3b435b51404eeaad3b435b51404ee a88d1e18706d3aa676e01e5943d15911 + HomeGroupUser$ 1002 aad3b435b51404eeaad3b435b51404ee af10ecac6ea817d2bb56e3e5c33ce1cd + Dennis 1003 aad3b435b51404eeaad3b435b51404ee cf96684bbc7877920adaa9663698bf54 + +``windows.hashdump`` helps us to list the hashes of the users in the system. + + From dcc774787cf333ac574ae1dd402f752616e946a4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Jul 2022 22:11:24 +0100 Subject: [PATCH 087/526] Core: Convert try/except/pass to contextlib.supress --- volatility3/framework/automagic/pdbscan.py | 13 ++++---- volatility3/framework/interfaces/objects.py | 5 ++- volatility3/framework/layers/crash.py | 5 ++- volatility3/framework/layers/registry.py | 21 ++++++------ volatility3/framework/layers/resources.py | 4 +-- volatility3/framework/layers/vmware.py | 26 +++++++-------- .../framework/plugins/linux/check_syscall.py | 8 ++--- .../framework/plugins/windows/dlllist.py | 13 ++++---- .../framework/plugins/windows/envars.py | 21 ++++-------- .../framework/plugins/windows/mftscan.py | 7 ++-- .../plugins/windows/registry/userassist.py | 18 ++++------- volatility3/framework/renderers/conversion.py | 6 ++-- .../symbols/mac/extensions/__init__.py | 32 +++++++------------ .../framework/symbols/windows/__init__.py | 14 ++++---- .../symbols/windows/extensions/__init__.py | 21 ++++-------- .../symbols/windows/extensions/pool.py | 28 +++++++--------- .../symbols/windows/extensions/registry.py | 6 ++-- 17 files changed, 97 insertions(+), 151 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index cedbc4919..5cbdbfe0e 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -7,10 +7,11 @@ from loaded PE files. This module contains a standalone scanner, and also a :class:`~volatility3.framework.interfaces.layers.ScannerInterface` based scanner for use within the framework by calling :func:`~volatility3.framework.interfaces.layers.DataLayerInterface.scan`. """ +import contextlib import logging import math import os -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union, Callable +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.configuration import requirements @@ -139,7 +140,8 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]: + def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ + ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): # Rule out kernels that couldn't find a suitable MZ header @@ -159,7 +161,8 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - def test_physical_kernel(physical_layer_name:str , virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]: + def test_physical_kernel(physical_layer_name: str, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ + ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): # Rule out kernels that couldn't find a suitable MZ header @@ -274,7 +277,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] virtual_layer_name = vlayer.name - try: + with contextlib.suppress(exceptions.InvalidAddressException): if vlayer.read(address, 0x2) == b'MZ': res = list( PDBUtility.pdbname_scan(ctx = context, @@ -286,8 +289,6 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): end = address + self.max_pdb_size)) if res: valid_kernel = (virtual_layer_name, address, res[0]) - except exceptions.InvalidAddressException: - pass return valid_kernel # List of methods to be run, in order, to determine the valid kernels diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 2240c58c9..0f8e742fb 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -6,6 +6,7 @@ interpreted values of data from a layer.""" import abc import collections import collections.abc +import contextlib import logging from typing import Any, Dict, List, Mapping, Optional @@ -187,11 +188,9 @@ class ObjectInterface(metaclass = abc.ABCMeta): """ if self.has_member(member_name): # noinspection PyBroadException - try: + with contextlib.suppress(Exception): _ = getattr(self, member_name) return True - except Exception: - pass return False def has_valid_members(self, member_names: List[str]) -> bool: diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index c690c8d8f..6194501ee 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -1,6 +1,7 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib import logging import struct from typing import Tuple, Optional @@ -202,11 +203,9 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: for layer in [WindowsCrashDump32Layer, WindowsCrashDump64Layer]: - try: + with contextlib.suppress(WindowsCrashDumpFormatException): layer.check_header(context.layers[layer_name]) new_name = context.layers.free_layer_name(layer.__name__) context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name return layer(context, new_name, new_name) - except WindowsCrashDumpFormatException: - pass return None diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 55a6e5186..ec7aed217 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import logging from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union @@ -92,11 +92,9 @@ class RegistryHive(linear.LinearlyMappedLayer): @property def root_cell_offset(self) -> int: """Returns the offset for the root cell in this hive.""" - try: + with contextlib.suppress(InvalidAddressException): if self._base_block.Signature.cast("string", max_length = 4, encoding = "latin-1") == 'regf': return self._base_block.RootCell - except InvalidAddressException: - pass return 0x20 def get_cell(self, cell_offset: int) -> 'objects.StructType': @@ -201,11 +199,11 @@ class RegistryHive(linear.LinearlyMappedLayer): if offset & 0x7fffffff > self._get_hive_maxaddr(volatile): vollog.log(constants.LOGLEVEL_VVV, "Layer {} couldn't translate offset {}, greater than {} in {} store of {}".format( - self.name, - hex(offset & 0x7fffffff), - hex(self._get_hive_maxaddr(volatile)), - "volative" if volatile else "non-volatile", - self.get_name())) + self.name, + hex(offset & 0x7fffffff), + hex(self._get_hive_maxaddr(volatile)), + "volative" if volatile else "non-volatile", + self.get_name())) raise RegistryInvalidIndex(self.name, "Mapping request for value greater than maxaddr") storage = self.hive.Storage[volatile] @@ -252,14 +250,13 @@ class RegistryHive(linear.LinearlyMappedLayer): def is_valid(self, offset: int, length: int = 1) -> bool: """Returns a boolean based on whether the offset is valid or not.""" - try: + with contextlib.suppress(exceptions.InvalidAddressException): # Pass this to the lower layers for now return all([ self.context.layers[layer].is_valid(offset, length) for (_, _, offset, length, layer) in self.mapping(offset, length) ]) - except exceptions.InvalidAddressException: - return False + return False @property def minimum_address(self) -> int: diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 8a0e96208..dca215c85 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -184,14 +184,12 @@ class ResourceAccessor(object): stop = False while not stop: detected = None - try: + with contextlib.suppress(AttributeError, IOError): # Detect the content detected = magic.detect_from_fobj(curfile) IMPORTED_MAGIC = True # This is because python-magic and file provide a magic module # Only file's python has magic.detect_from_fobj - except (AttributeError, IOError): - pass if detected: if detected.mime_type == 'application/x-xz': diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 85e961b24..ae4a7d55e 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -1,14 +1,14 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import logging import struct from typing import Any, Dict, List, Optional -from volatility3.framework import interfaces, constants, exceptions +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.layers import physical, segmented, resources +from volatility3.framework.layers import physical, resources, segmented from volatility3.framework.symbols import native vollog = logging.getLogger(__name__) @@ -87,13 +87,13 @@ class VmwareLayer(segmented.SegmentedLayer): offset = offset + name_len + 2 + (index * index_len), layer_name = self._meta_layer)) data_len = flags & 0x3f - + if data_len in [62, 63]: # Handle special data sizes that indicate a longer data stream data_len = 4 if version == 0 else 8 # Read the size of the data data_size = self._context.object(self._choose_type(data_len), - layer_name = self._meta_layer, - offset = offset + 2 + name_len + (indices_len * index_len)) + layer_name = self._meta_layer, + offset = offset + 2 + name_len + (indices_len * index_len)) # Skip two bytes of padding (as it seems?) # Read the actual data data = self._context.object("vmware!bytes", @@ -113,9 +113,9 @@ class VmwareLayer(segmented.SegmentedLayer): if tags[("regionsCount", ())][1] == 0: raise VmwareFormatException(self.name, "VMware VMEM is not split into regions") for region in range(tags[("regionsCount", ())][1]): - offset = tags[("regionPPN", (region, ))][1] * self._page_size - mapped_offset = tags[("regionPageNum", (region, ))][1] * self._page_size - length = tags[("regionSize", (region, ))][1] * self._page_size + offset = tags[("regionPPN", (region,))][1] * self._page_size + mapped_offset = tags[("regionPageNum", (region,))][1] * self._page_size + length = tags[("regionSize", (region,))][1] * self._page_size self._segments.append((offset, mapped_offset, length, length)) @property @@ -153,23 +153,19 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): current_layer_name) vmss_success = False - try: + with contextlib.suppress(IOError): _ = resources.ResourceAccessor().open(vmss).read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmss_success = True - except IOError: - pass vmsn_success = False if not vmss_success: - try: + with contextlib.suppress(IOError): _ = resources.ResourceAccessor().open(vmsn).read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmsn context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmsn_success = True - except IOError: - pass vollog.log(constants.LOGLEVEL_VVVV, f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})") diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 50fd05fa5..6ec5fd354 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -3,11 +3,11 @@ # """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" +import contextlib import logging from typing import List -from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers, constants +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.renderers import format_hints @@ -40,11 +40,9 @@ class Check_syscall(plugins.PluginInterface): symbol_list = [] for sn in vmlinux.symbols: - try: + with contextlib.suppress(exceptions.SymbolError): # When requesting the symbol from the module, a full resolve is performed symbol_list.append((vmlinux.get_symbol(sn).address, sn)) - except exceptions.SymbolError: - pass sorted_symbols = sorted(symbol_list) sym_address = 0 diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 2fd7deeaf..cb7626dfa 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -1,18 +1,19 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib import datetime import logging import ntpath from typing import List, Optional, Type -from volatility3.framework import exceptions, renderers, interfaces, constants +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins import timeliner -from volatility3.plugins.windows import pslist, info +from volatility3.plugins.windows import info, pslist vollog = logging.getLogger(__name__) @@ -28,7 +29,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), requirements.ListRequirement(name = 'pid', @@ -107,12 +108,10 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for entry in proc.load_order_modules(): BaseDllName = FullDllName = renderers.UnreadableValue() - try: + with contextlib.suppress(exceptions.InvalidAddressException): BaseDllName = entry.BaseDllName.get_string() # We assume that if the BaseDllName points to an invalid buffer, so will FullDllName FullDllName = entry.FullDllName.get_string() - except exceptions.InvalidAddressException: - pass if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index 9791fa580..e9015280a 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -1,9 +1,10 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +import contextlib import logging from typing import List -from volatility3.framework import renderers, interfaces, objects, exceptions, constants +from volatility3.framework import constants, exceptions, interfaces, objects, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.plugins.windows import pslist @@ -23,7 +24,7 @@ class Envars(interfaces.plugins.PluginInterface): # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -61,13 +62,11 @@ class Envars(interfaces.plugins.PluginInterface): key = hive.get_key('CurrentControlSet\\Control\\Session Manager\\Environment') sys = True except KeyError: - try: + with contextlib.suppress(KeyError): key = hive.get_key('ControlSet001\\Control\\Session Manager\\Environment') sys = True - except KeyError: - pass if sys: - try: + with contextlib.suppress(KeyError): for node in key.get_values(): try: value_node_name = node.get_name() @@ -78,17 +77,13 @@ class Envars(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVV, "Error while parsing global environment variables keys (some keys might be excluded)") continue - except KeyError: - pass ## The user-specific variables - try: + with contextlib.suppress(KeyError): key = hive.get_key('Environment') ntuser = True - except KeyError: - pass if ntuser: - try: + with contextlib.suppress(KeyError): for node in key.get_values(): try: value_node_name = node.get_name() @@ -99,8 +94,6 @@ class Envars(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVV, "Error while parsing user environment variables keys (some keys might be excluded)") continue - except KeyError: - pass ## The volatile user variables try: diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 654e26db7..c96fd9522 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -1,7 +1,7 @@ # This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import datetime import logging @@ -56,7 +56,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): - try: + with contextlib.suppress(exceptions.PagedInvalidAddressException): mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset @@ -131,9 +131,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) - except exceptions.PagedInvalidAddressException: - pass - def generate_timeline(self): for row in self._generator(): _depth, row_data = row diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index a788f058f..30b5db695 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -3,17 +3,18 @@ # import codecs +import contextlib import datetime import json import logging import os -from typing import Any, List, Tuple, Generator +from typing import Any, Generator, List, Tuple -from volatility3.framework import exceptions, renderers, constants, interfaces +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers.physical import BufferDataLayer from volatility3.framework.layers.registry import RegistryHive -from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.plugins.windows.registry import hivelist @@ -38,7 +39,7 @@ class UserAssist(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.IntRequirement(name = 'offset', description = "Hive Offset", default = None, optional = True), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) ] @@ -126,11 +127,9 @@ class UserAssist(interfaces.plugins.PluginInterface): hive_name = hive.hive.cast(kernel.symbol_table_name + constants.BANG + "_CMHIVE").get_name() if self._win7 is None: - try: + with contextlib.suppress(exceptions.SymbolError): self._win7 = self._win7_or_later() - except exceptions.SymbolError: # self._win7 will be None and only registry value rawdata will be output - pass self._determine_userassist_type() @@ -163,7 +162,6 @@ class UserAssist(interfaces.plugins.PluginInterface): # output any subkeys under Count for subkey in countkey.get_subkeys(): - subkey_name = subkey.get_name() result = (1, ( renderers.format_hints.Hex(hive.hive_offset), @@ -185,10 +183,8 @@ class UserAssist(interfaces.plugins.PluginInterface): for value in countkey.get_values(): value_name = value.get_name() - try: + with contextlib.suppress(UnicodeDecodeError): value_name = codecs.encode(value_name, "rot_13") - except UnicodeDecodeError: - pass if self._win7: guid = value_name.split("\\")[0] diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index 996cf03a5..3ce49bbde 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import datetime import ipaddress import socket @@ -27,10 +27,8 @@ def unixtime_to_datetime(unixtime: int) -> Union[interfaces.renderers.BaseAbsent ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = renderers.UnparsableValue() if unixtime > 0: - try: + with contextlib.suppress(ValueError): ret = datetime.datetime.utcfromtimestamp(unixtime) - except ValueError: - pass return ret diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 94045d2e7..a66bfb534 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -1,19 +1,18 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib +import logging from typing import Generator, Iterable, Optional, Set, Tuple -import logging - -from volatility3.framework import constants, objects, renderers -from volatility3.framework import exceptions, interfaces +from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic vollog = logging.getLogger(__name__) + class proc(generic.GenericIntelProcess): def get_task(self): @@ -29,10 +28,8 @@ class proc(generic.GenericIntelProcess): if not isinstance(parent_layer, interfaces.layers.TranslationLayerInterface): raise TypeError("Parent layer is not a translation layer, unable to construct process layer") - try: + with contextlib.suppress(exceptions.InvalidAddressException): dtb = self.get_task().map.pmap.pm_cr3 - except exceptions.InvalidAddressException: - return None if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.p_pid}" @@ -41,10 +38,8 @@ class proc(generic.GenericIntelProcess): return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - try: + with contextlib.suppress(exceptions.InvalidAddressException): task = self.get_task() - except exceptions.InvalidAddressException: - return try: current_map = task.map.hdr.links.next @@ -55,9 +50,9 @@ class proc(generic.GenericIntelProcess): for i in range(task.map.hdr.nentries): if (not current_map or - current_map.vol.offset in seen or - not self._context.layers[task.vol.native_layer_name].is_valid(current_map.dereference().vol.offset, current_map.dereference().vol.size)): - + current_map.vol.offset in seen or + not self._context.layers[task.vol.native_layer_name].is_valid(current_map.dereference().vol.offset, + current_map.dereference().vol.size)): vollog.log(constants.LOGLEVEL_VVV, "Breaking process maps iteration due to invalid state.") break @@ -102,10 +97,8 @@ class fileglob(objects.StructType): if self.has_member("fg_type"): ret = self.fg_type elif self.fg_ops != 0: - try: + with contextlib.suppress(exceptions.InvalidAddressException): ret = self.fg_ops.fo_type - except exceptions.InvalidAddressException: - pass if ret: ret = str(ret.description).replace("DTYPE_", "") @@ -456,7 +449,7 @@ class queue_entry(objects.StructType): seen = set() for attr in ['next', 'prev']: - try: + with contextlib.suppress(exceptions.InvalidAddressException): n = getattr(self, attr).dereference().cast(type_name) while n is not None and n.vol.offset != list_head: @@ -473,9 +466,6 @@ class queue_entry(objects.StructType): n = getattr(n.member(attr = member_name), attr).dereference().cast(type_name) - except exceptions.InvalidAddressException: - pass - class ifnet(objects.StructType): diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index 899b89dc2..cfac87e2c 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -1,10 +1,11 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import extensions -from volatility3.framework.symbols.windows.extensions import registry, pool, pe +from volatility3.framework.symbols.windows.extensions import pe, pool, registry class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -39,26 +40,23 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('_VACB', extensions.VACB) self.set_type_class('_POOL_TRACKER_BIG_PAGES', pool.POOL_TRACKER_BIG_PAGES) self.set_type_class('_IMAGE_DOS_HEADER', pe.IMAGE_DOS_HEADER) - + # Might not necessarily defined in every version of windows self.optional_set_type_class('_IMAGE_NT_HEADERS', pe.IMAGE_NT_HEADERS) self.optional_set_type_class('_IMAGE_NT_HEADERS64', pe.IMAGE_NT_HEADERS) # This doesn't exist in very specific versions of windows - try: + with contextlib.suppress(ValueError): if self.get_type("_POOL_TRACKER_BIG_PAGES").has_member("PoolType"): self.set_type_class('_POOL_HEADER', pool.POOL_HEADER_VISTA) else: self.set_type_class('_POOL_HEADER', pool.POOL_HEADER) - except ValueError: - pass # these don't exist in windows XP self.optional_set_type_class('_MMADDRESS_NODE', extensions.MMVAD_SHORT) - + # these were introduced starting in windows 8 self.optional_set_type_class('_MM_AVL_NODE', extensions.MMVAD_SHORT) - + # these were introduced starting in windows 7 self.optional_set_type_class('_RTL_BALANCED_NODE', extensions.MMVAD_SHORT) - \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b5ee272a0..7be9c4791 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -3,6 +3,7 @@ # import collections.abc +import contextlib import datetime import functools import logging @@ -305,7 +306,7 @@ class MMVAD(MMVAD_SHORT): file_name = renderers.NotApplicableValue() - try: + with contextlib.suppress(exceptions.InvalidAddressException): # this is for xp and 2003 if self.has_member("ControlArea"): filename_obj = self.ControlArea.FilePointer.FileName @@ -318,9 +319,6 @@ class MMVAD(MMVAD_SHORT): if filename_obj.Length > 0: file_name = filename_obj.get_string() - except exceptions.InvalidAddressException: - pass - return file_name @@ -364,6 +362,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): yield device device = device.AttachedDevice.dereference() + class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" @@ -374,7 +373,7 @@ class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" - device = self.DeviceObject.dereference() + device = self.DeviceObject.dereference() while device: yield device device = device.NextDevice.dereference() @@ -413,15 +412,11 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. if self._context.layers[self.vol.native_layer_name].is_valid(self.DeviceObject): - try: + with contextlib.suppress(ValueError): name = f"\\Device\\{self.DeviceObject.get_device_name()}" - except ValueError: - pass - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): name += self.FileName.String - except (TypeError, exceptions.InvalidAddressException): - pass return name @@ -1114,12 +1109,10 @@ class SHARED_CACHE_MAP(objects.StructType): iterval = 0 while (iterval < full_blocks) and (full_blocks <= 4): vacb_obj = self.InitialVacbs[iterval] - try: + with contextlib.suppress(exceptions.InvalidAddressException): # Make sure that the SharedCacheMap member of the VACB points back to the parent object. if vacb_obj.SharedCacheMap == self.vol.offset: self.save_vacb(vacb_obj, vacb_list) - except exceptions.InvalidAddressException: - pass iterval += 1 # We also have to account for the spill over data that is not found in the full blocks. diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index 368765497..79ea60027 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -1,12 +1,14 @@ +import contextlib import functools import logging import struct -from typing import Optional, Tuple, List, Dict, Union +from typing import Dict, List, Optional, Tuple, Union -from volatility3.framework import objects, interfaces, constants, symbols, exceptions, renderers -from volatility3.framework.renderers import conversion from volatility3.plugins.windows.poolscanner import PoolConstraint +from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols +from volatility3.framework.renderers import conversion + vollog = logging.getLogger(__name__) @@ -138,7 +140,7 @@ class POOL_HEADER(objects.StructType): if addr - optional_headers_length >= padding_length > addr: continue - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): mem_object = self._context.object(symbol_table_name + constants.BANG + type_name, layer_name = self.vol.layer_name, offset = addr + body_offset + start_offset, @@ -147,15 +149,13 @@ class POOL_HEADER(objects.StructType): if mem_object.is_valid(): yield mem_object - except (TypeError, exceptions.InvalidAddressException): - pass - # use the bottom up approach for windows 7 and earlier else: type_size = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + type_name).size if constraint.additional_structures: for additional_structure in constraint.additional_structures: - type_size += self._context.symbol_space.get_type(symbol_table_name + constants.BANG + additional_structure).size + type_size += self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + additional_structure).size rounded_size = conversion.round(type_size, alignment, up = True) @@ -164,11 +164,9 @@ class POOL_HEADER(objects.StructType): offset = self.vol.offset + self.BlockSize * alignment - rounded_size, native_layer_name = native_layer_name) - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): if mem_object.is_valid(): yield mem_object - except (TypeError, exceptions.InvalidAddressException): - pass @classmethod @functools.lru_cache() @@ -177,20 +175,18 @@ class POOL_HEADER(objects.StructType): headers = [] sizes = [] for header in [ - 'CREATOR_INFO', 'NAME_INFO', 'HANDLE_INFO', 'QUOTA_INFO', 'PROCESS_INFO', 'AUDIT_INFO', 'EXTENDED_INFO', - 'HANDLE_REVOCATION_INFO', 'PADDING_INFO' + 'CREATOR_INFO', 'NAME_INFO', 'HANDLE_INFO', 'QUOTA_INFO', 'PROCESS_INFO', 'AUDIT_INFO', 'EXTENDED_INFO', + 'HANDLE_REVOCATION_INFO', 'PADDING_INFO' ]: - try: + with contextlib.suppress(AttributeError, exceptions.SymbolError): type_name = f"{symbol_table_name}{constants.BANG}_OBJECT_HEADER_{header}" header_type = context.symbol_space.get_type(type_name) headers.append(header) sizes.append(header_type.size) - except (AttributeError, exceptions.SymbolError): # Some of these may not exist, for example: # if build < 9200: PADDING_INFO else: AUDIT_INFO # if build == 10586: HANDLE_REVOCATION_INFO else EXTENDED_INFO # based on what's present and what's not, this list should be the right order and the right length - pass return headers, sizes def is_free_pool(self): diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 47ff24506..c71fcf49b 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import enum import logging import struct @@ -75,12 +75,10 @@ class CMHIVE(objects.StructType): """ for attr in ["FileFullPath", "FileUserName", "HiveRootPath"]: - try: + with contextlib.suppress(AttributeError, exceptions.InvalidAddressException): name = getattr(self, attr) if name.Length > 0: return name.get_string() - except (AttributeError, exceptions.InvalidAddressException): - pass return None From 3679134f01abcd901f430be1292d99de21c093fc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Jul 2022 22:45:11 +0100 Subject: [PATCH 088/526] Core: Prevent circular dependency on imports --- volatility3/framework/interfaces/objects.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 3cc23e759..d1f442d69 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -243,11 +243,10 @@ class ObjectInterface(metaclass = abc.ABCMeta): @classmethod @abc.abstractmethod - def child_template(cls, template: 'Template', child: str) -> interfaces.objects.Template: + def child_template(cls, template: 'Template', child: str) -> 'interfaces.objects.Template': """Returns the template of the child member from the parent.""" raise KeyError(f"Template does not contain any children: {template.vol.type_name}") - @classmethod @abc.abstractmethod def has_member(cls, template: 'Template', member_name: str) -> bool: From b1b4d21bbcfd0095a10d363e99d8b4bcbfb1a015 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Jul 2022 22:46:31 +0100 Subject: [PATCH 089/526] Core: Prevent circular dependency on imports - take 2 --- volatility3/framework/interfaces/objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index d1f442d69..fcf3c8d6c 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -312,7 +312,7 @@ class Template: offset.""" @abc.abstractmethod - def child_template(self, child: str) -> interfaces.objects.Template: + def child_template(self, child: str) -> 'interfaces.objects.Template': """Returns the `child` member template from its parent.""" @abc.abstractmethod From ec78fe7d8dd015c8ebfc366a5937cebe2bf92b3e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 4 Jul 2022 15:49:03 +0900 Subject: [PATCH 090/526] Fix: try/except/pass to contextlib.supress by #782 --- volatility3/plugins/windows/registry/certificates.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index e873fd1d6..429db96a6 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,3 +1,4 @@ +import contextlib import logging import struct from typing import List, Iterator, Optional, Tuple, Type @@ -67,7 +68,7 @@ class Certificates(interfaces.plugins.PluginInterface): "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - try: + with contextlib.suppress(KeyError, exceptions.SwappedInvalidAddressException): # Walk it node_path = hive.get_key(top_key, return_list = True) for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): @@ -83,13 +84,6 @@ class Certificates(interfaces.plugins.PluginInterface): file_handle.close() yield (0, (top_key, reg_section, key_hash, name)) - except KeyError: - # Key wasn't found in this hive, carry on - vollog.log(constants.LOGLEVEL_VVVV, "Key wasn't found in this hive") - pass - except exceptions.SwappedInvalidAddressException as exp: - vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)") - pass def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), From 2f25312a5c96376b58817772bde1371b424d3f49 Mon Sep 17 00:00:00 2001 From: Malware Utkonos Date: Mon, 4 Jul 2022 13:08:30 -0400 Subject: [PATCH 091/526] Refactor try to reduce size of clause to only what is needed. Based on feedback, memory_object.get_available_pages() might raise this type of exception, so it's still inside the try clause. --- .../framework/plugins/windows/dumpfiles.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 58166ee7f..2c3f8c2d5 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -63,29 +63,28 @@ class DumpFiles(interfaces.plugins.PluginInterface): :return: result status """ filedata = open_method(desired_file_name) - try: - # Description of these variables: - # memoffset: offset in the specified layer where the page begins - # fileoffset: write to this offset in the destination file - # datasize: size of the page + # Description of these variables: + # memoffset: offset in the specified layer where the page begins + # fileoffset: write to this offset in the destination file + # datasize: size of the page - # track number of bytes written so we don't write empty files to disk - bytes_written = 0 + # track number of bytes written so we don't write empty files to disk + bytes_written = 0 + try: for memoffset, fileoffset, datasize in memory_object.get_available_pages(): data = layer.read(memoffset, datasize, pad = True) bytes_written += len(data) filedata.seek(fileoffset) filedata.write(data) - - if not bytes_written: - vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") - return None - else: - vollog.debug(f"Stored {filedata.preferred_filename}") - return filedata except exceptions.InvalidAddressException: vollog.debug(f"Unable to dump file at {file_object.vol.offset:#x}") return None + if not bytes_written: + vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") + return None + vollog.debug(f"Stored {filedata.preferred_filename}") + + return filedata @classmethod def process_file_object(cls, context: interfaces.context.ContextInterface, primary_layer_name: str, From 772ae98eb1966b4b0fa7451673352c6f8f97095c Mon Sep 17 00:00:00 2001 From: Malware Utkonos Date: Mon, 4 Jul 2022 13:27:17 -0400 Subject: [PATCH 092/526] Style changes including yapf according to .style.yapf in package root --- .../framework/plugins/windows/dumpfiles.py | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 58166ee7f..26b637fc3 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -5,6 +5,7 @@ import logging import ntpath from typing import List, Tuple, Type, Optional, Generator + from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -32,8 +33,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.IntRequirement(name = 'pid', description = "Process ID to include (all other processes are excluded)", optional = True), @@ -98,12 +100,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): :param open_method: class for constructing output files :param file_obj: the FILE_OBJECT """ - # Filtering by these types of devices prevents us from processing other types of devices that # use the "File" object type, such as \Device\Tcp and \Device\NamedPipe. if file_obj.DeviceObject.DeviceType not in [FILE_DEVICE_DISK, FILE_DEVICE_NETWORK_FILE_SYSTEM]: - vollog.log(constants.LOGLEVEL_VVV, - f"The file object at {file_obj.vol.offset:#x} is not a file on disk") + vollog.log(constants.LOGLEVEL_VVV, f"The file object at {file_obj.vol.offset:#x} is not a file on disk") return # Depending on the type of object (DataSection, ImageSection, SharedCacheMap) we may need to @@ -120,7 +120,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): # layer to read from, # file extension to apply, # ) - dump_parameters = [] + dump_parameters = list() # The DataSectionObject and ImageSectionObject caches are handled in basically the same way. # We carve these "pages" from the memory_layer. @@ -131,8 +131,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if control_area.is_valid(): dump_parameters.append((control_area, memory_layer, extension)) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") # The SharedCacheMap is handled differently than the caches above. # We carve these "pages" from the primary_layer. @@ -142,8 +141,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if shared_cache_map.is_valid(): dump_parameters.append((shared_cache_map, primary_layer, "vacb")) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] @@ -151,7 +149,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): memory_object.vol.offset, cache_name, ntpath.basename(obj_name), extension) - file_handle = DumpFiles.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) + file_handle = cls.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) file_output = "Error dumping file" if file_handle: @@ -185,8 +183,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): try: object_table = proc.ObjectTable except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") continue for entry in handles_plugin.handles(object_table): @@ -218,12 +215,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue - for result in self.process_file_object(self.context, kernel.layer_name, self.open, - file_obj): + for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot extract file from VAD at {vad.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"Cannot extract file from VAD at {vad.vol.offset:#x}") elif offsets: # Now process any offsets explicitly requested by the user. @@ -234,10 +229,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not is_virtual: layer_name = self.context.layers[layer_name].config["memory_layer"] - file_obj = self.context.object( - kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", + file_obj = self.context.object(kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", layer_name = layer_name, - native_layer_name = kernel.layer_name, + native_layer_name = kernel.layer_name, offset = offset) for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) @@ -246,9 +240,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): def run(self): # a list of tuples (, ) where is the address and is True for virtual. - offsets = [] + offsets = list() # a list of processes matching the pid filter. all files for these process(es) will be dumped. - procs = [] + procs = list() kernel = self.context.modules[self.config['kernel']] if self.config.get("virtaddr", None) is not None: From b30cb5d96842178085967f9462487e1b08b3ec19 Mon Sep 17 00:00:00 2001 From: Malware Utkonos Date: Mon, 4 Jul 2022 13:47:23 -0400 Subject: [PATCH 093/526] Move debug logging based on feedback. --- volatility3/framework/plugins/windows/dumpfiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 2c3f8c2d5..7e1480cbe 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -82,8 +82,8 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not bytes_written: vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") return None - vollog.debug(f"Stored {filedata.preferred_filename}") + vollog.debug(f"Stored {filedata.preferred_filename}") return filedata @classmethod From 84d26ba4bdf46b0280b343b3bd3c3b7ee54238c8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 5 Jul 2022 11:08:41 +0100 Subject: [PATCH 094/526] Core: Add in API_CHANGES updates --- API_CHANGES.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/API_CHANGES.md b/API_CHANGES.md index 274d1d8bb..4d8733286 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,14 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.3.0 +===== +Add in `child_template` to template class + +2.2.0 +===== +Changes to linux core calls + 2.1.0 ===== Add in the linux `task.get_threads` method to the API. From 179d35d03dded64676339ceea37b04ef4a7107c5 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:15:48 +0530 Subject: [PATCH 095/526] volatility to volatiliy3 --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 294f8ff63..d8b30ff28 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -1,7 +1,7 @@ Linux Tutorial ============== -This guide gives you a brief introduction to how volatility3 works and some demonstration on suite of plugins available from +This guide gives you a brief introduction to how volatility3 works and some demonstration of several of the plugins available from Acquiring memory ---------------- From 92f308b5e7c7b8556072e411d66a5e089828e658 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:16:07 +0530 Subject: [PATCH 096/526] volatility3 specified --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index d8b30ff28..28fa4e0cd 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -6,7 +6,7 @@ This guide gives you a brief introduction to how volatility3 works and some demo Acquiring memory ---------------- -Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `LiME `_ for this purpose. +Volatility3 does not provide the ability to acquire memory. In this tutorial we will see how we can use `LiME `_ for this purpose. It supports 32 and 64 bit captures from native Intel hardware as well as virtual machine guests. It also supports capture from Android devices. See below for example commands building and running LiME: From b782e1d751d57fabea3e382ebdee55966eb8eeaf Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:27:36 +0530 Subject: [PATCH 097/526] path adjustments made to have relative and generic --- doc/source/Linux.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 28fa4e0cd..3c77a16b6 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -16,12 +16,12 @@ It also supports capture from Android devices. See below for example commands bu $ cd lime-forensics-1.1-r14/src $ make .... - CC [M] /home/mhl/Downloads/src/tcp.o - CC [M] /home/mhl/Downloads/src/disk.o + CC [M] lime-forensics-1.1-r14/src/tcp.o + CC [M] lime-forensics-1.1-r14/src/disk.o .... - $ sudo insmod lime-3.2.0-23-generic.ko "path=/home/mhl/ubuntu.lime format=lime" - $ ls -alh /home/mhl/ubuntu.lime - -r--r--r-- 1 root root 2.0G Aug 17 19:37 /home/mhl/ubuntu.lime + $ sudo insmod lime-3.2.0-23-generic.ko "path=/tmp/ubuntu.lime format=lime" + $ ls -alh /tmp/ubuntu.lime + -r--r--r-- 1 root root 2.0G Aug 17 19:37 /tmp/ubuntu.lime Procedure to create symbol tables for linux -------------------------------------------- From fc48e7b83f59320642952466e40e515b242aa0ec Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:33:33 +0530 Subject: [PATCH 098/526] regarding ISF server its moved to tips section --- doc/source/Linux.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 3c77a16b6..21b8659fa 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -27,8 +27,9 @@ Procedure to create symbol tables for linux -------------------------------------------- To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. -We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. -After creating the file or downloading the file from the ISF server, please place the file under the directory ``volatility3/symbols/linux``. Make a directory linux under symbols. + +.. tip:: We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. + After creating the file or downloading the file from the ISF server, please place the file under the directory ``volatility3/symbols/linux``. Make a directory linux under symbols. Listing plugins From 89f1116374a1bd2e57696d2109fc7a9475d43c5c Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:36:58 +0530 Subject: [PATCH 099/526] Sentence reframed and clarrified regarding sample plugin list --- doc/source/Linux.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 21b8659fa..82b42b251 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -35,7 +35,7 @@ To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symb Listing plugins --------------- -Following are the list of linux plugins available for volatility3. More plugins will be available on future releases. +Following are the sample of linux plugins available for volatility3. More plugins will be available on future releases. For plugin requests, Please create an issue with description of the plugin. .. code-block:: shell-session @@ -47,6 +47,8 @@ For plugin requests, Please create an issue with description of the plugin. linux.check_creds.Check_creds linux.check_idt.Check_idt +.. note:: Here the the command is piped to grep and head in-order to give you sample list of plugins. + Using plugins ------------- From 3da043c60192a706e82b115952065b84d892de9e Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:39:08 +0530 Subject: [PATCH 100/526] Command syntax angular bracket added --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 82b42b251..bf6b28be9 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -57,7 +57,7 @@ The following is the syntax to run volatility tool. .. code-block:: shell-session - $ python3 vol.py -f plugin_name plugin_option + $ python3 vol.py -f Example From ed361893c7e3b2a055d7d5c5785102da453e3f4c Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:40:43 +0530 Subject: [PATCH 101/526] command fix vol.py to python3 vol.py --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index bf6b28be9..8919a83ce 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -40,7 +40,7 @@ For plugin requests, Please create an issue with description of the plugin. .. code-block:: shell-session - $ vol3 --help | grep -i linux. | head -n 5 + $ python3 vol.py --help | grep -i linux. | head -n 5 banners.Banners Attempts to identify potential linux banners in an linux.bash.Bash Recovers bash command history from memory. linux.check_afinfo.Check_afinfo From 20b76830f7ccefcb6b2d01dd6c0423083891a9a1 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:42:49 +0530 Subject: [PATCH 102/526] Removed external link to memory dump --- doc/source/Linux.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 8919a83ce..773aa80e4 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -66,8 +66,7 @@ Example Example 1 ~~~~~~~~~ -In this example we will be using memory dump from Insomni'hack teaser 2020 CTF. Challenge name Getdents, you can find the memory dump -in the link `here `_ . We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. +In this example we will be using memory dump from Insomni'hack teaser 2020 CTF. Challenge name Getdents. We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. I'd like to say thanks to `stuxnet `_ for providing this memory dump and `writeup `_. From 0844929610893ca5e239037c1b1f2031316acbb2 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:45:13 +0530 Subject: [PATCH 103/526] Use same voltility3 version in documentation volatility3 2.0.1 --- doc/source/Linux.rst | 8 ++++---- doc/source/Windows.rst | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 773aa80e4..8132453ce 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -74,7 +74,7 @@ I'd like to say thanks to `stuxnet `_ for provid $ python3 vol.py -f memory.vmem banners - Volatility 3 Framework 2.0.3 + Volatility 3 Framework 2.0.1 Progress: 100.00 PDB scanning finished Offset Banner @@ -96,7 +96,7 @@ If you do not find the ISF file then, please follow the instructions on :ref:`Li $ python3 vol.py -f memory.vmem linux.pslist - Volatility 3 Framework 2.0.3 Stacking attempts finished + Volatility 3 Framework 2.0.1 Stacking attempts finished PID PPID COMM @@ -123,7 +123,7 @@ If you do not find the ISF file then, please follow the instructions on :ref:`Li .. code-block:: shell-session $ python3 vol.py -f memory.vmem linux.pstree - Volatility 3 Framework 2.0.3 + Volatility 3 Framework 2.0.1 Progress: 100.00 Stacking attempts finished PID PPID COMM @@ -167,7 +167,7 @@ Now to find the commands ran in bash shell. Lets use ``linux.bash``. $ python3 vol.py -f memory.vmem linux.bash - Volatility 3 Framework 2.0.3 + Volatility 3 Framework 2.0.1 Progress: 100.00 Stacking attempts finished PID Process CommandTime Command diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index 3e6844f22..e722be0a3 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -45,7 +45,7 @@ In windows memory forensics using volatility3, most of the times we do not requi $ python3 vol.py -f MemDump.DMP windows.pslist | head -n 10 - Volatility 3 Framework 2.0.2 PDB scanning finished + Volatility 3 Framework 2.0.1 PDB scanning finished PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output @@ -61,7 +61,7 @@ In windows memory forensics using volatility3, most of the times we do not requi .. code-block:: shell-session $ python3 vol.py -f MemDump.DMP windows.pstree | head -n 20 - Volatility 3 Framework 2.0.2 PDB scanning finished + Volatility 3 Framework 2.0.1 PDB scanning finished PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime From 1ccd31b506768ef5e9201d5722aeff59be309919 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:47:47 +0530 Subject: [PATCH 104/526] Added note regarding pipe in windows doc and moved winPEM to tip --- doc/source/Windows.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index e722be0a3..2f8d58a04 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -6,7 +6,9 @@ This guide gives you a brief introduction to how volatility3 works and some demo Acquiring memory ---------------- -Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `WinPmem `_ for this purpose. +Volatility does not provide the ability to acquire memory. + +.. tip:: You could use `WinPmem `_ for collecting windows memory dump. Listing Plugins --------------- @@ -20,6 +22,8 @@ Listing Plugins windows.dlllist.DllList Lists the loaded modules in a particular windows +.. note:: Here the the command is piped to grep and head in-order to give you sample list of plugins. + Using plugins ------------- From 73eec3386dc843761e37bfff6420e173cff891e8 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:48:49 +0530 Subject: [PATCH 105/526] Reference to memory dump removed --- doc/source/Windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index 2f8d58a04..a9e712fc9 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -40,7 +40,7 @@ Example Example 1 ~~~~~~~~~ -In this example we will be using memory dump from PragyanCTF'22. The dump is available `here `_. +In this example we will be using memory dump from PragyanCTF'22. We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. In windows memory forensics using volatility3, most of the times we do not require creating a ISF file. From bcc923b1b92567c66f3e96c5996084fdff892895 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:50:33 +0530 Subject: [PATCH 106/526] Info regarding pipe added --- doc/source/Windows.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index a9e712fc9..d26f41aa8 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -88,6 +88,9 @@ In windows memory forensics using volatility3, most of the times we do not requi ``windows.pstree`` helps us to display the parent child relation of processes. +.. note:: Here the the command is piped to head in-order to give you smaller output of process here top 20. + + .. code-block:: shell-session $ python3 vol.py -f MemDump.DMP windows.hashdump From 596047c251a6363e49656e79284a56e974b0c8a3 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:51:32 +0530 Subject: [PATCH 107/526] small adjustment made in note regarding pipe --- doc/source/Linux.rst | 2 +- doc/source/Windows.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 8132453ce..b5a3db09c 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -47,7 +47,7 @@ For plugin requests, Please create an issue with description of the plugin. linux.check_creds.Check_creds linux.check_idt.Check_idt -.. note:: Here the the command is piped to grep and head in-order to give you sample list of plugins. +.. note:: Here the the command is piped to grep and head in-order to give you sample list of linux plugins. Using plugins diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index d26f41aa8..55677a67c 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -22,7 +22,7 @@ Listing Plugins windows.dlllist.DllList Lists the loaded modules in a particular windows -.. note:: Here the the command is piped to grep and head in-order to give you sample list of plugins. +.. note:: Here the the command is piped to grep and head in-order to give you sample list of windows plugins. Using plugins ------------- From e7b33f6c841b250db1a1014d2acd412f456705df Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:54:37 +0530 Subject: [PATCH 108/526] Description on listing plugins in windows added --- doc/source/Windows.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index 55677a67c..a6c67780e 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -13,6 +13,9 @@ Volatility does not provide the ability to acquire memory. Listing Plugins --------------- +Following are the sample of linux plugins available for volatility3. More plugins will be available on future releases. +For plugin requests, Please create an issue with description of the plugin. + .. code-block:: shell-session $ python3 vol.py --help | grep windows | head -n 5 From 3714fa8c9254c6a29bc2657788521de285baf6f3 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:56:45 +0530 Subject: [PATCH 109/526] Note regarding using sudo added --- doc/source/Linux.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index b5a3db09c..180e7c697 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -23,6 +23,8 @@ It also supports capture from Android devices. See below for example commands bu $ ls -alh /tmp/ubuntu.lime -r--r--r-- 1 root root 2.0G Aug 17 19:37 /tmp/ubuntu.lime +.. note:: The above command required sudo inorder to access the files which are root only. + Procedure to create symbol tables for linux -------------------------------------------- From df277b9e802899368186aa04c4d106ba06de1e9b Mon Sep 17 00:00:00 2001 From: Frank Gomulka Date: Wed, 13 Jul 2022 13:49:30 -0500 Subject: [PATCH 110/526] Add testing framework --- .github/workflows/test.yaml | 54 +++++ test/README.md | 34 +++ test/conftest.py | 40 ++++ test/known_files.json | 19 ++ test/requirements-testing.txt | 8 + test/test_volatility.py | 381 ++++++++++++++++++++++++++++++++++ 6 files changed, 536 insertions(+) create mode 100644 .github/workflows/test.yaml create mode 100644 test/README.md create mode 100644 test/conftest.py create mode 100644 test/known_files.json create mode 100644 test/requirements-testing.txt create mode 100644 test/test_volatility.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..5a3f90565 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,54 @@ +name: Test Volatility3 +on: [push] +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install Cmake + pip install setuptools wheel + pip install -U pytest + pip install -r ./test/requirements-testing.txt + + - name: Build PyPi packages + run: | + python setup.py sdist --formats=gztar,zip + python setup.py bdist_wheel + + - name: Download images + run: | + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz" + gunzip linux-sample-1.bin.gz + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" + gunzip win-xp-laptop-2005-06-25.img.gz + + - name: Download and Extract symbols + run: | + cd ./volatility3/symbols + curl -sLO https://downloads.volatilityfoundation.org/volatility3/symbols/linux.zip + unzip linux.zip + cd - + + - name: Testing... + run: | + py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v + py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + + - name: Clean up post-test + run: | + rm -rf *.lime + rm -rf *.img + cd volatility3/symbols + rm -rf linux + rm -rf linux.zip + cd - diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..dcbe289b0 --- /dev/null +++ b/test/README.md @@ -0,0 +1,34 @@ +# Volatility 3 Testing Framework + +## Requirements + +The Volatility 3 Testing Framework requires the same version of Python as Volatility3 itself. To install the current set of dependencies that the framework requires, use a command like this: + +```shell +pip3 install -r requirements-testing.txt +``` + +NOTE: `requirements-testing.txt` can be found in this current `test/` directory. + +## Quick Start: Manual Testing + +1. To test Volatility 3 on an image, first download one with a command such as: + +```shell +curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" +gunzip win-xp-laptop-2005-06-25.img.gz +``` + +2. In many cases, more symbols are required to be downloaded to the `./volatility3/symbols` directory. + +3. To manually run the tests, run a command, such as: + +```shell +py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows +``` + +The above command runs all available tests for windows on the `win-xp-laptop-2005-06-25.img` image. To choose a more specific set of tests, change the phrase after `-k` in this command. + +## Github Actions + +This framework currently tests two images (one linux image and one windows image) after every push on any branch. For more information/context, find the actions setup in `./github/workflows/test.yaml` \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 000000000..9d3d27fc5 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,40 @@ +# This file is used to augment the test configuration + +import os +import pytest + +def pytest_addoption(parser): + parser.addoption("--volatility", action="store", default=None, + required=True, + help="path to the volatility script") + + parser.addoption("--python", action="store", default="python3", + help="The name of the interpreter to use when running the volatility script") + + parser.addoption("--image", action="append", default=[], + help="path to an image to test") + + parser.addoption("--image-dir", action="append", default=[], + help="path to a directory containing images to test") + +def pytest_generate_tests(metafunc): + """Parameterize tests based on image names""" + + images = metafunc.config.getoption('image') + for d in metafunc.config.getoption('image_dir'): + images = images + [os.path.join(d, x) for x in os.listdir(d)] + + # tests with "image" parameter are run against images + if 'image' in metafunc.fixturenames: + metafunc.parametrize("image", + images, + ids=[os.path.basename(image) for image in images]) + +# Fixtures +@pytest.fixture +def volatility(request): + return request.config.getoption("--volatility") + +@pytest.fixture +def python(request): + return request.config.getoption("--python") diff --git a/test/known_files.json b/test/known_files.json new file mode 100644 index 000000000..fbc40e48b --- /dev/null +++ b/test/known_files.json @@ -0,0 +1,19 @@ +{ + "windows_dumpfiles": { + "win-xp-laptop-2005-06-25.img": { + "0x82220e78": [ + "9bdd5532286f1660f3778e68bc36efe6", + "e3bc1e9e7370e3b5a661ebe591ecf4ec" + ], + "0x82350bf8": [ + "e5c5e8d97b6280745b41f6572c85d1f0", + "8589f1463422884dbf1411aaad278465" + ], + "0x81eaf418": [ + "f7a1ae2060a58f8470b97affdb46dccf", + "54fd611021fa784912530b8007545986" + ], + "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" + } + } + } \ No newline at end of file diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt new file mode 100644 index 000000000..d37dc93c3 --- /dev/null +++ b/test/requirements-testing.txt @@ -0,0 +1,8 @@ +# These packages are required for core functionality. +pefile>=2017.8.1 #foo + +# The following packages are optional. +# If certain packages are not necessary, place a comment (#) at the start of the line. + +# This is required for the yara plugins +yara-python>=3.8.0 diff --git a/test/test_volatility.py b/test/test_volatility.py new file mode 100644 index 000000000..527d86dd3 --- /dev/null +++ b/test/test_volatility.py @@ -0,0 +1,381 @@ +# volatility3 tests +# + +# +# IMPORTS +# + +import os +import subprocess +import sys +import shutil +import tempfile +import hashlib +import ntpath +import json + +import pytest + +# +# HELPER FUNCTIONS +# + +def runvol(args, volatility, python): + volpy = volatility + python_cmd = python + + cmd = [python_cmd, volpy] + args + print(" ".join(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + print("stdout:") + sys.stdout.write(str(stdout)) + print("") + print("stderr:") + sys.stdout.write(str(stderr)) + print("") + + return p.returncode, stdout, stderr + +def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): + args = globalargs + [ + "--single-location", + img, + "-q", + plugin, + ] + pluginargs + + return runvol(args, volatility, python) + +# +# TESTS +# + +# WINDOWS + +def test_windows_pslist(image, volatility, python): + rc, out, err = runvol_plugin("windows.pslist.PsList", image, volatility, python) + out = out.lower() + assert out.find(b"system") != -1 + assert out.find(b"csrss.exe") != -1 + assert out.find(b"svchost.exe") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + + rc, out, err = runvol_plugin( + "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"]) + out = out.lower() + assert out.find(b"system") != -1 + assert out.count(b"\n") < 10 + assert rc == 0 + assert rc == 0 + +def test_windows_psscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) + out = out.lower() + assert out.find(b"system") != -1 + assert out.find(b"csrss.exe") != -1 + assert out.find(b"svchost.exe") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_dlllist(image, volatility, python): + rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) + out = out.lower() + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_modules(image, volatility, python): + rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) + out = out.lower() + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_hivelist(image, volatility, python): + rc, out, err = runvol_plugin("windows.registry.hivelist.HiveList", image, volatility, python) + out = out.lower() + + not_xp = out.find(b"\\systemroot\\system32\\config\\software") + if not_xp == -1: + assert out.find(b"\\device\\harddiskvolume1\\windows\\system32\\config\\software") != -1 + + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_windows_dumpfiles(image, volatility, python): + + json_file = open('./test/known_files.json') + + known_files = json.load(json_file) + + failed_chksms = 0 + + if sys.platform == 'win32': + file_name = ntpath.basename(image) + else: + file_name = os.path.basename(image) + + try: + for addr in known_files["windows_dumpfiles"][file_name]: + + path = tempfile.mkdtemp() + + rc, out, err = runvol_plugin("windows.dumpfiles.DumpFiles", image, volatility, python, globalargs=["-o", path], pluginargs=["--virtaddr", addr]) + + for file in os.listdir(path): + fp = open(os.path.join(path, file), "rb") + if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: + failed_chksms += 1 + fp.close() + + shutil.rmtree(path) + + json_file.close() + + assert failed_chksms == 0 + assert rc == 0 + except Exception as e: + json_file.close() + print("Key Error raised on " + str(e)) + assert False + +def test_windows_handles(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"System Pid 4") != -1 + assert out.find(b"MACHINE\\SYSTEM\\CONTROLSET001\\CONTROL\\SESSION MANAGER\\MEMORY MANAGEMENT\\PREFETCHPARAMETERS") != -1 + assert out.find(b"MACHINE\\SYSTEM\\SETUP") != -1 + assert out.count(b"\n") > 500 + assert rc == 0 + +def test_windows_svcscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) + + assert out.find(b"Microsoft ACPI Driver") != -1 + assert out.count(b"\n") > 250 + assert rc == 0 + +def test_windows_privileges(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"SeCreateTokenPrivilege") != -1 + assert out.find(b"SeCreateGlobalPrivilege") != -1 + assert out.find(b"SeAssignPrimaryTokenPrivilege") != -1 + assert out.count(b"\n") > 20 + assert rc == 0 + +def test_windows_getsids(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"Local System") != -1 + assert out.find(b"Administrators") != -1 + assert out.find(b"Everyone") != -1 + assert out.find(b"Authenticated Users") != -1 + assert rc == 0 + +def test_windows_envars(image, volatility, python): + rc, out, err = runvol_plugin("windows.envars.Envars", image, volatility, python) + + assert out.find(b"PATH") != -1 + assert out.find(b"PROCESSOR_ARCHITECTURE") != -1 + assert out.find(b"USERNAME") != -1 + assert out.find(b"SystemRoot") != -1 + assert out.find(b"CommonProgramFiles") != -1 + assert out.count(b"\n") > 500 + assert rc == 0 + +def test_windows_callbacks(image, volatility, python): + rc, out, err = runvol_plugin("windows.callbacks.Callbacks", image, volatility, python) + + assert out.find(b"PspCreateProcessNotifyRoutine") != -1 + assert out.find(b"KeBugCheckCallbackListHead") != -1 + assert out.find(b"KeBugCheckReasonCallbackListHead") != -1 + assert out.count(b"KeBugCheckReasonCallbackListHead ") > 5 + assert rc == 0 + +# LINUX + +def test_linux_pslist(image, volatility, python): + rc, out, err = runvol_plugin("linux.pslist.PsList", image, volatility, python) + out = out.lower() + + assert ((out.find(b"init") != -1) or (out.find(b"systemd") != -1)) + assert out.find(b"watchdog") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_check_idt(image, volatility, python): + rc, out, err = runvol_plugin("linux.check_idt.Check_idt", image, volatility, python) + out = out.lower() + + assert out.count(b"__kernel__") >= 10 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_check_syscall(image, volatility, python): + rc, out, err = runvol_plugin("linux.check_syscall.Check_syscall", image, volatility, python) + out = out.lower() + + assert out.find(b"sys_close") != -1 + assert out.find(b"sys_open") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_linux_lsmod(image, volatility, python): + rc, out, err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_lsof(image, volatility, python): + rc, out, err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) + out = out.lower() + + assert out.count(b"socket:") >= 10 + assert out.count(b"\n") > 35 + assert rc == 0 + +def test_linux_proc_maps(image, volatility, python): + rc, out, err = runvol_plugin("linux.proc.Maps", image, volatility, python) + out = out.lower() + + assert out.count(b"anonymous mapping") >= 10 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_linux_tty_check(image, volatility, python): + rc, out, err = runvol_plugin("linux.tty_check.tty_check", image, volatility, python) + out = out.lower() + + assert out.find(b"__kernel__") != -1 + assert out.count(b"\n") >= 5 + assert rc == 0 + +# MAC + +def test_mac_pslist(image, volatility, python): + rc, out, err = runvol_plugin("mac.pslist.PsList", image, volatility, python) + out = out.lower() + + assert ((out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1)) + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_check_syscall(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_syscall.Check_syscall", image, volatility, python) + out = out.lower() + + assert out.find(b"chmod") != -1 + assert out.find(b"chown") != -1 + assert out.find(b"nosys") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_mac_check_sysctl(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_sysctl.Check_sysctl", image, volatility, python) + out = out.lower() + + assert out.find(b"__kernel__") != -1 + assert out.count(b"\n") > 250 + assert rc == 0 + +def test_mac_check_trap_table(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_trap_table.Check_trap_table", image, volatility, python) + out = out.lower() + + assert out.count(b"kern_invalid") >= 10 + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_ifconfig(image, volatility, python): + rc, out, err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) + out = out.lower() + + assert out.find(b"127.0.0.1") != -1 + assert out.find(b"false") != -1 + assert out.count(b"\n") > 9 + assert rc == 0 + +def test_mac_lsmod(image, volatility, python): + rc, out, err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) + out = out.lower() + + assert out.find(b"com.apple") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_lsof(image, volatility, python): + rc, out, err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_malfind(image, volatility, python): + rc, out, err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 20 + assert rc == 0 + +def test_mac_mount(image, volatility, python): + rc, out, err = runvol_plugin("mac.mount.Mount", image, volatility, python) + out = out.lower() + + assert out.find(b"/dev") != -1 + assert out.count(b"\n") > 7 + assert rc == 0 + +def test_mac_netstat(image, volatility, python): + rc, out, err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) + + assert out.find(b"TCP") != -1 + assert out.find(b"UDP") != -1 + assert out.find(b"UNIX") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_proc_maps(image, volatility, python): + rc, out, err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) + out = out.lower() + + assert out.find(b"[heap]") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_mac_psaux(image, volatility, python): + rc, out, err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) + out = out.lower() + + assert out.find(b"executable_path") != -1 + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_socket_filters(image, volatility, python): + rc, out, err = runvol_plugin("mac.socket_filters.Socket_filters", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 9 + assert rc == 0 + +def test_mac_timers(image, volatility, python): + rc, out, err = runvol_plugin("mac.timers.Timers", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 6 + assert rc == 0 + +def test_mac_trustedbsd(image, volatility, python): + rc, out, err = runvol_plugin("mac.trustedbsd.Trustedbsd", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 From f96f004b13c15b3c0334f7ca96b2c0459a8f9cf1 Mon Sep 17 00:00:00 2001 From: Frank Gomulka Date: Fri, 15 Jul 2022 18:01:48 -0500 Subject: [PATCH 111/526] @digitalisx suggested changes --- .github/workflows/test.yaml | 6 +++--- test/test_volatility.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5a3f90565..a3ecd7c7e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,5 +1,5 @@ name: Test Volatility3 -on: [push] +on: [push, pull_request] jobs: build: @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.x + - name: Set up Python 3.6 uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.6' - name: Install dependencies run: | diff --git a/test/test_volatility.py b/test/test_volatility.py index 527d86dd3..a55dffb27 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -127,10 +127,9 @@ def test_windows_dumpfiles(image, volatility, python): rc, out, err = runvol_plugin("windows.dumpfiles.DumpFiles", image, volatility, python, globalargs=["-o", path], pluginargs=["--virtaddr", addr]) for file in os.listdir(path): - fp = open(os.path.join(path, file), "rb") - if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: - failed_chksms += 1 - fp.close() + with open(os.path.join(path, file), "rb") as fp: + if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: + failed_chksms += 1 shutil.rmtree(path) From f295e5d91a6b7ecc7a1f03099d2984a98cf43eda Mon Sep 17 00:00:00 2001 From: fgomulka <60993471+fgomulka@users.noreply.github.com> Date: Sat, 16 Jul 2022 16:34:47 -0500 Subject: [PATCH 112/526] Add newline Co-authored-by: Donghyun Kim --- test/known_files.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/known_files.json b/test/known_files.json index fbc40e48b..089896714 100644 --- a/test/known_files.json +++ b/test/known_files.json @@ -16,4 +16,5 @@ "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" } } - } \ No newline at end of file + } + \ No newline at end of file From 3e748e7d488eeb96904d94039ca02d57c6368fff Mon Sep 17 00:00:00 2001 From: fgomulka <60993471+fgomulka@users.noreply.github.com> Date: Sat, 16 Jul 2022 16:35:26 -0500 Subject: [PATCH 113/526] Use more descriptive variable names Co-authored-by: Donghyun Kim --- test/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 9d3d27fc5..9057e1676 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -21,8 +21,8 @@ def pytest_generate_tests(metafunc): """Parameterize tests based on image names""" images = metafunc.config.getoption('image') - for d in metafunc.config.getoption('image_dir'): - images = images + [os.path.join(d, x) for x in os.listdir(d)] + for image_dir in metafunc.config.getoption('image_dir'): + images = images + [os.path.join(image_dir, dir) for dir in os.listdir(image_dir)] # tests with "image" parameter are run against images if 'image' in metafunc.fixturenames: From 5bc517aa42f09bb467136866d92811760a92169b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 00:15:57 +0000 Subject: [PATCH 114/526] Automagic: Use sqlite to cache identifiers --- volatility3/framework/automagic/linux.py | 35 +- volatility3/framework/automagic/mac.py | 28 +- .../framework/automagic/symbol_cache.py | 480 ++++++++++++------ .../framework/automagic/symbol_finder.py | 25 +- .../framework/configuration/requirements.py | 12 +- volatility3/framework/constants/__init__.py | 7 +- volatility3/framework/interfaces/automagic.py | 9 +- volatility3/framework/plugins/isfinfo.py | 39 +- volatility3/framework/symbols/intermed.py | 3 +- .../framework/symbols/windows/pdbutil.py | 58 +-- 10 files changed, 417 insertions(+), 279 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f1d6c91e4..2c152996d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -5,8 +5,9 @@ import logging from typing import Optional, Tuple, Type -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder +from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import linux @@ -23,6 +24,13 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify linux within this layer.""" + # Version check the SQlite cache + required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + vollog.info( + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + return None + # Bail out by default unless we can stack properly layer = context.layers[layer_name] join = interfaces.configuration.path_join @@ -32,7 +40,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - linux_banners = LinuxBannerCache.load_banners() + linux_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + operating_system = 'linux') # If we have no banners, don't bother scanning if not linux_banners: vollog.info("No Linux banners found - if this is a linux plugin, please check your symbol files location") @@ -43,15 +52,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = linux_banners.get(banner, None) - if symbol_files: - if len(symbol_files) > 1: - using = "*" - vollog.warning(f"Multiple symbol files identified (using {using}):") - for symbol_file in symbol_files: - vollog.warning(f" {using} {symbol_file}") - using = " " - isf_path = symbol_files[0] + isf_path = linux_banners.get(banner, None) + if isf_path: table_name = context.symbol_space.free_table_name('LintelStacker') table = linux.LinuxKernelIntermedSymbols(context, 'temporary.' + table_name, @@ -147,20 +149,11 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): return addr - 0xc0000000 -class LinuxBannerCache(symbol_cache.SymbolBannerCache): - """Caches the banners found in the Linux symbol files.""" - - os = "linux" - symbol_name = "linux_banner" - banner_path = constants.LINUX_BANNERS_PATH - exclusion_list = ['mac', 'windows'] - - class LinuxSymbolFinder(symbol_finder.SymbolFinder): """Linux symbol loader based on uname signature strings.""" banner_config_key = "kernel_banner" - banner_cache = LinuxBannerCache + operating_system = 'linux' symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] exclusion_list = ['mac', 'windows'] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index c37aef463..246462878 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -6,8 +6,9 @@ import logging import struct from typing import Optional -from volatility3.framework import interfaces, constants, layers, exceptions +from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.automagic import symbol_cache, symbol_finder +from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import mac @@ -24,6 +25,13 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify mac within this layer.""" + # Version check the SQlite cache + required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + vollog.info( + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + return None + # Bail out by default unless we can stack properly layer = context.layers[layer_name] new_layer = None @@ -34,7 +42,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - mac_banners = MacBannerCache.load_banners() + mac_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + operating_system = 'mac') # If we have no banners, don't bother scanning if not mac_banners: vollog.info("No Mac banners found - if this is a mac plugin, please check your symbol files location") @@ -46,9 +55,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = mac_banners.get(banner, None) - if symbol_files: - isf_path = symbol_files[0] + isf_path = mac_banners.get(banner, None) + if isf_path: table_name = context.symbol_space.free_table_name('MacintelStacker') table = mac.MacKernelIntermedSymbols(context = context, config_path = join('temporary', table_name), @@ -197,19 +205,11 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): yield offset, banner -class MacBannerCache(symbol_cache.SymbolBannerCache): - """Caches the banners found in the Mac symbol files.""" - os = "mac" - symbol_name = "version" - banner_path = constants.MAC_BANNERS_PATH - exclusion_list = ['windows', 'linux'] - - class MacSymbolFinder(symbol_finder.SymbolFinder): """Mac symbol loader based on uname signature strings.""" banner_config_key = 'kernel_banner' - banner_cache = MacBannerCache + operating_system = 'mac' find_aslr = MacIntelStacker.find_aslr symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols" exclusion_list = ['windows', 'linux'] diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 7b6adf9b4..fe717b8be 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -2,18 +2,20 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import base64 -import gc import json import logging import os -import pickle +import sqlite3 import urllib import urllib.parse import urllib.request -import zipfile -from typing import Dict, List, Optional +from abc import abstractmethod +from typing import Dict, Generator, List, Optional -from volatility3.framework import constants, exceptions, interfaces +import volatility3.framework +import volatility3.schemas +from volatility3.framework import constants, interfaces +from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources from volatility3.framework.symbols import intermed @@ -22,164 +24,324 @@ vollog = logging.getLogger(__name__) BannersType = Dict[bytes, List[str]] -class SymbolBannerCache(interfaces.automagic.AutomagicInterface): - """Runs through all symbols tables and caches their banners.""" +### Identifiers - # Since this is necessary for ConstructionMagic, we set a lower priority - # The user would run it eventually either way, but running it first means it can be used that run +class IdentifierProcessor: + operating_system = None + + def __init__(self): + pass + + @classmethod + @abstractmethod + def get_identifier(cls, json) -> Optional[bytes]: + """Method to extract the identifier from a particular operating system's JSON + + Returns: + identifier is valid or None if not found + """ + raise NotImplemented("This base class has no get_identifier method defined") + + +class WindowsIdentifier(IdentifierProcessor): + operating_system = 'windows' + separator = '|' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + """Returns the identifier for the file if one can be found""" + windows_metadata = json.get('metadata', {}).get('windows', {}).get('pdb', {}) + if windows_metadata: + guid = windows_metadata.get('GUID', None) + age = windows_metadata.get('age', None) + database = windows_metadata.get('database', None) + if guid and age and database: + return cls.generate(database, guid, age) + return None + + @classmethod + def generate(cls, pdb_name: str, guid: str, age: int) -> bytes: + return bytes(cls.separator.join([pdb_name, guid.upper(), str(age)]), 'latin-1') + + +class MacIdentifier(IdentifierProcessor): + operating_system = 'mac' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + mac_banner = json.get('symbols', {}).get('version', {}).get('constant_data', None) + if mac_banner: + return base64.b64decode(mac_banner) + return None + + +class LinuxIdentifier(IdentifierProcessor): + operating_system = 'linux' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + linux_banner = json.get('symbols', {}).get('linux_banner', {}).get('constant_data', None) + if linux_banner: + return base64.b64decode(linux_banner) + return None + + +### CacheManagers + +class CacheManagerInterface(interfaces.configuration.VersionableInterface): + def __init__(self, filename: str): + super().__init__() + self._filename = filename + self._classifiers = {} + for subclazz in volatility3.framework.class_subclasses(IdentifierProcessor): + self._classifiers[subclazz.operating_system] = subclazz + + def add_identifier(self, location: str, operating_system: str, identifier: str): + """Adds an identifier to the store""" + pass + + def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + """Returns the location of the symbol file given the identifier + + Args: + identifier: string that uniquely identifies a particular symbolt table + operating_system: optional string to restrict identifiers to just those for a particular operating system + + Returns: + The location of the symbols file that matches the identifier + """ + pass + + def get_local_locations(self) -> List[str]: + """Returns a list of all the local locations""" + pass + + def update(self): + """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. + This also updates remote locations based on a cache timeout. + + """ + pass + + def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ + Dict[bytes, str]: + """Returns a dictionary of identifiers and locations + + Args: + operating_system: If set, limits responses to a specific operating system + local_only: Returns only local locations + + Returns: + A dictionary of identifiers mapped to a location + """ + pass + + def get_identifier(self, location: str) -> Optional[bytes]: + """Returns an identifier based on a specific location or None""" + pass + + def get_identifiers(self, operating_system: Optional[str]): + """Returns all identifiers for a particular operating system""" + pass + + +class SqliteCache(CacheManagerInterface): + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + def __init__(self, filename: str): + super().__init__(filename) + try: + self._database = self._connect_storage(filename) + except sqlite3.DatabaseError: + os.unlink(filename) + self._database = self._connect_storage(filename) + + def _connect_storage(self, path: str): + database = sqlite3.connect(path, isolation_level = None) + database.row_factory = sqlite3.Row + database.cursor().execute( + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + return database + + def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + """Returns the location of the symbol file given the identifier. + If multiple locations exist for an identifier, the last found is returned + + Args: + identifier: string that uniquely identifies a particular symbolt table + operating_system: optional string to restrict identifiers to just those for a particular operating system + + Returns: + The location of the symbols file that matches the identifier or None + """ + statement = 'SELECT location FROM cache WHERE identifier = ?' + parameters = (identifier,) + if operating_system is not None: + statement = 'SELECT location FROM cache WHERE identifier = ? AND operating_system = ?' + parameters = (identifier, operating_system) + results = self._database.cursor().execute(statement, parameters).fetchall() + result = None + for row in results: + result = row['location'] + return result + + def get_local_locations(self) -> Generator[str, None, None]: + result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = True').fetchall() + for row in result: + yield row['location'] + + def is_url_local(self, url: str) -> bool: + """Determines whether an url is local or not""" + parsed = urllib.parse.urlparse(url) + if parsed.scheme in ['file', 'jar']: + return True + + def get_identifier(self, location: str) -> Optional[bytes]: + results = self._database.cursor().execute('SELECT identifier FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['identifier'] + return None + + def update(self, progress_callback = None): + """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. + This also updates remote locations based on a cache timeout. + + """ + on_disk_locations = set([filename for filename in intermed.IntermediateSymbolTable.file_symbol_url('')]) + cached_locations = set(self.get_local_locations()) + + new_locations = on_disk_locations.difference(cached_locations) + missing_locations = cached_locations.difference(on_disk_locations) + + cache_update = set() + files_to_timestamp = on_disk_locations.intersection(cached_locations) + if files_to_timestamp: + result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " + "AND cached < date('now', '-3 days');") + for row in result: + if row['location'] in files_to_timestamp: + cache_update.add(row['location']) + + idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + + counter = 0 + files_to_process = new_locations.union(cache_update) + number_files_to_process = len(files_to_process) + for location in files_to_process: + # Open location + counter += 1 + progress_callback(counter * 100 / number_files_to_process, + "Updating caches for {number_files_to_process} files...") + try: + with resources.ResourceAccessor().open(location) as fp: + json_obj = json.load(fp) + identifier = None + for idextractor in idextractors: + identifier = idextractor.get_identifier(json_obj) + operating_system = idextractor.operating_system + if identifier is not None: + break + if identifier is not None: + # We don't try to validate schemas here, we do that on first use + # Store in database + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") + else: + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + None, + None, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") + except Exception as excp: + vollog.log(constants.LOGLEVEL_VVVV, excp) + + if not constants.OFFLINE and constants.REMOTE_ISF_URL: + remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) + for operating_system in ['mac', 'linux', 'windows']: + identifiers = remote_identifiers.process({}, operating_system = operating_system) + for identifier in identifiers: + for location in identifiers[identifier]: + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now')", + (location, identifier, operating_system, False) + ) + + if missing_locations: + self._database.cursor().execute( + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + + def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ + Dict[bytes, str]: + output = {} + additions = [] + statement = 'SELECT location, identifier FROM cache' + if local_only: + additions.append('local = True') + if operating_system: + additions.append(f"operating_system = '{operating_system}'") + if additions: + statement += f" WHERE {' AND '.join(additions)}" + results = self._database.cursor().execute(statement) + for row in results: + if row['identifier'] in output and row['identifier'] and row['location']: + vollog.debug( + f"Duplicate entry for identifier {row['identifier']}: {row['location']} and {output[row['identifier']]}") + output[row['identifier']] = row['location'] + return output + + def get_identifiers(self, operating_system: Optional[str]): + if operating_system: + results = self._database.cursor().execute('SELECT identifier FROM cache WHERE operating_system = ?', + (operating_system,)).fetchall() + else: + results = self._database.cursor().execute('SELECT identifier FROM cache').fetchall() + output = [] + for row in results: + output.append(row['identifier']) + return output + + +### Automagic + +class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): + """Runs through all symbol tables and caches their identifiers""" priority = 0 - os: Optional[str] = None - symbol_name: str = "banner_name" - banner_path: Optional[str] = None - - @classmethod - def load_banners(cls) -> BannersType: - if not cls.banner_path: - raise ValueError("Banner_path not appropriately set") - banners: BannersType = {} - if os.path.exists(cls.banner_path): - with open(cls.banner_path, "rb") as f: - # We use pickle over JSON because we're dealing with bytes objects - banners.update(pickle.load(f)) - - # Remove possibilities that can't exist locally. - remove_banners = [] - for banner in banners: - for path in banners[banner]: - url = urllib.parse.urlparse(path) - if url.scheme == 'file' and not os.path.exists(urllib.request.url2pathname(url.path)): - vollog.log( - constants.LOGLEVEL_VV, "Removing cached path {} for banner {}: file does not exist".format( - path, str(banner or b'', 'latin-1'))) - banners[banner].remove(path) - # This is probably excessive, but it's here if we need it - if url.scheme == 'jar': - zip_file, zip_path = url.path.split("!") - zip_file = urllib.parse.urlparse(zip_file).path - if ((not os.path.exists(zip_file)) or (zip_path not in zipfile.ZipFile(zip_file).namelist())): - vollog.log(constants.LOGLEVEL_VV, - "Removing cached path {} for banner {}: file does not exist".format(path, banner)) - banners[banner].remove(path) - - if not banners[banner]: - remove_banners.append(banner) - for remove_banner in remove_banners: - del banners[remove_banner] - return banners - - @classmethod - def save_banners(cls, banners): - - with open(cls.banner_path, "wb") as f: - pickle.dump(banners, f) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cache = SqliteCache(constants.IDENTIFIERS_PATH) def __call__(self, context, config_path, configurable, progress_callback = None): """Runs the automagic over the configurable.""" - - # Bomb out if we're just the generic interface - if self.os is None: - return - - # We only need to be called once, so no recursion necessary - banners = self.load_banners() - - cacheables = self.find_new_banner_files(banners, self.os) - - new_banners = self.read_new_banners(context, config_path, cacheables, self.symbol_name, self.os, - progress_callback) - - # Add in any new banners to the existing list - for new_banner in new_banners: - banner_list = banners.get(new_banner, []) - banners[new_banner] = list(set(banner_list + new_banners[new_banner])) - - # Do remote banners *after* the JSON loading, so that it doesn't pull down all the remote JSON - self.remote_banners(banners, self.os) - - # Rewrite the cached banners each run, since writing is faster than the banner_cache validation portion - self.save_banners(banners) - - if progress_callback is not None: - progress_callback(100, f"Built {self.os} caches") + self._cache.update(progress_callback) @classmethod - def read_new_banners(cls, context: interfaces.context.ContextInterface, config_path: str, new_urls: List[str], - symbol_name: str, operating_system: str = None, - progress_callback = None) -> Optional[Dict[bytes, List[str]]]: - """Reads the any new banners for the OS in question""" - if operating_system is None: - return None - - banners = {} - - total = len(new_urls) - if total > 0: - vollog.info(f"Building {operating_system} caches...") - for current in range(total): - if progress_callback is not None: - progress_callback(current * 100 / total, f"Building {operating_system} caches") - isf_url = new_urls[current] - - isf = None - try: - # Loading the symbol table will be very slow until it's been validated - isf = intermed.IntermediateSymbolTable(context, config_path, "temp", isf_url, validate = False) - - # We should store the banner against the filename - # We don't bother with the hash (it'll likely take too long to validate) - # but we should check at least that the banner matches on load. - banner = isf.get_symbol(symbol_name).constant_data - vollog.log(constants.LOGLEVEL_VV, f"Caching banner {banner} for file {isf_url}") - - bannerlist = banners.get(banner, []) - bannerlist.append(isf_url) - banners[banner] = bannerlist - except exceptions.SymbolError: - pass - except json.JSONDecodeError: - vollog.log(constants.LOGLEVEL_VV, f"Caching file {isf_url} failed due to JSON error") - finally: - # Get rid of the loaded file, in case it sits in memory - if isf: - del isf - gc.collect() - return banners - - @classmethod - def find_new_banner_files(cls, banners: Dict[bytes, List[str]], operating_system: str) -> List[str]: - """Gathers all files and remove existing banners""" - cacheables = list(intermed.IntermediateSymbolTable.file_symbol_url(operating_system)) - for banner in banners: - for json_file in banners[banner]: - if json_file in cacheables: - cacheables.remove(json_file) - return cacheables - - @classmethod - def remote_banners(cls, banners: Dict[bytes, List[str]], operating_system = None, banner_location = None): - """Adds remote URLs to the banner list""" - if operating_system is None: - return None - - if banner_location is None: - banner_location = constants.REMOTE_ISF_URL - - if not constants.OFFLINE and banner_location is not None: - try: - rbf = RemoteBannerFormat(banner_location) - rbf.process(banners, operating_system) - except urllib.error.URLError: - vollog.debug(f"Unable to download remote banner list from {banner_location}") + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + """Returns a list of RequirementInterface objects required by this + object.""" + return [requirements.VersionRequirement(name = 'SQLiteCache', component = SqliteCache, version = (1, 0, 0))] -class RemoteBannerFormat: +class RemoteIdentifierFormat: def __init__(self, location: str): self._location = location with resources.ResourceAccessor().open(url = location) as fp: self._data = json.load(fp) if not self._verify(): - raise ValueError("Unsupported version for remote banner list format") + raise ValueError("Unsupported version for remote identifier list format") def _verify(self) -> bool: version = self._data.get('version', 0) @@ -188,23 +350,23 @@ class RemoteBannerFormat: return True return False - def process(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): - raise ValueError("Banner List version not verified") + def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]): + raise ValueError("Identifier List version not verified") - def process_v1(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): + def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]): if operating_system in self._data: - for banner in self._data[operating_system]: - binary_banner = base64.b64decode(banner) - file_list = banners.get(binary_banner, []) - for value in self._data[operating_system][banner]: + for identifier in self._data[operating_system]: + binary_identifier = base64.b64decode(identifier) + file_list = identifiers.get(binary_identifier, []) + for value in self._data[operating_system][identifier]: if value not in file_list: file_list = file_list + [value] - banners[binary_banner] = file_list + identifiers[binary_identifier] = file_list if 'additional' in self._data: for location in self._data['additional']: try: - subrbf = RemoteBannerFormat(location) - subrbf.process(banners, operating_system) + subrbf = RemoteIdentifierFormat(location) + subrbf.process(identifiers, operating_system) except IOError: vollog.debug(f"Remote file not found: {location}") - return banners + return identifiers diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 143abd02e..610ed0e18 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -3,9 +3,9 @@ # import logging -from typing import Any, Iterable, List, Tuple, Type, Optional, Callable +from typing import Any, Callable, Iterable, List, Optional, Tuple -from volatility3.framework import interfaces, constants, layers +from volatility3.framework import constants, interfaces, layers from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners @@ -18,7 +18,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): priority = 40 banner_config_key: str = "banner" - banner_cache: Optional[Type[symbol_cache.SymbolBannerCache]] = None + operating_system: Optional[str] = None symbol_class: Optional[str] = None find_aslr: Optional[Callable] = None @@ -27,14 +27,21 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): self._requirements: List[Tuple[str, interfaces.configuration.RequirementInterface]] = [] self._banners: symbol_cache.BannersType = {} + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.VersionRequirement(name = 'SQLiteCache', + component = symbol_cache.SqliteCache, + version = (1, 0, 0)) + ] + @property def banners(self) -> symbol_cache.BannersType: """Creates a cached copy of the results, but only it's been requested.""" if not self._banners: - if not self.banner_cache: - raise RuntimeError(f"Cache has not been properly defined for {self.__class__.__name__}") - self._banners = self.banner_cache.load_banners() + cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system) return self._banners def __call__(self, @@ -103,8 +110,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): vollog.debug(f"Identified banner: {repr(banner)}") symbol_files = self.banners.get(banner, None) if symbol_files: - isf_path = symbol_files[0] - vollog.debug(f"Using symbol library: {symbol_files[0]}") + isf_path = symbol_files + vollog.debug(f"Using symbol library: {symbol_files}") clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join @@ -117,7 +124,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): break else: if symbol_files: - vollog.debug(f"Symbol library path not found: {symbol_files[0]}") + vollog.debug(f"Symbol library path not found: {symbol_files}") # print("Kernel", banner, hex(banner_offset)) else: vollog.debug("No existing banners found") diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 4edc6d17c..b31c4767f 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -408,13 +408,19 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type config_path = interfaces.configuration.path_join(config_path, self.name) - if len(self._version) > 0 and self._component.version[0] != self._version[0]: - return {config_path: self} - if len(self._version) > 1 and self._component.version[1] < self._version[1]: + if not self.matches_required(self._version, self._component.version): return {config_path: self} context.config[interfaces.configuration.path_join(config_path, self.name)] = True return {} + @classmethod + def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]): + if len(required) > 0 and version[0] != required[0]: + return False + if len(required) > 1 and version[1] < required[1]: + return False + return True + class PluginRequirement(VersionRequirement): diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index f08819f29..322e574e1 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -68,10 +68,13 @@ if sys.platform == 'win32': os.makedirs(CACHE_PATH, exist_ok = True) LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") -""""Default location to record information about available linux banners""" +"""Default location to record information about available linux banners""" MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") -""""Default location to record information about available mac banners""" +"""Default location to record information about available mac banners""" + +IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") +"""Default location to record information about available identifiers""" BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index c96c9bdbe..713f91da0 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -9,9 +9,9 @@ that a user has not filled. """ import logging from abc import ABCMeta -from typing import Any, List, Optional, Tuple, Union, Type +from typing import Any, List, Optional, Tuple, Type, Union -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -47,9 +47,10 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla super().__init__(context, config_path) for requirement in self.get_requirements(): if not isinstance(requirement, (interfaces.configuration.SimpleTypeRequirement, - requirements.ChoiceRequirement, requirements.ListRequirement)): + requirements.ChoiceRequirement, requirements.ListRequirement, + requirements.VersionRequirement)): raise TypeError( - "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement or ListRequirement") + "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement, ListRequirement or VersionRequirement") def __call__(self, context: interfaces.context.ContextInterface, diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 575f25426..b2960733d 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -1,17 +1,16 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import base64 import json import logging import os import pathlib import zipfile -from typing import List, Type, Any, Generator +from typing import Generator, List from volatility3 import schemas, symbols -from volatility3.framework import interfaces, renderers, constants -from volatility3.framework.automagic import mac, linux, symbol_cache +from volatility3.framework import constants, interfaces, renderers +from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import resources @@ -23,7 +22,7 @@ class IsfInfo(plugins.PluginInterface): """Determines information about the currently available ISF files, or a specific one""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -39,7 +38,10 @@ class IsfInfo(plugins.PluginInterface): requirements.BooleanRequirement(name = 'validate', description = 'Validate against schema if possible', default = False, - optional = True) + optional = True), + requirements.VersionRequirement(name = 'SQLiteCache', + component = symbol_cache.SqliteCache, + version = (1, 0, 0)) ] @classmethod @@ -62,14 +64,6 @@ class IsfInfo(plugins.PluginInterface): if filename.endswith(extension): yield pathlib.Path(base_name).as_uri() - def _get_banner(self, clazz: Type[symbol_cache.SymbolBannerCache], data: Any) -> str: - """Gets a banner from an ISF file""" - banner_symbol = data.get('symbols', {}).get(clazz.symbol_name, {}).get('constant_data', - renderers.NotAvailableValue()) - if not isinstance(banner_symbol, interfaces.renderers.BaseAbsentValue): - banner_symbol = str(base64.b64decode(banner_symbol), encoding = 'latin-1') - return banner_symbol - def _generator(self): if self.config.get('isf', None) is not None: file_list = [self.config['isf']] @@ -101,7 +95,6 @@ class IsfInfo(plugins.PluginInterface): # Process the filtered list for entry in filtered_list: num_types = num_enums = num_bases = num_symbols = 0 - windows_info = linux_banner = mac_banner = renderers.NotAvailableValue() valid = "Unknown" with resources.ResourceAccessor().open(url = entry) as fp: try: @@ -111,20 +104,20 @@ class IsfInfo(plugins.PluginInterface): num_enums = len(data.get('enums', [])) num_bases = len(data.get('base_types', [])) - linux_banner = self._get_banner(linux.LinuxBannerCache, data) - mac_banner = self._get_banner(mac.MacBannerCache, data) - if not linux_banner and not mac_banner: - windows_info = os.path.splitext(os.path.basename(entry))[0] + identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifier = identifier_cache.get_identifier(location = entry) + if identifier: + identifier = identifier.decode('utf-8', errors = 'replace') + else: + identifier = renderers.NotAvailableValue() valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {entry}") - yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, windows_info, linux_banner, - mac_banner)) + yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) # Try to open the file, load it as JSON, read the data from it def run(self): return renderers.TreeGrid([("URI", str), ("Valid", str), ("Number of base_types", int), ("Number of types", int), ("Number of symbols", int), - ("Number of enums", int), ("Windows info", str), ("Linux banner", str), - ("Mac banner", str)], self._generator()) + ("Number of enums", int), ("Identifying infomration", str)], self._generator()) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index a6a7a0fae..1fceb1bcc 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -202,8 +202,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): pass # Finally try looking in zip files - zip_path = os.path.join(path, sub_path + ".zip") - if os.path.exists(zip_path): + for zip_path in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + '.zip'): # We have a zipfile, so run through it and look for sub files that match the filename with zipfile.ZipFile(zip_path) as zfile: for name in zfile.namelist(): diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 41037d464..af3741bbe 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -14,6 +14,8 @@ from urllib import parse, request from volatility3 import symbols from volatility3.framework import constants, contexts, exceptions, interfaces +from volatility3.framework.automagic import symbol_cache +from volatility3.framework.configuration import requirements from volatility3.framework.configuration.requirements import SymbolTableRequirement from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbconv @@ -74,9 +76,15 @@ class PDBUtility(interfaces.configuration.VersionableInterface): isf_path = None # Take the first result of search for the intermediate file - for value in intermed.IntermediateSymbolTable.file_symbol_url("windows", filter_string): + if not requirements.VersionRequirement.matches_required((1, 0, 0), symbol_cache.SqliteCache.version): + vollog.debug(f"Required version of SQLiteCache not found") + return None + + value = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).find_location( + symbol_cache.WindowsIdentifier.generate(pdb_name.strip('\x00'), guid.upper(), age), 'windows') + + if value: isf_path = value - break else: # If none are found, attempt to download the pdb, convert it and try again cls.download_pdb_isf(context, guid.upper(), age, pdb_name, progress_callback) @@ -336,46 +344,12 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - module_name = guid["pdb_name"].strip('.pdb') - - symbol_table_name = cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) - - new_module_name = None - if create_module: - new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], - symbol_table_name = symbol_table_name) - new_module_name = new_module.name - - return new_module_name, symbol_table_name - - @classmethod - def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, - pdb_name: str, module_offset: int = None, module_size: int = None) -> str: - """Creates a module in the specified layer_name based on a pdb name. - - Searches the memory section of the loaded module for its PDB GUID - and loads the associated symbol table into the symbol space. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - config_path: The config path where to find symbol files - layer_name: The name of the layer on which to operate - module_offset: This memory dump's module image offset - module_size: The size of the module for this dump - - Returns: - The name of the constructed and loaded symbol table - """ - - module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, - module_size, create_module = True) - - return module_name + return cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) class PdbSignatureScanner(interfaces.layers.ScannerInterface): From 2729d25d89576b3d31785c1326671eb86455495e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 00:40:01 +0000 Subject: [PATCH 115/526] Automagic: speed up caching by db commit when necessary --- .../framework/automagic/symbol_cache.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index fe717b8be..4b27b8e0f 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -10,7 +10,7 @@ import urllib import urllib.parse import urllib.request from abc import abstractmethod -from typing import Dict, Generator, List, Optional +from typing import Dict, Generator, List, Optional, Tuple import volatility3.framework import volatility3.schemas @@ -158,10 +158,11 @@ class SqliteCache(CacheManagerInterface): self._database = self._connect_storage(filename) def _connect_storage(self, path: str): - database = sqlite3.connect(path, isolation_level = None) + database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + database.commit() return database def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: @@ -229,6 +230,7 @@ class SqliteCache(CacheManagerInterface): counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) + cursor = self._database.cursor() for location in files_to_process: # Open location counter += 1 @@ -246,7 +248,7 @@ class SqliteCache(CacheManagerInterface): if identifier is not None: # We don't try to validate schemas here, we do that on first use # Store in database - self._database.cursor().execute( + cursor.execute( "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", ( location, @@ -256,7 +258,7 @@ class SqliteCache(CacheManagerInterface): )) vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") else: - self._database.cursor().execute( + cursor.execute( "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", ( location, @@ -267,21 +269,27 @@ class SqliteCache(CacheManagerInterface): vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") except Exception as excp: vollog.log(constants.LOGLEVEL_VVVV, excp) + self._database.commit() if not constants.OFFLINE and constants.REMOTE_ISF_URL: + progress_callback(0, 'Reading remote ISF list') remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) + progress_callback(50, 'Reading remote ISF list') + cursor = self._database.cursor() for operating_system in ['mac', 'linux', 'windows']: identifiers = remote_identifiers.process({}, operating_system = operating_system) - for identifier in identifiers: - for location in identifiers[identifier]: - self._database.cursor().execute( - "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now')", - (location, identifier, operating_system, False) - ) + for identifier, location in identifiers: + cursor.execute( + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + (location, identifier, operating_system, False) + ) + progress_callback(100, 'Reading remote ISF list') + self._database.commit() if missing_locations: self._database.cursor().execute( f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ Dict[bytes, str]: @@ -350,23 +358,23 @@ class RemoteIdentifierFormat: return True return False - def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]): + def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]) -> Generator[ + Tuple[bytes, str], None, None]: raise ValueError("Identifier List version not verified") - def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]): + def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]) -> Generator[ + Tuple[bytes, str], None, None]: if operating_system in self._data: for identifier in self._data[operating_system]: binary_identifier = base64.b64decode(identifier) file_list = identifiers.get(binary_identifier, []) for value in self._data[operating_system][identifier]: - if value not in file_list: - file_list = file_list + [value] - identifiers[binary_identifier] = file_list + yield binary_identifier, value if 'additional' in self._data: for location in self._data['additional']: try: subrbf = RemoteIdentifierFormat(location) - subrbf.process(identifiers, operating_system) + yield from subrbf.process(identifiers, operating_system) except IOError: vollog.debug(f"Remote file not found: {location}") return identifiers From 57a202ae1d69de5968a6a49e9bc199724d364152 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 01:04:32 +0000 Subject: [PATCH 116/526] Automagic: Use cache delay for remote locations --- volatility3/framework/automagic/symbol_cache.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 4b27b8e0f..3c7049986 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -149,6 +149,8 @@ class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) + cache_period = '-3 days' + def __init__(self, filename: str): super().__init__(filename) try: @@ -220,13 +222,15 @@ class SqliteCache(CacheManagerInterface): files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " - "AND cached < date('now', '-3 days');") + f"AND cached < date('now', {self.cache_period});") for row in result: if row['location'] in files_to_timestamp: cache_update.add(row['location']) idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + # New or not recently updated + counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) @@ -271,11 +275,15 @@ class SqliteCache(CacheManagerInterface): vollog.log(constants.LOGLEVEL_VVVV, excp) self._database.commit() + # Remote Entries + if not constants.OFFLINE and constants.REMOTE_ISF_URL: progress_callback(0, 'Reading remote ISF list') + cursor = self._database.cursor() + cursor.execute( + f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') - cursor = self._database.cursor() for operating_system in ['mac', 'linux', 'windows']: identifiers = remote_identifiers.process({}, operating_system = operating_system) for identifier, location in identifiers: @@ -286,6 +294,8 @@ class SqliteCache(CacheManagerInterface): progress_callback(100, 'Reading remote ISF list') self._database.commit() + # Missing entries + if missing_locations: self._database.cursor().execute( f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) From fe466386406556a17ba2f474558257e4bb4e8457 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 01:36:17 +0000 Subject: [PATCH 117/526] Automagic: Update to use more recent OS categories --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 3c7049986..8bbedf3e8 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -284,7 +284,7 @@ class SqliteCache(CacheManagerInterface): f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') - for operating_system in ['mac', 'linux', 'windows']: + for operating_system in constants.OS_CATEGORIES: identifiers = remote_identifiers.process({}, operating_system = operating_system) for identifier, location in identifiers: cursor.execute( From 371267f38a61f03007bde4f880b9c45a4b4c2e41 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 21:50:08 +0000 Subject: [PATCH 118/526] Automagic: Ensure partial caching survives --- .../framework/automagic/symbol_cache.py | 80 ++++++++++--------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 8bbedf3e8..54ee13ca2 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -222,7 +222,7 @@ class SqliteCache(CacheManagerInterface): files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " - f"AND cached < date('now', {self.cache_period});") + f"AND cached < date('now', '{self.cache_period}');") for row in result: if row['location'] in files_to_timestamp: cache_update.add(row['location']) @@ -235,45 +235,47 @@ class SqliteCache(CacheManagerInterface): files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) cursor = self._database.cursor() - for location in files_to_process: - # Open location - counter += 1 - progress_callback(counter * 100 / number_files_to_process, - "Updating caches for {number_files_to_process} files...") - try: - with resources.ResourceAccessor().open(location) as fp: - json_obj = json.load(fp) - identifier = None - for idextractor in idextractors: - identifier = idextractor.get_identifier(json_obj) - operating_system = idextractor.operating_system + try: + for location in files_to_process: + # Open location + counter += 1 + progress_callback(counter * 100 / number_files_to_process, + f"Updating caches for {number_files_to_process} files...") + try: + with resources.ResourceAccessor().open(location) as fp: + json_obj = json.load(fp) + identifier = None + for idextractor in idextractors: + identifier = idextractor.get_identifier(json_obj) + operating_system = idextractor.operating_system + if identifier is not None: + break if identifier is not None: - break - if identifier is not None: - # We don't try to validate schemas here, we do that on first use - # Store in database - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - identifier, - operating_system, - self.is_url_local(location) - )) - vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") - else: - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - None, - None, - self.is_url_local(location) - )) - vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") - except Exception as excp: - vollog.log(constants.LOGLEVEL_VVVV, excp) - self._database.commit() + # We don't try to validate schemas here, we do that on first use + # Store in database + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") + else: + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + None, + None, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") + except Exception as excp: + vollog.log(constants.LOGLEVEL_VVVV, excp) + finally: + self._database.commit() # Remote Entries From d16861b5925a473c0bf36a0949bc052321197399 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 22:15:35 +0000 Subject: [PATCH 119/526] Documentation: Update documentation for isf caching feature --- doc/source/symbol-tables.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 4dea6077d..d41e8797a 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -12,20 +12,20 @@ Volatility will automatically decompress them on use. It will also cache their under the user's home directory, in :file:`.cache/volatility3`, along with other useful data. The cache directory currently cannot be altered. -Symbol table JSON files live, by default, under the :file:`volatility3/symbols`, underneath an operating system directory -(currently one of :file:`windows`, :file:`mac` or :file:`linux`). The symbols directory is configurable within the framework and can -usually be set within the user interface. +Symbol table JSON files live, by default, under the :file:`volatility3/symbols` directory. The symbols directory is +configurable within the framework and can usually be set within the user interface. These files can also be compressed into ZIP files, which Volatility will process in order to locate symbol files. -The ZIP file must be named after the appropriate operating system (such as `linux.zip`, `mac.zip` or `windows.zip`). -Inside the ZIP file, the directory structure should match the uncompressed operating system directory. + +Volatility maintains a cache mapping the appropriate identifier for each symbol file against its filename. This cache +is update by automagic called as part of the standard automagic that's run each time a plugin is run. Windows symbol tables --------------------- For Windows systems, Volatility accepts a string made up of the GUID and Age of the required PDB file. It then -searches all files under the configured symbol directories under the windows subdirectory. Any that match the filename -pattern of :file:`/-.json` (or any compressed variant) will be used. If such a symbol table cannot be found, then +searches all files under the configured symbol directories under the windows subdirectory. Any that contain metadata +which matches the pdb name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then the associated PDB file will be downloaded from Microsoft's Symbol Server and converted into the appropriate JSON format, and will be saved in the correct location. @@ -41,11 +41,10 @@ or a virtual environment. Mac/Linux symbol tables ----------------------- -For Mac/Linux systems, both use the same mechanism for identification. JSON files live under the symbol directories, -under either the :file:`linux` or :file:`mac` directories. The generated files contain an identifying string (the operating system +For Mac/Linux systems, both use the same mechanism for identification. The generated files contain an identifying string (the operating system banner), which Volatility's automagic can detect. Volatility caches the mapping between the strings and the symbol tables they come from, meaning the precise file names don't matter and can be organized under any necessary hierarchy -under the operating system directory. +under the symbols directory. Linux and Mac symbol tables can be generated from a DWARF file using a tool called `dwarf2json `_. Currently a kernel with debugging symbols is the only suitable means for recovering all the information required by From 2d64deb18ec0b341a40f416429da3e8b0d1ddb44 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 22:52:04 +0000 Subject: [PATCH 120/526] Plugins: Update isfinfo to use the cache unless --live --- .../framework/automagic/symbol_cache.py | 96 +++++++++++++++---- volatility3/framework/constants/__init__.py | 3 + volatility3/framework/plugins/isfinfo.py | 56 ++++++----- 3 files changed, 112 insertions(+), 43 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 54ee13ca2..c09904713 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -104,7 +104,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns the location of the symbol file given the identifier Args: - identifier: string that uniquely identifies a particular symbolt table + identifier: string that uniquely identifies a particular symbol table operating_system: optional string to restrict identifiers to just those for a particular operating system Returns: @@ -144,6 +144,18 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns all identifiers for a particular operating system""" pass + def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + """Returns ISF statistics based on the location + + Returns: + A tuple of base_types, types, enums, symbols, or None is location not found""" + + def get_verified(self, location: str) -> bool: + """Returns whether a location ISF has been verified against its schema""" + + def set_verified(self, location: str, state: bool = True) -> None: + """Sets the verified state of a location based on whether it has been successfully verified against its schema""" + class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) @@ -163,7 +175,23 @@ class SqliteCache(CacheManagerInterface): database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( - 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCEMA_VERSION})') + schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() + if not schema_version: + database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCEMA_VERSION})') + elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCEMA_VERSION: + # All good, so pass and move on + pass + else: + vollog.info(f"Previous cache schema version found: {schema_version['schema_version']}") + # TODO: Implement code if the schema changes + # Current this should never happen so we start over again + database.close() + os.unlink(path) + return self._connect_storage(path) + database.cursor().execute( + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, verified BOOL DEFAULT False,' + 'stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)') database.commit() return database @@ -207,6 +235,25 @@ class SqliteCache(CacheManagerInterface): return row['identifier'] return None + def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + results = self._database.cursor().execute( + 'SELECT stats_base_types, stats_types, stats_enums, stats_symbols FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['stats_base_types'], row['stats_types'], row['stats_enums'], row['stats_symbols'] + return None + + def get_verified(self, location: str) -> bool: + results = self._database.cursor().execute('SELECT verified FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['verified'] + return False + + def set_verified(self, location: str, state: bool = True) -> None: + self._database.cursor().execute('UPDATE cache (verified) VALUES (?) WHERE location = ?', + (state, location,)) + def update(self, progress_callback = None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. @@ -245,32 +292,39 @@ class SqliteCache(CacheManagerInterface): with resources.ResourceAccessor().open(location) as fp: json_obj = json.load(fp) identifier = None + + # Get stats + stats_base_types = len(json_obj.get('base_types', {})) + stats_types = len(json_obj.get('types', {})) + stats_enums = len(json_obj.get('enums', {})) + stats_symbols = len(json_obj.get('symbols', {})) + + operating_system = None for idextractor in idextractors: identifier = idextractor.get_identifier(json_obj) - operating_system = idextractor.operating_system if identifier is not None: + operating_system = idextractor.operating_system break + + # We don't try to validate schemas here, we do that on first use + # Store in database + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, " + "stats_base_types, stats_types, stats_enums, stats_symbols, " + "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + stats_base_types, + stats_types, + stats_enums, + stats_symbols, + self.is_url_local(location) + )) if identifier is not None: - # We don't try to validate schemas here, we do that on first use - # Store in database - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - identifier, - operating_system, - self.is_url_local(location) - )) vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") else: - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - None, - None, - self.is_url_local(location) - )) vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") except Exception as excp: vollog.log(constants.LOGLEVEL_VVVV, excp) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 322e574e1..3b499adea 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -76,6 +76,9 @@ MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") """Default location to record information about available identifiers""" +CACHE_SQLITE_SCEMA_VERSION = 1 +"""Version for the sqlite3 cache schema""" + BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" ProgressCallback = Optional[Callable[[float, str], None]] diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index b2960733d..b94cfd69a 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -41,7 +41,11 @@ class IsfInfo(plugins.PluginInterface): optional = True), requirements.VersionRequirement(name = 'SQLiteCache', component = symbol_cache.SqliteCache, - version = (1, 0, 0)) + version = (1, 0, 0)), + requirements.BooleanRequirement(name = 'live', + description = 'Traverse all files, rather than use the cache', + default = False, + optional = True) ] @classmethod @@ -92,28 +96,36 @@ class IsfInfo(plugins.PluginInterface): def check_valid(data): return "Unknown" - # Process the filtered list - for entry in filtered_list: - num_types = num_enums = num_bases = num_symbols = 0 - valid = "Unknown" - with resources.ResourceAccessor().open(url = entry) as fp: - try: - data = json.load(fp) - num_symbols = len(data.get('symbols', [])) - num_types = len(data.get('user_types', [])) - num_enums = len(data.get('enums', [])) - num_bases = len(data.get('base_types', [])) + if self.config['live']: + # Process the filtered list + for entry in filtered_list: + num_types = num_enums = num_bases = num_symbols = 0 + valid = "Unknown" + with resources.ResourceAccessor().open(url = entry) as fp: + try: + data = json.load(fp) + num_symbols = len(data.get('symbols', [])) + num_types = len(data.get('user_types', [])) + num_enums = len(data.get('enums', [])) + num_bases = len(data.get('base_types', [])) - identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) - identifier = identifier_cache.get_identifier(location = entry) - if identifier: - identifier = identifier.decode('utf-8', errors = 'replace') - else: - identifier = renderers.NotAvailableValue() - valid = check_valid(data) - except (UnicodeDecodeError, json.decoder.JSONDecodeError): - vollog.warning(f"Invalid ISF: {entry}") - yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) + identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifier = identifier_cache.get_identifier(location = entry) + if identifier: + identifier = identifier.decode('utf-8', errors = 'replace') + else: + identifier = renderers.NotAvailableValue() + valid = check_valid(data) + except (UnicodeDecodeError, json.decoder.JSONDecodeError): + vollog.warning(f"Invalid ISF: {entry}") + yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) + else: + cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + valid = 'Unknown' + for identifier, location in cache.get_identifier_dictionary().items(): + num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) + if identifier: + yield (0, (location, valid, num_bases, num_types, num_symbols, num_enums, str(identifier))) # Try to open the file, load it as JSON, read the data from it From 1f02fea5d10be5c193f2b100bb18c973335504be Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 23:40:47 +0000 Subject: [PATCH 121/526] Automagic: Change database to store ISF hash instead of verified state --- .../framework/automagic/symbol_cache.py | 27 ++++++++----------- volatility3/framework/plugins/isfinfo.py | 12 +++++++++ volatility3/schemas/__init__.py | 16 +++++++++-- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index c09904713..77bc46265 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -14,6 +14,7 @@ from typing import Dict, Generator, List, Optional, Tuple import volatility3.framework import volatility3.schemas +from volatility3 import schemas from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -150,11 +151,8 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A tuple of base_types, types, enums, symbols, or None is location not found""" - def get_verified(self, location: str) -> bool: - """Returns whether a location ISF has been verified against its schema""" - - def set_verified(self, location: str, state: bool = True) -> None: - """Sets the verified state of a location based on whether it has been successfully verified against its schema""" + def get_hash(self, location: str) -> bool: + """Returns the hash of the JSON from within a location ISF""" class SqliteCache(CacheManagerInterface): @@ -190,7 +188,7 @@ class SqliteCache(CacheManagerInterface): os.unlink(path) return self._connect_storage(path) database.cursor().execute( - 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, verified BOOL DEFAULT False,' + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, hash TEXT,' 'stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)') database.commit() return database @@ -243,16 +241,11 @@ class SqliteCache(CacheManagerInterface): return row['stats_base_types'], row['stats_types'], row['stats_enums'], row['stats_symbols'] return None - def get_verified(self, location: str) -> bool: - results = self._database.cursor().execute('SELECT verified FROM cache WHERE location = ?', + def get_hash(self, location: str) -> Optional[str]: + results = self._database.cursor().execute('SELECT hash FROM cache WHERE location = ?', (location,)).fetchall() for row in results: - return row['verified'] - return False - - def set_verified(self, location: str, state: bool = True) -> None: - self._database.cursor().execute('UPDATE cache (verified) VALUES (?) WHERE location = ?', - (state, location,)) + return row['hash'] def update(self, progress_callback = None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. @@ -291,6 +284,7 @@ class SqliteCache(CacheManagerInterface): try: with resources.ResourceAccessor().open(location) as fp: json_obj = json.load(fp) + hash = schemas.create_json_hash(json_obj) identifier = None # Get stats @@ -309,13 +303,14 @@ class SqliteCache(CacheManagerInterface): # We don't try to validate schemas here, we do that on first use # Store in database cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, " + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, hash," "stats_base_types, stats_types, stats_enums, stats_symbols, " - "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", + "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", ( location, identifier, operating_system, + hash, stats_base_types, stats_types, stats_enums, diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index b94cfd69a..af095b69d 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -125,6 +125,18 @@ class IsfInfo(plugins.PluginInterface): for identifier, location in cache.get_identifier_dictionary().items(): num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) if identifier: + json_hash = cache.get_hash(location) + if json_hash and json_hash in schemas.cached_validations: + valid = 'True (cached)' + if self.config['validate']: + # Even if we're not live, if we've been explicitly asked to validate, then do-so + with resources.ResourceAccessor().open(url = location) as fp: + try: + data = json.load(fp) + valid = check_valid(data) + except (UnicodeDecodeError, json.decoder.JSONDecodeError): + vollog.warning(f"Invalid ISF: {location}") + yield (0, (location, valid, num_bases, num_types, num_symbols, num_enums, str(identifier))) # Try to open the file, load it as JSON, read the data from it diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 65329a4f5..8666680b3 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -6,7 +6,7 @@ import hashlib import json import logging import os -from typing import Set, Any, Dict +from typing import Any, Dict, Optional, Set from volatility3.framework import constants @@ -51,9 +51,21 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: return valid(input, schema, use_cache) -def create_json_hash(input: Dict[str, Any], schema: Dict[str, Any]) -> str: +def create_json_hash(input: Dict[str, Any], schema: Optional[Dict[str, Any]] = None) -> Optional[str]: """Constructs the hash of the input and schema to create a unique identifier for a particular JSON file.""" + if schema is None: + format = input.get('metadata', {}).get('format', None) + if not format: + vollog.debug("No schema format defined") + return None + basepath = os.path.abspath(os.path.dirname(__file__)) + schema_path = os.path.join(basepath, 'schema-' + format + '.json') + if not os.path.exists(schema_path): + vollog.debug(f"Schema for format not found: {schema_path}") + return None + with open(schema_path, 'r') as s: + schema = json.load(s) return hashlib.sha1(bytes(json.dumps((input, schema), sort_keys = True), 'utf-8')).hexdigest() From bda200168a378f79c76acacf93edb6500f766e55 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sat, 28 May 2022 23:49:15 +0100 Subject: [PATCH 122/526] Update volatility3/framework/plugins/isfinfo.py Yep, good spot as ever, thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/plugins/isfinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index af095b69d..6b13f10b6 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -144,4 +144,4 @@ class IsfInfo(plugins.PluginInterface): def run(self): return renderers.TreeGrid([("URI", str), ("Valid", str), ("Number of base_types", int), ("Number of types", int), ("Number of symbols", int), - ("Number of enums", int), ("Identifying infomration", str)], self._generator()) + ("Number of enums", int), ("Identifying information", str)], self._generator()) From ebab09e53a0c56632edc45260e013b42f8097af2 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sat, 28 May 2022 23:50:23 +0100 Subject: [PATCH 123/526] Update volatility3/framework/automagic/symbol_cache.py Cool, I always forget about that, I think it's just what I'm used to, thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 77bc46265..2908774c0 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -276,7 +276,7 @@ class SqliteCache(CacheManagerInterface): number_files_to_process = len(files_to_process) cursor = self._database.cursor() try: - for location in files_to_process: + for counter, location in enumerate(files_to_process): # Open location counter += 1 progress_callback(counter * 100 / number_files_to_process, From a4aa93f05945ab3c3778a25a0c0d3e4709e62c01 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 28 May 2022 23:51:54 +0100 Subject: [PATCH 124/526] Core: Clean up unneeded counter variable, now we're using enumerate --- volatility3/framework/automagic/symbol_cache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 2908774c0..156a1e8c2 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -271,14 +271,12 @@ class SqliteCache(CacheManagerInterface): # New or not recently updated - counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) cursor = self._database.cursor() try: for counter, location in enumerate(files_to_process): # Open location - counter += 1 progress_callback(counter * 100 / number_files_to_process, f"Updating caches for {number_files_to_process} files...") try: From 1e80bb54deb5c8a7cf82057f996bf197058828e7 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:34:44 +0100 Subject: [PATCH 125/526] Update volatility3/framework/configuration/requirements.py Yep, not sure why I forgot, thanks 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index b31c4767f..cc4f05ae6 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -414,7 +414,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): return {} @classmethod - def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]): + def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]) -> bool: if len(required) > 0 and version[0] != required[0]: return False if len(required) > 1 and version[1] < required[1]: From 6f34e1350e67ca893d0f1c5984c45813d9892b5a Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:35:42 +0100 Subject: [PATCH 126/526] Update volatility3/framework/automagic/symbol_cache.py Hehehe, I guess I'm just a little shy about handing out complex objects, but you're right and it is a private method. 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 156a1e8c2..efe1ce601 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -169,7 +169,7 @@ class SqliteCache(CacheManagerInterface): os.unlink(filename) self._database = self._connect_storage(filename) - def _connect_storage(self, path: str): + def _connect_storage(self, path: str) -> sqlite3.Connection: database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( From 98f7fe17433b11a2ef3950e87fabaab8e757e67d Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:49:20 +0100 Subject: [PATCH 127/526] Update volatility3/framework/automagic/symbol_cache.py Quite right, thanks for the catch! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index efe1ce601..47ff66121 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -151,7 +151,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A tuple of base_types, types, enums, symbols, or None is location not found""" - def get_hash(self, location: str) -> bool: + def get_hash(self, location: str) -> Optional[str]: """Returns the hash of the JSON from within a location ISF""" From 504229e46886d9f6d8d3c6a6b8782d67e3656600 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 May 2022 10:52:29 +0100 Subject: [PATCH 128/526] Automagic: include fixes from @digitalisx on review --- volatility3/framework/automagic/symbol_cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 47ff66121..378424ef5 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -10,7 +10,7 @@ import urllib import urllib.parse import urllib.request from abc import abstractmethod -from typing import Dict, Generator, List, Optional, Tuple +from typing import Dict, Generator, Iterable, List, Optional, Tuple import volatility3.framework import volatility3.schemas @@ -113,7 +113,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """ pass - def get_local_locations(self) -> List[str]: + def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" pass @@ -141,7 +141,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns an identifier based on a specific location or None""" pass - def get_identifiers(self, operating_system: Optional[str]): + def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" pass @@ -369,7 +369,7 @@ class SqliteCache(CacheManagerInterface): output[row['identifier']] = row['location'] return output - def get_identifiers(self, operating_system: Optional[str]): + def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: if operating_system: results = self._database.cursor().execute('SELECT identifier FROM cache WHERE operating_system = ?', (operating_system,)).fetchall() From 47accf520bb322040e0cbf1facfa74e32dc944bb Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:55:17 +0100 Subject: [PATCH 129/526] Update volatility3/framework/automagic/symbol_cache.py Yep, you're quite right, not sure how that got left behind. Thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 378424ef5..ed64746fd 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -426,7 +426,6 @@ class RemoteIdentifierFormat: if operating_system in self._data: for identifier in self._data[operating_system]: binary_identifier = base64.b64decode(identifier) - file_list = identifiers.get(binary_identifier, []) for value in self._data[operating_system][identifier]: yield binary_identifier, value if 'additional' in self._data: From c475b792a53305fe7769134f46d1cf502e29e9b6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jun 2022 17:34:22 +0100 Subject: [PATCH 130/526] Automgic: Fix removing stale entries --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index ed64746fd..46431b773 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -347,7 +347,7 @@ class SqliteCache(CacheManagerInterface): if missing_locations: self._database.cursor().execute( - f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", [x for x in missing_locations]) self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ From 225c36631403fe3fa58208befc9d36ad93424b83 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jun 2022 18:54:00 +0100 Subject: [PATCH 131/526] Windows: Update PDB to store correct age value --- volatility3/framework/symbols/windows/pdbconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index da8254ffd..15b5c733a 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -521,7 +521,7 @@ class PdbReader: self.metadata['windows']['pdb'] = { "GUID": self.convert_bytes_to_guid(pdb_info.GUID), - "age": pdb_info.age, + "age": self._dbiheader.age, "database": self._database_name or 'unknown.pdb', "machine_type": self._dbiheader.machine } From ae48a8ab479cc1f60550eef6fe9502b0d49f2e74 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 20 Jul 2022 20:40:23 +0100 Subject: [PATCH 132/526] Documentation: Update text about long cache updates --- README.md | 3 +++ doc/source/symbol-tables.rst | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9f9c1bbb7..348121e44 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,9 @@ Symbol tables zip files must be placed, as named, into the `volatility3/symbols` Windows symbols that cannot be found will be queried, downloaded, generated and cached. Mac and Linux symbol tables must be manually produced by a tool such as [dwarf2json](https://github.com/volatilityfoundation/dwarf2json). +Important: The first run of volatility with new symbol files will require the cache to be updated. The symbol packs contain a large number of symbol files and so may take some time to update! +However, this process only needs to be run once on each new symbol file, so assuming the pack stays in the same location will not need to be done again. Please also note it can be interrupted and next run will restart itself. + Please note: These are representative and are complete up to the point of creation for Windows and Mac. Due to the ease of compiling Linux kernels and the inability to uniquely distinguish them, an exhaustive set of Linux symbol tables cannot easily be supplied. ## Documentation diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index d41e8797a..fd8b8933e 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -18,7 +18,9 @@ configurable within the framework and can usually be set within the user interfa These files can also be compressed into ZIP files, which Volatility will process in order to locate symbol files. Volatility maintains a cache mapping the appropriate identifier for each symbol file against its filename. This cache -is update by automagic called as part of the standard automagic that's run each time a plugin is run. +is updated by automagic called as part of the standard automagic that's run each time a plugin is run. If a large number of new +symbols file are detected, this may take some time, but can be safely interrupted and restarted and will not need to run again +as long as the symbol files stay in the same location. Windows symbol tables --------------------- @@ -92,4 +94,4 @@ file, the banners must match exactly (down to the compilation date). * Copy the `.json` file to the symbols directory into `[symbols directory]/linux` - * For Mac change `linux` to `mac` \ No newline at end of file + * For Mac change `linux` to `mac` From d5c7ef1a9e61fca00b40db1dab7ed52343637278 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:16 +0900 Subject: [PATCH 133/526] Remove: pytest install command --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a3ecd7c7e..cf70b66cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -17,7 +17,6 @@ jobs: python -m pip install --upgrade pip pip install Cmake pip install setuptools wheel - pip install -U pytest pip install -r ./test/requirements-testing.txt - name: Build PyPi packages From 5db182d4303db48dbc4ba401f22b9ad5ff354f7c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:39 +0900 Subject: [PATCH 134/526] Add: .gitignore for test --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index d26e17d91..b3c86d49b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ ENV/ # Memory dump files *.dmp *.vmem +*.img + +# PyTest cache files +.pytest_cache/ \ No newline at end of file From 723fd9b4293b5e5231a2dacd05aa973567c0a9ca Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:51 +0900 Subject: [PATCH 135/526] Fix: json prettier --- test/known_files.json | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/test/known_files.json b/test/known_files.json index 089896714..a579c8053 100644 --- a/test/known_files.json +++ b/test/known_files.json @@ -1,20 +1,19 @@ { - "windows_dumpfiles": { - "win-xp-laptop-2005-06-25.img": { - "0x82220e78": [ - "9bdd5532286f1660f3778e68bc36efe6", - "e3bc1e9e7370e3b5a661ebe591ecf4ec" - ], - "0x82350bf8": [ - "e5c5e8d97b6280745b41f6572c85d1f0", - "8589f1463422884dbf1411aaad278465" - ], - "0x81eaf418": [ - "f7a1ae2060a58f8470b97affdb46dccf", - "54fd611021fa784912530b8007545986" - ], - "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" - } + "windows_dumpfiles": { + "win-xp-laptop-2005-06-25.img": { + "0x82220e78": [ + "9bdd5532286f1660f3778e68bc36efe6", + "e3bc1e9e7370e3b5a661ebe591ecf4ec" + ], + "0x82350bf8": [ + "e5c5e8d97b6280745b41f6572c85d1f0", + "8589f1463422884dbf1411aaad278465" + ], + "0x81eaf418": [ + "f7a1ae2060a58f8470b97affdb46dccf", + "54fd611021fa784912530b8007545986" + ], + "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" } } - \ No newline at end of file +} From 8e9bf4f27cf26cb4578a053808d5c305c15d171c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:50:46 +0900 Subject: [PATCH 136/526] Add: pytest in requirements-test.txt --- test/requirements-testing.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index d37dc93c3..e47f72fa2 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -6,3 +6,5 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 + +pytest>=7.1.2 \ No newline at end of file From 929d19aa50b8b7eef492bcc4215f39b29055fc16 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:51:54 +0900 Subject: [PATCH 137/526] Add: EOF in requirements-test.txt --- test/requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index e47f72fa2..e5966906c 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -7,4 +7,4 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 -pytest>=7.1.2 \ No newline at end of file +pytest>=7.1.2 From 3587828820a21d02fdb901a6d2c61dd1733ae935 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:54:36 +0900 Subject: [PATCH 138/526] Add: EOF in requirements-test.txt --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b3c86d49b..328ba5f83 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,4 @@ ENV/ *.img # PyTest cache files -.pytest_cache/ \ No newline at end of file +.pytest_cache/ From 986088b1a1b0084c8739200b7ff0792f025cc79b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:56:17 +0900 Subject: [PATCH 139/526] Fix: pytest version --- test/requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index e5966906c..7afe19b94 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -7,4 +7,4 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 -pytest>=7.1.2 +pytest>=7.0.0 From 7755328226af96ba811a63d06c8d222877cf28a8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 22 Jul 2022 01:11:35 +0900 Subject: [PATCH 140/526] Fix: psscan required framework version bump --- volatility3/framework/plugins/windows/psscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index cc030b4bf..335624672 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -22,7 +22,7 @@ vollog = logging.getLogger(__name__) class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for processes present in a particular windows memory image.""" - _required_framework_version = (2, 2, 1) + _required_framework_version = (2, 3, 1) _version = (1, 1, 0) @classmethod From 92696068c08a12e9b5f9d7891c82f417c4727185 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 22 Jul 2022 18:55:34 +0900 Subject: [PATCH 141/526] Add: test for vadwalk plugin --- test/test_volatility.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index a55dffb27..2b7fdaf07 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -199,6 +199,16 @@ def test_windows_callbacks(image, volatility, python): assert out.count(b"KeBugCheckReasonCallbackListHead ") > 5 assert rc == 0 +def test_windows_vadwalk(image, volatility, python): + rc, out, err = runvol_plugin("windows.vadwalk.VadWalk", image, volatility, python) + + assert out.find(b"Vad") != -1 + assert out.find(b"VadS") != -1 + assert out.find(b"Vadl") != -1 + assert out.find(b"VadF") != -1 + assert out.find(b"0x0") != -1 + assert rc == 0 + # LINUX def test_linux_pslist(image, volatility, python): From 0fe1f47c9c4978c993de4b9fdb6af3919f72370f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 22 Jul 2022 21:10:43 +0900 Subject: [PATCH 142/526] Fix: support swapped exceptions --- volatility3/framework/plugins/windows/devicetree.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index 8e92de0cc..1b6b55cb7 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -78,7 +78,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): """Listing tree based on drivers and attached devices in a particular windows memory image.""" _required_framework_version = (2, 0, 3) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -96,7 +96,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): try: try: driver_name = driver.get_driver_name() - except (ValueError, exceptions.PagedInvalidAddressException): + except (ValueError, exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Failed to get Driver name : {driver.vol.offset:x}") driver_name = renderers.UnparsableValue() @@ -114,7 +114,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): for device in driver.get_devices(): try: device_name = device.get_device_name() - except (ValueError, exceptions.PagedInvalidAddressException): + except (ValueError, exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Failed to get Device name : {device.vol.offset:x}") device_name = renderers.UnparsableValue() @@ -134,7 +134,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): for level, attached_device in enumerate(device.get_attached_devices(), start=2): try: device_name = attached_device.get_device_name() - except (ValueError, exceptions.PagedInvalidAddressException): + except (ValueError, exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Failed to get Attached Device Name: {attached_device.vol.offset:x}") device_name = renderers.UnparsableValue() @@ -151,7 +151,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): attached_device_type )) - except(exceptions.PagedInvalidAddressException): + except(exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in drivers and devices: {driver.vol.offset:x}") continue From 65d825626e6d847d1a007e9672aa686138bf447b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 22 Jul 2022 21:11:01 +0900 Subject: [PATCH 143/526] Add: test for windows.devicetree --- test/test_volatility.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index a55dffb27..eb713783b 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -199,6 +199,17 @@ def test_windows_callbacks(image, volatility, python): assert out.count(b"KeBugCheckReasonCallbackListHead ") > 5 assert rc == 0 +def test_windows_devicetree(image, volatility, python): + rc, out, err = runvol_plugin("windows.devicetree.DeviceTree", image, volatility, python) + + assert out.find(b"DEV") != -1 + assert out.find(b"DRV") != -1 + assert out.find(b"ATT") != -1 + assert out.find(b"FILE_DEVICE_CONTROLLER") != -1 + assert out.find(b"FILE_DEVICE_DISK") != -1 + assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 + assert rc == 0 + # LINUX def test_linux_pslist(image, volatility, python): From 5a33acd8f955df513f8660d2a5d449a025818262 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 28 Jul 2022 17:44:03 +0300 Subject: [PATCH 144/526] bugfix --- volatility3/framework/automagic/symbol_cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 46431b773..c7cb6a5b8 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -216,7 +216,7 @@ class SqliteCache(CacheManagerInterface): return result def get_local_locations(self) -> Generator[str, None, None]: - result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = True').fetchall() + result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = 1').fetchall() for row in result: yield row['location'] @@ -261,7 +261,7 @@ class SqliteCache(CacheManagerInterface): cache_update = set() files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: - result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " + result = self._database.cursor().execute("SELECT location FROM cache WHERE local = 1 " f"AND cached < date('now', '{self.cache_period}');") for row in result: if row['location'] in files_to_timestamp: @@ -330,7 +330,7 @@ class SqliteCache(CacheManagerInterface): progress_callback(0, 'Reading remote ISF list') cursor = self._database.cursor() cursor.execute( - f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") + f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') for operating_system in constants.OS_CATEGORIES: @@ -356,7 +356,7 @@ class SqliteCache(CacheManagerInterface): additions = [] statement = 'SELECT location, identifier FROM cache' if local_only: - additions.append('local = True') + additions.append('local = 1') if operating_system: additions.append(f"operating_system = '{operating_system}'") if additions: From 95c0ca4ffa0756854a8e16d657175aa67bd7077a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 01:47:18 +0900 Subject: [PATCH 145/526] Remove: pytest module --- test/test_volatility.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index eb713783b..515bef1cc 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -14,8 +14,6 @@ import hashlib import ntpath import json -import pytest - # # HELPER FUNCTIONS # @@ -61,7 +59,6 @@ def test_windows_pslist(image, volatility, python): assert out.find(b"svchost.exe") != -1 assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 rc, out, err = runvol_plugin( "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"]) @@ -69,7 +66,6 @@ def test_windows_pslist(image, volatility, python): assert out.find(b"system") != -1 assert out.count(b"\n") < 10 assert rc == 0 - assert rc == 0 def test_windows_psscan(image, volatility, python): rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) @@ -79,21 +75,18 @@ def test_windows_psscan(image, volatility, python): assert out.find(b"svchost.exe") != -1 assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 def test_windows_dlllist(image, volatility, python): rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 def test_windows_modules(image, volatility, python): rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 def test_windows_hivelist(image, volatility, python): rc, out, err = runvol_plugin("windows.registry.hivelist.HiveList", image, volatility, python) From 64621d90e97cbb2300689559142f009f1af83f9e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:20:01 +0900 Subject: [PATCH 146/526] Add: VSL for frameworkinfo plugin --- volatility3/framework/plugins/frameworkinfo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/frameworkinfo.py b/volatility3/framework/plugins/frameworkinfo.py index b7c887d5c..63ba24d09 100644 --- a/volatility3/framework/plugins/frameworkinfo.py +++ b/volatility3/framework/plugins/frameworkinfo.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from typing import List from volatility3 import framework From 9bfa80e59cb1907163b5adfc054a6a192332630f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:26:21 +0900 Subject: [PATCH 147/526] Add: VSL for initialize file --- volatility3/framework/layers/codecs/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index 550161e6d..e019bcbcd 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + """Codecs used for encoding or decoding data should live here From 54f11d7e18b12c7a8fd384fa333406e5c3dded25 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:26:47 +0900 Subject: [PATCH 148/526] Add: VSL for automagic/module --- volatility3/framework/automagic/module.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index 3d2bb584a..6810a58e2 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from volatility3.framework import interfaces, constants, configuration From a78bf32bd8df8fc075f52fed211e0bf4a9bb7840 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:26:58 +0900 Subject: [PATCH 149/526] Add: VSL for layers/avml --- volatility3/framework/layers/avml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index acc4493f4..f31737232 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + """Functions that read AVML files. The user of the file doesn't have to worry about the compression, From 85a94efd67c41993b15d066343c545da05c2c898 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:27:08 +0900 Subject: [PATCH 150/526] Add: VSL for layers/leechcore --- volatility3/framework/layers/leechcore.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index 8c492ca85..fb0442cfe 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import io import logging import urllib.parse From 5c76dc88e9bf4ec809f1e914a35ace5f14b446c2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:27:15 +0900 Subject: [PATCH 151/526] Add: VSL for layers/linear --- volatility3/framework/layers/linear.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/linear.py b/volatility3/framework/layers/linear.py index c5cb47bdc..383f3d558 100644 --- a/volatility3/framework/layers/linear.py +++ b/volatility3/framework/layers/linear.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import functools from typing import List, Optional, Tuple, Iterable From 6f991f8d4f6d663bd69d33b8c83f61ce37ea39f2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Aug 2022 22:01:21 +0100 Subject: [PATCH 152/526] Core: Fix up LGTM issues across the codebase --- volatility3/framework/automagic/symbol_cache.py | 13 ++++++------- volatility3/framework/automagic/symbol_finder.py | 5 ++--- volatility3/framework/plugins/linux/psaux.py | 16 ++++++++-------- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index c7cb6a5b8..558bfb2f1 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -12,9 +12,7 @@ import urllib.request from abc import abstractmethod from typing import Dict, Generator, Iterable, List, Optional, Tuple -import volatility3.framework -import volatility3.schemas -from volatility3 import schemas +from volatility3 import framework, schemas from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -41,7 +39,7 @@ class IdentifierProcessor: Returns: identifier is valid or None if not found """ - raise NotImplemented("This base class has no get_identifier method defined") + raise NotImplementedError("This base class has no get_identifier method defined") class WindowsIdentifier(IdentifierProcessor): @@ -94,7 +92,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): super().__init__() self._filename = filename self._classifiers = {} - for subclazz in volatility3.framework.class_subclasses(IdentifierProcessor): + for subclazz in framework.class_subclasses(IdentifierProcessor): self._classifiers[subclazz.operating_system] = subclazz def add_identifier(self, location: str, operating_system: str, identifier: str): @@ -267,7 +265,7 @@ class SqliteCache(CacheManagerInterface): if row['location'] in files_to_timestamp: cache_update.add(row['location']) - idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + idextractors = list(framework.class_subclasses(IdentifierProcessor)) # New or not recently updated @@ -347,7 +345,8 @@ class SqliteCache(CacheManagerInterface): if missing_locations: self._database.cursor().execute( - f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", [x for x in missing_locations]) + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", + [x for x in missing_locations]) self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 610ed0e18..a9221a7cc 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -123,9 +123,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): requirement.construct(context, config_path) break else: - if symbol_files: - vollog.debug(f"Symbol library path not found: {symbol_files}") - # print("Kernel", banner, hex(banner_offset)) + vollog.debug(f"Symbol library path not found for: {banner}") + # print("Kernel", banner, hex(banner_offset)) else: vollog.debug("No existing banners found") # TODO: Fallback to generic regex search? diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index ed91c66f2..d8b844ca4 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -4,11 +4,12 @@ from typing import Optional +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework import symbols, exceptions, renderers, interfaces +from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.plugins.linux import pslist -from volatility3.framework.interfaces import plugins + class PsAux(plugins.PluginInterface): """ Lists processes with their command line arguments """ @@ -29,7 +30,7 @@ class PsAux(plugins.PluginInterface): ] def _get_command_line_args(self, task: interfaces.objects.ObjectInterface, - name: str) -> Optional[str]: + name: str) -> Optional[str]: """ Reads the command line arguments of a process These are stored on the userland stack @@ -104,8 +105,7 @@ class PsAux(plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], - self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) - + self._generator( + pslist.PsList.list_tasks(self.context, + self.config['kernel'], + filter_func = filter_func))) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index af3741bbe..430ad6a30 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -13,7 +13,7 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Union from urllib import parse, request from volatility3 import symbols -from volatility3.framework import constants, contexts, exceptions, interfaces +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.configuration.requirements import SymbolTableRequirement From 4c4ccbf4e0e1893b8eefacdb4264ff14585a8802 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Aug 2022 22:03:26 +0100 Subject: [PATCH 153/526] Core: Fix remaining LGTM error --- volatility3/framework/layers/physical.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 0633637ca..b09055c90 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -5,7 +5,7 @@ import logging import threading from typing import Any, Dict, IO, List, Optional, Union -from volatility3.framework import exceptions, interfaces, constants +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -191,7 +191,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): """Closes the file handle.""" self._file.close() - def __exit__(self) -> None: + def __exit__(self, type, value, traceback) -> None: self.destroy() @classmethod From f8506862c4d92422a5e8927f70778f7faf69faf9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Aug 2022 22:59:45 +0100 Subject: [PATCH 154/526] Core: Move jsonschema to dev requirements --- requirements-dev.txt | 26 ++++++++++++++++++++++++++ requirements.txt | 3 --- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..3ff7c50b8 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,26 @@ +# The following packages are required for core functionality. +pefile>=2017.8.1 + +# The following packages are optional. +# If certain packages are not necessary, place a comment (#) at the start of the line. + +# This is required for the yara plugins +yara-python>=3.8.0 + +# This is required for several plugins that perform malware analysis and disassemble code. +# It can also improve accuracy of Windows 8 and later memory samples. +capstone>=3.0.5 + +# This is required by plugins that decrypt passwords, password hashes, etc. +pycryptodome + +# This can improve error messages regarding improperly configured ISF files, +# but is only recommended for development +# jsonschema>=2.3.0 + +# This is required for memory acquisition via leechcore/pcileech. +leechcorepyc>=2.4.0 + +# This is required for analyzing Linux samples compressed using AVMLs native +# compression format. It is not required for AVML's standard LiME compression. +python-snappy==0.6.0 diff --git a/requirements.txt b/requirements.txt index 290d9ca97..1793012f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,9 +14,6 @@ capstone>=3.0.5 # This is required by plugins that decrypt passwords, password hashes, etc. pycryptodome -# This can improve error messages regarding improperly configured ISF files. -jsonschema>=2.3.0 - # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 From 989b4c73273b1acfd767f64a63599fbc511d77dc Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 5 Aug 2022 03:08:18 +0900 Subject: [PATCH 155/526] Fix: cache path for python of Windows Store version --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3b499adea..e83d27108 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -64,7 +64,7 @@ CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" if sys.platform == 'win32': - CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") + CACHE_PATH = os.path.realpath(os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3")) os.makedirs(CACHE_PATH, exist_ok = True) LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") From 837e1ef39df3f5db6422d541b163b47d8226bb83 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 15:58:07 +0900 Subject: [PATCH 156/526] Fix: error handling for netstat plugin --- volatility3/framework/plugins/windows/netstat.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 486957565..3051b950e 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -433,7 +433,8 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self.context, interfaces.configuration.path_join(self.config_path, 'tcpip'), kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) except exceptions.VolatilityException: - vollog.warning("Unable to locate symbols for the memory image's tcpip module") + vollog.error("Unable to locate symbols for the memory image's tcpip module") + raise for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): From a04cb4e031f0a0092aec57ff72c371485893dd66 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 16:13:29 +0900 Subject: [PATCH 157/526] Fix: return syntax --- volatility3/framework/plugins/windows/netstat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 3051b950e..4d6ec5f62 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -434,7 +434,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) except exceptions.VolatilityException: vollog.error("Unable to locate symbols for the memory image's tcpip module") - raise + return for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): From 4f77be32a541563b35279dc7bfda7ee6a52ca853 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 16:30:14 +0900 Subject: [PATCH 158/526] Remove: dump file namespace --- volatility3/plugins/windows/registry/certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 429db96a6..8ef5abcdd 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -47,7 +47,7 @@ class Certificates(interfaces.plugins.PluginInterface): Optional[interfaces.plugins.FileHandlerInterface]: try: if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) + dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) file_handle = open_method(dump_name) file_handle.write(certificate_data) return file_handle From 0c8d4f75ae63a2396deceef229e1c4d2e26135f3 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 16:57:43 +0900 Subject: [PATCH 159/526] Fix: wide exceptions --- volatility3/plugins/windows/registry/certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 8ef5abcdd..6029c0a5c 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -68,7 +68,7 @@ class Certificates(interfaces.plugins.PluginInterface): "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - with contextlib.suppress(KeyError, exceptions.SwappedInvalidAddressException): + with contextlib.suppress(KeyError, exceptions.InvalidAddressException): # Walk it node_path = hive.get_key(top_key, return_list = True) for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): From 471551fda0bc5871f8738f05b06fef1f35c5f826 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 03:13:40 +0900 Subject: [PATCH 160/526] Add: initialize for windows.joblinks plugin --- test/test_volatility.py | 5 ++ .../framework/plugins/windows/joblinks.py | 72 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 volatility3/framework/plugins/windows/joblinks.py diff --git a/test/test_volatility.py b/test/test_volatility.py index 515bef1cc..1126aa9d7 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -203,6 +203,11 @@ def test_windows_devicetree(image, volatility, python): assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 assert rc == 0 +def test_windows_joblinks(image, volatility, python): + rc, out, err = runvol_plugin("windows.joblinks.JobLinks", image, volatility, python) + + assert rc == 0 + # LINUX def test_linux_pslist(image, volatility, python): diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py new file mode 100644 index 000000000..c440cad29 --- /dev/null +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -0,0 +1,72 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterable, Iterator, List, Tuple + +from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.constants import LOGLEVEL_VVVV +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + +class JobLinks(interfaces.plugins.PluginInterface): + """Print process job link information""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls)-> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.BooleanRequirement(name = 'physical', + description = "Display physical offset instead of virtual", + default = False, + optional = True), + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) + ] + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config['kernel']] + memory = self.context.layers[kernel.layer_name] + + for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, + kernel.symbol_table_name): + try: + if not self.config['physical']: + offset = proc.vol.offset + else: + (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + + job = proc.Job.dereference() + + yield (0, ( + format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), + job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, + renderers.NotApplicableValue(), + "(Original Process)" + )) + + vollog.log(LOGLEVEL_VVVV, proc.JobLinks) + vollog.log(LOGLEVEL_VVVV, job.JobLinks) + vollog.log(LOGLEVEL_VVVV, job.ProcessListHead) + + except (exceptions.InvalidAddressException): + continue + + def run(self)-> renderers.TreeGrid: + offsettype = "(V)" if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT) else "(P)" + + return renderers.TreeGrid([ + (f"Offset{offsettype}", format_hints.Hex), ("Name", str), ("PID", int), + ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), + ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str) + ], self._generator()) \ No newline at end of file From e0edb87d7f15b883aea2fbec5539ec1850307037 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 03:25:21 +0900 Subject: [PATCH 161/526] Add: EOF --- volatility3/framework/plugins/windows/joblinks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index c440cad29..2ca32e09d 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -69,4 +69,4 @@ class JobLinks(interfaces.plugins.PluginInterface): (f"Offset{offsettype}", format_hints.Hex), ("Name", str), ("PID", int), ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str) - ], self._generator()) \ No newline at end of file + ], self._generator()) From 7bec33b07d9ab74e798bee315ae9bb8ed8eab26d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 14:40:35 +0900 Subject: [PATCH 162/526] Add: debug log code --- .../framework/plugins/windows/joblinks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 2ca32e09d..e5a206af2 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -56,8 +56,24 @@ class JobLinks(interfaces.plugins.PluginInterface): )) vollog.log(LOGLEVEL_VVVV, proc.JobLinks) + vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Flink)) + vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Blink)) vollog.log(LOGLEVEL_VVVV, job.JobLinks) + vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Flink)) + vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Blink)) vollog.log(LOGLEVEL_VVVV, job.ProcessListHead) + vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Flink)) + vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Blink)) + vollog.log(LOGLEVEL_VVVV, "") + + for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): + yield (1, ( + format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, + entry.InheritedFromUniqueProcessId, entry.get_session_id(), renderers.NotApplicableValue(), entry.get_is_wow64(), + renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + "(Original Process)" + )) except (exceptions.InvalidAddressException): continue From fa686a9fa69c23361df9410b860823b34e31fe38 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:04:24 +0900 Subject: [PATCH 163/526] Add: Peb.ProcessParameters.ImagePathName --- volatility3/framework/plugins/windows/joblinks.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index e5a206af2..1ee64544f 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -55,24 +55,13 @@ class JobLinks(interfaces.plugins.PluginInterface): "(Original Process)" )) - vollog.log(LOGLEVEL_VVVV, proc.JobLinks) - vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Flink)) - vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Blink)) - vollog.log(LOGLEVEL_VVVV, job.JobLinks) - vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Flink)) - vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Blink)) - vollog.log(LOGLEVEL_VVVV, job.ProcessListHead) - vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Flink)) - vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Blink)) - vollog.log(LOGLEVEL_VVVV, "") - for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): yield (1, ( format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), renderers.NotApplicableValue(), entry.get_is_wow64(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - "(Original Process)" + "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string() )) except (exceptions.InvalidAddressException): From ea65649548d708aabe2b4568aa712ef3e8e58ff2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:08:13 +0900 Subject: [PATCH 164/526] Remove: test_windows_joblinks function for test --- test/test_volatility.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 1126aa9d7..515bef1cc 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -203,11 +203,6 @@ def test_windows_devicetree(image, volatility, python): assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 assert rc == 0 -def test_windows_joblinks(image, volatility, python): - rc, out, err = runvol_plugin("windows.joblinks.JobLinks", image, volatility, python) - - assert rc == 0 - # LINUX def test_linux_pslist(image, volatility, python): From 65e7b5302c12068cb78b712d8358f4870eab49dd Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:18:11 +0900 Subject: [PATCH 165/526] Fix: job detail info to zero --- volatility3/framework/plugins/windows/joblinks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 1ee64544f..321e6f791 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -58,8 +58,8 @@ class JobLinks(interfaces.plugins.PluginInterface): for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): yield (1, ( format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, - entry.InheritedFromUniqueProcessId, entry.get_session_id(), renderers.NotApplicableValue(), entry.get_is_wow64(), - renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), + entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), + 0, 0, 0, "Yes", entry.get_peb().ProcessParameters.ImagePathName.get_string() )) From 3556b2374a3f9593d58564431d057ead5859cea7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:22:07 +0900 Subject: [PATCH 166/526] Fix: indent for prettier code --- volatility3/framework/plugins/windows/joblinks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 321e6f791..e88b7a3d3 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -57,12 +57,12 @@ class JobLinks(interfaces.plugins.PluginInterface): for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): yield (1, ( - format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, - entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), - 0, 0, 0, - "Yes", - entry.get_peb().ProcessParameters.ImagePathName.get_string() - )) + format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, + entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), + 0, 0, 0, + "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string() + )) except (exceptions.InvalidAddressException): continue From 2918c13046b91ecdae1fba6599d291fc7ba95ce7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:24:40 +0900 Subject: [PATCH 167/526] Fix: offset for job entry --- volatility3/framework/plugins/windows/joblinks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index e88b7a3d3..39089a741 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -56,6 +56,12 @@ class JobLinks(interfaces.plugins.PluginInterface): )) for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): + + if not self.config['physical']: + offset = entry.vol.offset + else: + (_, _, offset, _, _) = list(memory.mapping(offset = entry.vol.offset, length = 0))[0] + yield (1, ( format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), From cd6a73939ed19426e47209c532481c962b223204 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:27:39 +0900 Subject: [PATCH 168/526] Refactor: apply code style by yapf --- .../framework/plugins/windows/joblinks.py | 49 ++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 39089a741..e30044538 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -15,6 +15,7 @@ from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) + class JobLinks(interfaces.plugins.PluginInterface): """Print process job link information""" @@ -22,62 +23,54 @@ class JobLinks(interfaces.plugins.PluginInterface): _version = (1, 0, 0) @classmethod - def get_requirements(cls)-> List[interfaces.configuration.RequirementInterface]: + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), requirements.BooleanRequirement(name = 'physical', description = "Display physical offset instead of virtual", default = False, optional = True), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) ] def _generator(self) -> Iterator[Tuple]: kernel = self.context.modules[self.config['kernel']] memory = self.context.layers[kernel.layer_name] - for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, - kernel.symbol_table_name): + for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, kernel.symbol_table_name): try: if not self.config['physical']: offset = proc.vol.offset else: (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] - + job = proc.Job.dereference() - - yield (0, ( - format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, - proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), - job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, - renderers.NotApplicableValue(), - "(Original Process)" - )) + + yield (0, (format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), + job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, + renderers.NotApplicableValue(), "(Original Process)")) for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): - if not self.config['physical']: offset = entry.vol.offset else: (_, _, offset, _, _) = list(memory.mapping(offset = entry.vol.offset, length = 0))[0] - yield (1, ( - format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, - entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), - 0, 0, 0, - "Yes", - entry.get_peb().ProcessParameters.ImagePathName.get_string() - )) + yield (1, (format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), + entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, + entry.get_is_wow64(), 0, 0, 0, "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string())) except (exceptions.InvalidAddressException): continue - def run(self)-> renderers.TreeGrid: + def run(self) -> renderers.TreeGrid: offsettype = "(V)" if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT) else "(P)" - return renderers.TreeGrid([ - (f"Offset{offsettype}", format_hints.Hex), ("Name", str), ("PID", int), - ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), - ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str) - ], self._generator()) + return renderers.TreeGrid([(f"Offset{offsettype}", format_hints.Hex), ("Name", str), + ("PID", int), ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), + ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str)], + self._generator()) From 0aedc6a071c9bc0a2b88a7ef978a80be3a9d2e03 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:40:17 +0900 Subject: [PATCH 169/526] Remove: unused module --- volatility3/framework/plugins/windows/joblinks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index e30044538..40d09b9ea 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -4,11 +4,10 @@ import logging -from typing import Iterable, Iterator, List, Tuple +from typing import Iterator, List, Tuple from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.constants import LOGLEVEL_VVVV from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist From 99672cbe759f22ff0518014a8c93d05bc2d188ca Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 15 Aug 2022 15:27:19 +0300 Subject: [PATCH 170/526] improv commit --- volatility3/framework/symbols/windows/pdbutil.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 430ad6a30..9f83ad973 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -253,7 +253,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): pdb_names: List[bytes], progress_callback: constants.ProgressCallback = None, start: Optional[int] = None, - end: Optional[int] = None) -> Generator[Dict[str, Optional[Union[bytes, str, int]]], None, None]: + end: Optional[int] = None, + maximum_invalid_count: int = 100) -> Generator[Dict[str, Optional[Union[bytes, str, int]]], None, None]: """Scans through `layer_name` at `ctx` looking for RSDS headers that indicate one of four common pdb kernel names (as listed in `self.pdb_names`) and returns the tuple (GUID, age, pdb_name, @@ -278,10 +279,15 @@ class PDBUtility(interfaces.configuration.VersionableInterface): sections = [(start, end - start)]): mz_offset = None sig_pfn = signature_offset // page_size + current_invalid_counter = 0 for i in range(sig_pfn, min_pfn, -1): - if not ctx.layers[layer_name].is_valid(i * page_size, 2): + if current_invalid_counter > maximum_invalid_count: break + + if not ctx.layers[layer_name].is_valid(i * page_size, 2): + current_invalid_counter += 1 + continue data = ctx.layers[layer_name].read(i * page_size, 2) if data == b'MZ': From 154659cd0d0049ba7be1911af9a7add6ba3e5fa8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 21 Aug 2022 04:58:54 +0900 Subject: [PATCH 171/526] Fix: typo for cache sqlite schema version --- volatility3/framework/automagic/symbol_cache.py | 6 +++--- volatility3/framework/constants/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 558bfb2f1..ab19965c7 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -171,11 +171,11 @@ class SqliteCache(CacheManagerInterface): database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( - f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCEMA_VERSION})') + f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})') schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() if not schema_version: - database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCEMA_VERSION})') - elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCEMA_VERSION: + database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCHEMA_VERSION})') + elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCHEMA_VERSION: # All good, so pass and move on pass else: diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3b499adea..af3f7c0a0 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -76,7 +76,7 @@ MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") """Default location to record information about available identifiers""" -CACHE_SQLITE_SCEMA_VERSION = 1 +CACHE_SQLITE_SCHEMA_VERSION = 1 """Version for the sqlite3 cache schema""" BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" From 3e071b563d03d69cc06042eb05dfd2136cc49b2e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 21 Aug 2022 05:02:34 +0900 Subject: [PATCH 172/526] Fix: typo for symbol table --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index ab19965c7..a24dc3fd0 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -196,7 +196,7 @@ class SqliteCache(CacheManagerInterface): If multiple locations exist for an identifier, the last found is returned Args: - identifier: string that uniquely identifies a particular symbolt table + identifier: string that uniquely identifies a particular symbol table operating_system: optional string to restrict identifiers to just those for a particular operating system Returns: From ed8d240a7cf1b7d3b39bc467af8ec67d4c8ac190 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 24 Aug 2022 10:24:14 +0300 Subject: [PATCH 173/526] return given layer by base --- volatility3/framework/automagic/windows.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index f5dd720d6..08a5027d1 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -214,6 +214,9 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): context.config[interfaces.configuration.path_join( config_path, "page_map_offset")] = base_layer.metadata['page_map_offset'] layer = layer_type(context, config_path = config_path, name = new_layer_name, metadata = {'os': 'Windows'}) + page_map_offset = context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] + vollog.debug(f"DTB was given to as by base layer: {hex(page_map_offset)}") + return layer # Self Referential finder for description, tests, sections in cls.test_sets: From 253c4b5bb1cc7411255277639a866f8b0c9f87ac Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 24 Aug 2022 10:26:04 +0300 Subject: [PATCH 174/526] typo --- volatility3/framework/automagic/windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 08a5027d1..aaef3e820 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -215,7 +215,7 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): config_path, "page_map_offset")] = base_layer.metadata['page_map_offset'] layer = layer_type(context, config_path = config_path, name = new_layer_name, metadata = {'os': 'Windows'}) page_map_offset = context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] - vollog.debug(f"DTB was given to as by base layer: {hex(page_map_offset)}") + vollog.debug(f"DTB was given to us by base layer: {hex(page_map_offset)}") return layer # Self Referential finder From afb17dfac8ef35950803e549bb127f920ae7eef0 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 24 Aug 2022 14:14:50 +0300 Subject: [PATCH 175/526] use the maximum_invalid_count --- volatility3/framework/automagic/pdbscan.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 5cbdbfe0e..0cb01485e 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -212,7 +212,8 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): start = start_scan_address, page_size = vlayer.page_size, pdb_names = kernel_pdb_names, - progress_callback = progress_callback) + progress_callback = progress_callback, + maximum_invalid_count = constants.windows.PE_MAX_EXTRACTION_SIZE // 0x1000) for kernel in kernels: valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) if valid_kernel is not None: From 9ca83763ba7e1b1012c09af4fb0f416a0b6de7cf Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Tue, 10 May 2022 09:17:03 -0500 Subject: [PATCH 176/526] refs #713 add a vad.get_size() method and fix several off-by-one issues with calculating vad size --- volatility3/framework/plugins/windows/malfind.py | 2 +- .../framework/plugins/windows/skeleton_key_check.py | 2 +- volatility3/framework/plugins/windows/vadinfo.py | 7 ++++--- volatility3/framework/plugins/windows/vadyarascan.py | 4 +--- .../framework/symbols/windows/extensions/__init__.py | 12 ++++++++---- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 700ced8ee..e63b81fb2 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -56,7 +56,7 @@ class Malfind(interfaces.plugins.PluginInterface): all_zero_page = b"\x00" * CHUNK_SIZE offset = 0 - vad_length = vad.get_end() - vad.get_start() + vad_length = vad.get_size() while offset < vad_length: next_addr = vad.get_start() + offset diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index 4a1b48c9a..cb1dd06c6 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -262,7 +262,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): if isinstance(filename, str) and filename.lower().endswith("cryptdll.dll"): base = vad.get_start() - return base, vad.get_end() - base + return base, vad.get_size() return None, None diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index e357b150a..50a69f8fb 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -132,7 +132,7 @@ class VadInfo(interfaces.plugins.PluginInterface): vollog.debug("Unable to find the starting/ending VPN member") return None - if 0 < maxsize < (vad_end - vad_start): + if 0 < maxsize < vad.get_size(): vollog.debug(f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit") return None @@ -151,8 +151,9 @@ class VadInfo(interfaces.plugins.PluginInterface): file_handle = open_method(file_name) chunk_size = 1024 * 1024 * 10 offset = vad_start - while offset < vad_end: - to_read = min(chunk_size, vad_end - offset) + vad_size = vad.get_size() + while offset < vad_start + vad_size: + to_read = min(chunk_size, vad_start + vad_size - offset) data = proc_layer.read(offset, to_read, pad = True) if not data: break diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 06a87d003..3954288eb 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -82,9 +82,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """ vad_root = task.get_vad_root() for vad in vad_root.traverse(): - end = vad.get_end() - start = vad.get_start() - yield (start, end - start) + yield (vad.get_start(), vad.get_size()) def run(self): return renderers.TreeGrid([('Offset', format_hints.Hex), ('PID', int), ('Rule', str), ('Component', str), diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index fe32a0322..bf44d1368 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -197,8 +197,8 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the parent member") - def get_start(self): - """Get the VAD's starting virtual address.""" + def get_start(self) -> int: + """Get the VAD's starting virtual address. This is the first accessible byte in the range.""" if self.has_member("StartingVpn"): @@ -216,8 +216,8 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the starting VPN member") - def get_end(self): - """Get the VAD's ending virtual address.""" + def get_end(self) -> int: + """Get the VAD's ending virtual address. This is the last accessible byte in the range.""" if self.has_member("EndingVpn"): @@ -234,6 +234,10 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the ending VPN member") + def get_size(self) -> int: + """Get the size of the VAD region. The OS ensures page granularity.""" + return (self.get_end() - self.get_start()) + 1 + def get_commit_charge(self): """Get the VAD's commit charge (number of committed pages)""" From 8bbcb51bcb3c27c7871dc6629d50570dc866e6bd Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 25 Aug 2022 01:47:19 +0900 Subject: [PATCH 177/526] Remove: return syntax --- volatility3/framework/plugins/windows/netstat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 4d6ec5f62..93ac3af93 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -434,7 +434,6 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) except exceptions.VolatilityException: vollog.error("Unable to locate symbols for the memory image's tcpip module") - return for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): From a5fe38339a038852cdff47acb0d4942e98fdaefd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Aug 2022 21:15:50 +0100 Subject: [PATCH 178/526] Core: Allow for deprecation of constants gracefully --- volatility3/framework/__init__.py | 2 +- volatility3/framework/automagic/linux.py | 4 ++- volatility3/framework/automagic/mac.py | 4 ++- .../framework/automagic/symbol_cache.py | 3 +- .../framework/automagic/symbol_finder.py | 4 ++- volatility3/framework/constants/__init__.py | 29 ++++++++++++++----- volatility3/framework/plugins/isfinfo.py | 6 ++-- .../framework/symbols/windows/pdbutil.py | 3 +- 8 files changed, 40 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 176eb2242..9b11143b2 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 6, 0) +required_python_version = (3, 7, 0) if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): raise RuntimeError( diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2c152996d..9bb2dae9b 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,6 +3,7 @@ # import logging +import os from typing import Optional, Tuple, Type from volatility3.framework import constants, interfaces @@ -40,7 +41,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - linux_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + linux_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( operating_system = 'linux') # If we have no banners, don't bother scanning if not linux_banners: diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 246462878..9bb3ad5f0 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -3,6 +3,7 @@ # import logging +import os import struct from typing import Optional @@ -42,7 +43,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - mac_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + mac_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( operating_system = 'mac') # If we have no banners, don't bother scanning if not mac_banners: diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 558bfb2f1..d69009721 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -388,7 +388,8 @@ class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._cache = SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + self._cache = SqliteCache(identifiers_path) def __call__(self, context, config_path, configurable, progress_callback = None): """Runs the automagic over the configurable.""" diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index a9221a7cc..7a197dffc 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -3,6 +3,7 @@ # import logging +import os from typing import Any, Callable, Iterable, List, Optional, Tuple from volatility3.framework import constants, interfaces, layers @@ -40,7 +41,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): """Creates a cached copy of the results, but only it's been requested.""" if not self._banners: - cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + cache = symbol_cache.SqliteCache(identifiers_path) self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system) return self._banners diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3b499adea..1f646416b 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,6 +9,7 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys +import warnings from typing import Callable, Optional import volatility3.framework.constants.linux @@ -67,13 +68,7 @@ if sys.platform == 'win32': CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") os.makedirs(CACHE_PATH, exist_ok = True) -LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") -"""Default location to record information about available linux banners""" - -MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") -"""Default location to record information about available mac banners""" - -IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") +IDENTIFIERS_FILENAME = "identifier.cache" """Default location to record information about available identifiers""" CACHE_SQLITE_SCEMA_VERSION = 1 @@ -107,3 +102,23 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" + +### +# DEPRECATED VALUES +### + +_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, 'linux_banners.cache') +"""This value is deprecated and is no longer used within volatility""" + +_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, 'mac_banners.cache') +"""This value is deprecated and is no longer used within volatility""" + +_deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) +"""This value is deprecated in favour of CACHE_PATH joined to IDENTIFIER_FILENAME""" + + +def __getattr__(name): + deprecated_tag = '_deprecated_' + if name in [x[len(deprecated_tag):] for x in globals() if x.startswith(deprecated_tag)]: + warnings.warn(f"{name} is deprecated", FutureWarning) + return globals()[f"{deprecated_tag}{name}"] diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 6b13f10b6..efffa9b87 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -109,7 +109,8 @@ class IsfInfo(plugins.PluginInterface): num_enums = len(data.get('enums', [])) num_bases = len(data.get('base_types', [])) - identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + identifier_cache = symbol_cache.SqliteCache(identifiers_path) identifier = identifier_cache.get_identifier(location = entry) if identifier: identifier = identifier.decode('utf-8', errors = 'replace') @@ -120,7 +121,8 @@ class IsfInfo(plugins.PluginInterface): vollog.warning(f"Invalid ISF: {entry}") yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) else: - cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + cache = symbol_cache.SqliteCache(identifiers_path) valid = 'Unknown' for identifier, location in cache.get_identifier_dictionary().items(): num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 430ad6a30..079b0e826 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -80,7 +80,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Required version of SQLiteCache not found") return None - value = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).find_location( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + value = symbol_cache.SqliteCache(identifiers_path).find_location( symbol_cache.WindowsIdentifier.generate(pdb_name.strip('\x00'), guid.upper(), age), 'windows') if value: From a337ec732a6feaf70032c405a48f1f3ceae39ae5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Aug 2022 21:20:56 +0100 Subject: [PATCH 179/526] Test: Update build tests to new minimum python version --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cf70b66cd..2d3729981 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.7' - name: Install dependencies run: | From d7301d653fca9c1195f83642c7133514c1f6a9a7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 25 Aug 2022 10:55:58 +0100 Subject: [PATCH 180/526] Core: Additional updates with the bump to python 3.7.0 Kindly pointed out by @digitalisx --- README.md | 2 +- setup.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 348121e44..502e26f10 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/setup.py b/setup.py index f6bb687f2..bce21ca66 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,10 @@ from volatility3.framework import constants with open("README.md", "r", encoding = "utf-8") as fh: long_description = fh.read() + def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -19,6 +20,7 @@ def get_install_requires(): requirements.append(stripped_line) return requirements + setuptools.setup(name = "volatility3", description = "Memory forensics framework", version = constants.PACKAGE_VERSION, @@ -34,7 +36,7 @@ setuptools.setup(name = "volatility3", "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires = '>=3.6.0', + python_requires = '>=3.7.0', include_package_data = True, exclude_package_data = { '': ['development', 'development.*'], From e8b4944f9a61e0c833354d8765174576069f48c4 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 27 Aug 2022 01:06:06 +0900 Subject: [PATCH 181/526] Fix: typo for simple-plugin.rst --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index e2143f1b7..c4908caf3 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -9,7 +9,7 @@ of a normal plugin, and reuses other plugins appropriately. .. note:: This document will not include the complete code necessary for a - working plugin (such as imports, etc) since it's designed to focus on the necessary componets for writing a plugin. + working plugin (such as imports, etc) since it's designed to focus on the necessary components for writing a plugin. For complete and functioning plugins, the ``framework/plugins`` directory should be consulted. Inherit from PluginInterface From 1f1355711d08e5e62b27156e5d186e3ed59366b2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 30 Aug 2022 10:54:26 +0100 Subject: [PATCH 182/526] Windows: Fix faulty pdbutil API Commit 5bc517aa appears to have been a broken merge that removed some of the changes made to the pdbutil API unintentionally. This was kindly pointed out in PR #822 by @digitalisx. --- .../framework/symbols/windows/pdbutil.py | 48 ++++++++++++++++--- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 430ad6a30..137d5f4a2 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -13,7 +13,7 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Union from urllib import parse, request from volatility3 import symbols -from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework import constants, contexts, exceptions, interfaces from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.configuration.requirements import SymbolTableRequirement @@ -344,12 +344,46 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - return cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) + module_name = guid["pdb_name"].strip('.pdb') + + symbol_table_name = cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) + + new_module_name = None + if create_module: + new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], + symbol_table_name = symbol_table_name) + new_module_name = new_module.name + + return new_module_name, symbol_table_name + + @classmethod + def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: + """Creates a module in the specified layer_name based on a pdb name. + + Searches the memory section of the loaded module for its PDB GUID + and loads the associated symbol table into the symbol space. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + config_path: The config path where to find symbol files + layer_name: The name of the layer on which to operate + module_offset: This memory dump's module image offset + module_size: The size of the module for this dump + + Returns: + The name of the constructed and loaded symbol table + """ + + module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size, create_module = True) + + return module_name class PdbSignatureScanner(interfaces.layers.ScannerInterface): From 26d15a3ad5069a99702c2d672d8d10c68f64f0b6 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 31 Aug 2022 09:40:22 +0300 Subject: [PATCH 183/526] code review --- volatility3/framework/automagic/pdbscan.py | 3 +-- volatility3/framework/symbols/windows/pdbutil.py | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 0cb01485e..5cbdbfe0e 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -212,8 +212,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): start = start_scan_address, page_size = vlayer.page_size, pdb_names = kernel_pdb_names, - progress_callback = progress_callback, - maximum_invalid_count = constants.windows.PE_MAX_EXTRACTION_SIZE // 0x1000) + progress_callback = progress_callback) for kernel in kernels: valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) if valid_kernel is not None: diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 9f83ad973..311cc6451 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -264,6 +264,14 @@ class PDBUtility(interfaces.configuration.VersionableInterface): The UI should always provide the user an opportunity to specify the appropriate types and PDB values themselves + Args: + layer_name: The layer name to scan + page_size: Size of page constant + pdb_names: List of pdb names to scan + progress_callback: Means of providing the user with feedback during long processes + start: Start address to start scanning from the pdb_names + end: Minimum address to scan the pdb_names + maximum_invalid_count: Amount of pages that can be invalid during scanning before aborting signature search """ min_pfn = 0 From 2a3212e77b74d0ab0d8c3aeaebea3d0a55866a29 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 31 Aug 2022 10:26:38 +0300 Subject: [PATCH 184/526] remove whitespace --- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 311cc6451..8b58442cc 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -292,7 +292,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): for i in range(sig_pfn, min_pfn, -1): if current_invalid_counter > maximum_invalid_count: break - + if not ctx.layers[layer_name].is_valid(i * page_size, 2): current_invalid_counter += 1 continue From 4ed534bc8411408194399dc9698cd688a8d6cf44 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 2 Sep 2022 15:48:46 +0900 Subject: [PATCH 185/526] Fix: typo for yapf style file --- .style.yapf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.style.yapf b/.style.yapf index 8159be910..3f154e07b 100644 --- a/.style.yapf +++ b/.style.yapf @@ -107,7 +107,7 @@ each_dict_entry_on_separate_line=True i18n_comment= # The i18n function call names. The presence of this function stops -# reformattting on that line, because the string it has cannot be moved +# reformatting on that line, because the string it has cannot be moved # away from the i18n comment. i18n_function_call= From a49e7cfeca434e622d68f14d3d9fd567c7d450e6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 02:43:05 +0900 Subject: [PATCH 186/526] Fix: duplicate comments --- volatility3/framework/plugins/windows/cachedump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index ddfa856b9..f77c6257b 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -46,7 +46,7 @@ class Cachedump(interfaces.plugins.PluginInterface): rc4 = ARC4.new(rc4key) data = rc4.encrypt(edata) # lgtm [py/weak-cryptographic-algorithm] else: - # based on Based on code from http://lab.mediaservice.net/code/cachedump.rb + # Based on code from http://lab.mediaservice.net/code/cachedump.rb aes = AES.new(nlkm[16:32], AES.MODE_CBC, ch) data = b"" for i in range(0, len(edata), 16): From 3da028c7346d34cea11198dd897cf817d1e8f621 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 02:54:26 +0900 Subject: [PATCH 187/526] Remove: unused module --- volatility3/framework/plugins/windows/ldrmodules.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 284d1afc2..ba8d049a6 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -1,5 +1,4 @@ -from volatility3.framework import interfaces, constants -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed From 9e578e66da923121c44b8940aa1c0c691352f616 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 02:59:26 +0900 Subject: [PATCH 188/526] Remove: duplicate paragraph --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 96f222187..2a37fd0ed 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -31,7 +31,7 @@ If you make any Additions available to others, such as by providing copies of th - You are responsible to ensure you have rights in Additions necessary to comply with this section. Contributing -If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution." +If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution." Trademarks This license grants you no rights to any trademarks or service marks. From 626e352b18c9288b70dcf1cebb615a8b03379989 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 03:15:28 +0900 Subject: [PATCH 189/526] Add: api changes description for 2.3.1 version --- API_CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/API_CHANGES.md b/API_CHANGES.md index 4d8733286..f74f754f9 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,10 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.3.1 +===== +Update in the windows `_EPROCESS.owning_process` method for support Windows Vista and later versions. + 2.3.0 ===== Add in `child_template` to template class From 97638ffc0dd05c587d031303f431f646ca3752f8 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 12 Sep 2022 16:35:31 +0300 Subject: [PATCH 190/526] fix lineterminator --- volatility3/cli/text_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ecb5179e0..623153fae 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -224,7 +224,7 @@ class CSVRenderer(CLIRenderer): # Ignore the type because namedtuples don't realize they have accessible attributes header_list.append(f"{column.name}") - writer = csv.DictWriter(outfd, header_list) + writer = csv.DictWriter(outfd, header_list, lineterminator='\n') writer.writeheader() def visitor(node: interfaces.renderers.TreeNode, accumulator): From dc31ae1ddadf6d4dcbcf4bfd34bc0354cc16febe Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Tue, 13 Sep 2022 15:21:24 +0300 Subject: [PATCH 191/526] Added new containing address flag to vadinfo plugin --- volatility3/framework/plugins/windows/vadinfo.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index e357b150a..3c51263b7 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -52,6 +52,10 @@ class VadInfo(interfaces.plugins.PluginInterface): "(all other address ranges are excluded). This must be " \ "a base address, not an address within the desired range.", optional = True), + requirements.IntRequirement(name='containing-address', + description="Process virtual memory address to include" \ + "This is a containing address in the VAD.", + optional=True), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -179,6 +183,13 @@ class VadInfo(interfaces.plugins.PluginInterface): filter_func = filter_function + if self.config.get('containing-address', None) is not None: + + def containing_filter_function(x: interfaces.objects.ObjectInterface) -> bool: + return not (x.get_start() <= self.config['containing-address'] <= x.get_end()) + + filter_func = containing_filter_function + for proc in procs: process_name = utility.array_to_string(proc.ImageFileName) From ee3895867f3c124f3aa80c5e2f4add5e02ada33b Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Wed, 21 Sep 2022 13:40:28 -0500 Subject: [PATCH 192/526] refs #713 bump VERSION_MINOR to 4 --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/malfind.py | 2 +- volatility3/framework/plugins/windows/skeleton_key_check.py | 2 +- volatility3/framework/plugins/windows/vadinfo.py | 2 +- volatility3/framework/plugins/windows/vadyarascan.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 6eec88d26..0e661a474 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 3 # Number of changes that only add to the interface +VERSION_MINOR = 4 # Number of changes that only add to the interface VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index e63b81fb2..9b5fab3f5 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -17,7 +17,7 @@ vollog = logging.getLogger(__name__) class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) @classmethod def get_requirements(cls): diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index cb1dd06c6..f6f41864a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -41,7 +41,7 @@ vollog = logging.getLogger(__name__) class Skeleton_Key_Check(interfaces.plugins.PluginInterface): """ Looks for signs of Skeleton Key malware """ - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) @classmethod def get_requirements(cls): diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 50a69f8fb..d3997c8c8 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -33,7 +33,7 @@ winnt_protections = { class VadInfo(interfaces.plugins.PluginInterface): """Lists process memory ranges.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) _version = (2, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 3954288eb..b71e2f605 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -17,7 +17,7 @@ vollog = logging.getLogger(__name__) class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) _version = (1, 0, 0) @classmethod From 941b5ff9c1aef35d1c06e665e5a119a2a82ba79e Mon Sep 17 00:00:00 2001 From: ikelos Date: Wed, 21 Sep 2022 20:19:42 +0100 Subject: [PATCH 193/526] Update volatility3/framework/constants/__init__.py Yep, quite right Co-authored-by: Donghyun Kim --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 0e661a474..00ae15f4e 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 4 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 4985dcd9a3ddff808004da71d636611f5956385b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Sep 2022 20:53:41 +0100 Subject: [PATCH 194/526] Windows: When constructing a buffer, manually dereference onto the native layer --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index fe32a0322..e290ef52d 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -488,7 +488,7 @@ class UNICODE_STRING(objects.StructType): # We manually construct an object rather than casting a dereferenced pointer in case # the buffer length is 0 and the pointer is a NULL pointer return self._context.object(self.vol.type_name.split(constants.BANG)[0] + constants.BANG + 'string', - layer_name = self.Buffer.vol.layer_name, + layer_name = self.Buffer.vol.native_layer_name, offset = self.Buffer, max_length = self.Length, errors = 'replace', encoding = 'utf16') From e5d4e599d3ea1b71853c530f82662e4d8d6c88bf Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Wed, 21 Sep 2022 14:55:17 -0500 Subject: [PATCH 195/526] refs #713 update API_CHANGES.md --- API_CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/API_CHANGES.md b/API_CHANGES.md index 4d8733286..a541e9619 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,10 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.4.0 +===== +Add a `get_size()` method to Windows VAD structures and fix several off-by-one issues when calculating VAD sizes. + 2.3.0 ===== Add in `child_template` to template class From 7529c7b246734ae02b51bb80dc22bbeddb819078 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Sep 2022 21:15:40 +0100 Subject: [PATCH 196/526] Core: Revert volatility 3.7 bump and associated features --- .github/workflows/test.yaml | 4 ++-- README.md | 2 +- setup.py | 6 ++---- volatility3/framework/__init__.py | 2 +- volatility3/framework/constants/__init__.py | 21 --------------------- 5 files changed, 6 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2d3729981..cf70b66cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.7 + - name: Set up Python 3.6 uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.6' - name: Install dependencies run: | diff --git a/README.md b/README.md index 502e26f10..348121e44 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/setup.py b/setup.py index bce21ca66..f6bb687f2 100644 --- a/setup.py +++ b/setup.py @@ -9,10 +9,9 @@ from volatility3.framework import constants with open("README.md", "r", encoding = "utf-8") as fh: long_description = fh.read() - def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -20,7 +19,6 @@ def get_install_requires(): requirements.append(stripped_line) return requirements - setuptools.setup(name = "volatility3", description = "Memory forensics framework", version = constants.PACKAGE_VERSION, @@ -36,7 +34,7 @@ setuptools.setup(name = "volatility3", "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires = '>=3.7.0', + python_requires = '>=3.6.0', include_package_data = True, exclude_package_data = { '': ['development', 'development.*'], diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 9b11143b2..176eb2242 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 7, 0) +required_python_version = (3, 6, 0) if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): raise RuntimeError( diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 1f646416b..d6fb96e1c 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,7 +9,6 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys -import warnings from typing import Callable, Optional import volatility3.framework.constants.linux @@ -102,23 +101,3 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" - -### -# DEPRECATED VALUES -### - -_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, 'linux_banners.cache') -"""This value is deprecated and is no longer used within volatility""" - -_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, 'mac_banners.cache') -"""This value is deprecated and is no longer used within volatility""" - -_deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) -"""This value is deprecated in favour of CACHE_PATH joined to IDENTIFIER_FILENAME""" - - -def __getattr__(name): - deprecated_tag = '_deprecated_' - if name in [x[len(deprecated_tag):] for x in globals() if x.startswith(deprecated_tag)]: - warnings.warn(f"{name} is deprecated", FutureWarning) - return globals()[f"{deprecated_tag}{name}"] From dad8fba67f1208ba32fd42926cf6b4abdfaa6f4e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Sep 2022 21:23:42 +0100 Subject: [PATCH 197/526] Core: Bump to python 3.7 with warnings for old global config variables --- .github/workflows/test.yaml | 4 ++-- README.md | 2 +- setup.py | 6 ++++-- volatility3/framework/__init__.py | 2 +- volatility3/framework/constants/__init__.py | 21 +++++++++++++++++++++ 5 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cf70b66cd..2d3729981 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.7' - name: Install dependencies run: | diff --git a/README.md b/README.md index 348121e44..502e26f10 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/setup.py b/setup.py index f6bb687f2..bce21ca66 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,10 @@ from volatility3.framework import constants with open("README.md", "r", encoding = "utf-8") as fh: long_description = fh.read() + def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -19,6 +20,7 @@ def get_install_requires(): requirements.append(stripped_line) return requirements + setuptools.setup(name = "volatility3", description = "Memory forensics framework", version = constants.PACKAGE_VERSION, @@ -34,7 +36,7 @@ setuptools.setup(name = "volatility3", "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires = '>=3.6.0', + python_requires = '>=3.7.0', include_package_data = True, exclude_package_data = { '': ['development', 'development.*'], diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 176eb2242..9b11143b2 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 6, 0) +required_python_version = (3, 7, 0) if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): raise RuntimeError( diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index e0083a539..a8af538f9 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,6 +9,7 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys +import warnings from typing import Callable, Optional import volatility3.framework.constants.linux @@ -101,3 +102,23 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" + +### +# DEPRECATED VALUES +### + +_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, 'linux_banners.cache') +"""This value is deprecated and is no longer used within volatility""" + +_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, 'mac_banners.cache') +"""This value is deprecated and is no longer used within volatility""" + +_deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) +"""This value is deprecated in favour of CACHE_PATH joined to IDENTIFIER_FILENAME""" + + +def __getattr__(name): + deprecated_tag = '_deprecated_' + if name in [x[len(deprecated_tag):] for x in globals() if x.startswith(deprecated_tag)]: + warnings.warn(f"{name} is deprecated", FutureWarning) + return globals()[f"{deprecated_tag}{name}"] From b8959065e3378f98dad718aeefd4464b451b6bd0 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 23 Sep 2022 00:59:21 +0900 Subject: [PATCH 198/526] Fix: ssl error for python37 --- volatility3/framework/layers/resources.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index dca215c85..4a3bd279a 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -120,11 +120,7 @@ class ResourceAccessor(object): fp = urllib.request.urlopen(url, context = self._context) except error.URLError as excp: if excp.args: - # TODO: As of python3.7 this can be removed - unverified_retrieval = (hasattr(ssl, "SSLCertVerificationError") and isinstance( - excp.args[0], ssl.SSLCertVerificationError)) or (isinstance(excp.args[0], ssl.SSLError) and - excp.args[0].reason == "CERTIFICATE_VERIFY_FAILED") - if unverified_retrieval: + if isinstance(excp.args[0], ssl.SSLCertVerificationError): vollog.warning("SSL certificate verification failed: attempting UNVERIFIED retrieval") non_verifying_ctx = ssl.SSLContext() non_verifying_ctx.check_hostname = False From 3523985d0a7123f2cf4648568a7e41865c9edd57 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 23 Sep 2022 07:47:03 +0900 Subject: [PATCH 199/526] Fix: to find_namepsace_packages method --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f6bb687f2..a4bd3fffe 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setuptools.setup(name = "volatility3", '': ['development', 'development.*'], 'development': ['*'] }, - packages = setuptools.find_packages(exclude = ["development", "development.*"]), + packages = setuptools.find_namespace_packages(exclude = ["development", "development.*"]), entry_points = { 'console_scripts': [ 'vol = volatility3.cli:main', From 949b15a36812d3f4ad33cf55e3e8fb387a0d7cee Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 23 Sep 2022 08:01:11 +0900 Subject: [PATCH 200/526] Fix: unsused module for objects initialize code --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 62e6de553..eedd22bb2 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -9,7 +9,7 @@ import struct from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload from volatility3.framework import constants, interfaces -from volatility3.framework.objects import templates, utility +from volatility3.framework.objects import templates vollog = logging.getLogger(__name__) From d9c434da3ff0425551e0fca530d8d63f6db689ec Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 28 Sep 2022 11:19:46 +0300 Subject: [PATCH 201/526] Removed extra flag --- volatility3/framework/plugins/windows/vadinfo.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 3c51263b7..dc7e4dff4 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -49,13 +49,8 @@ class VadInfo(interfaces.plugins.PluginInterface): # TODO: Convert this to a ListRequirement so that people can filter on sets of ranges requirements.IntRequirement(name = 'address', description = "Process virtual memory address to include " \ - "(all other address ranges are excluded). This must be " \ - "a base address, not an address within the desired range.", + "(all other address ranges are excluded).", optional = True), - requirements.IntRequirement(name='containing-address', - description="Process virtual memory address to include" \ - "This is a containing address in the VAD.", - optional=True), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -179,17 +174,10 @@ class VadInfo(interfaces.plugins.PluginInterface): if self.config.get('address', None) is not None: def filter_function(x: interfaces.objects.ObjectInterface) -> bool: - return x.get_start() not in [self.config['address']] + return not (x.get_start() <= self.config['address'] <= x.get_end()) filter_func = filter_function - if self.config.get('containing-address', None) is not None: - - def containing_filter_function(x: interfaces.objects.ObjectInterface) -> bool: - return not (x.get_start() <= self.config['containing-address'] <= x.get_end()) - - filter_func = containing_filter_function - for proc in procs: process_name = utility.array_to_string(proc.ImageFileName) From 1ffe9f222f984512a2441ee65e7317b7b4953531 Mon Sep 17 00:00:00 2001 From: a5hlynx Date: Fri, 7 Oct 2022 00:57:20 +0900 Subject: [PATCH 202/526] correct ImageFileName --- volatility3/framework/plugins/windows/handles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index ab11d30d6..bdff88075 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -323,7 +323,7 @@ class Handles(interfaces.plugins.PluginInterface): obj_name = item.file_name_with_device() elif obj_type == "Process": item = entry.Body.cast("_EPROCESS") - obj_name = f"{utility.array_to_string(proc.ImageFileName)} Pid {item.UniqueProcessId}" + obj_name = f"{utility.array_to_string(item.ImageFileName)} Pid {item.UniqueProcessId}" elif obj_type == "Thread": item = entry.Body.cast("_ETHREAD") obj_name = f"Tid {item.Cid.UniqueThread} Pid {item.Cid.UniqueProcess}" From 146afc0f0786a9e849e480b54e85562f9a2a19a1 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 16 Oct 2022 08:07:43 +0530 Subject: [PATCH 203/526] Incomplete sentence - fixedf --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 180e7c697..223c9fc07 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -1,7 +1,7 @@ Linux Tutorial ============== -This guide gives you a brief introduction to how volatility3 works and some demonstration of several of the plugins available from +This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite of plugins. Acquiring memory ---------------- From 95e4078b77fd802147b5b3d4662ca8734ef2d6df Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 16 Oct 2022 08:25:35 +0530 Subject: [PATCH 204/526] Added FTK as another example to avoid favouritism --- doc/source/Windows.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index a6c67780e..d732e0cdc 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -8,7 +8,8 @@ Acquiring memory Volatility does not provide the ability to acquire memory. -.. tip:: You could use `WinPmem `_ for collecting windows memory dump. +.. tip:: - You could use `WinPmem `_ for collecting windows memory dump. + - You could also use `FTK Imager `_ Listing Plugins --------------- From fba734b284e0bed5f5cb7b4ac91c3194ea8bab2b Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 16 Oct 2022 19:10:55 +0530 Subject: [PATCH 205/526] AVML added. Restructured Acquiring Memory. --- doc/source/Linux.rst | 7 ++++--- doc/source/Windows.rst | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 223c9fc07..9fb5a686e 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -6,9 +6,10 @@ This guide will give you a brief overview of how volatility3 works as well as a Acquiring memory ---------------- -Volatility3 does not provide the ability to acquire memory. In this tutorial we will see how we can use `LiME `_ for this purpose. -It supports 32 and 64 bit captures from native Intel hardware as well as virtual machine guests. -It also supports capture from Android devices. See below for example commands building and running LiME: +Volatility3 does not provide the ability to acquire memory. +You can use any of the following tools to Acquire memory or the ones you are convenient with: + - `AVML - Acquire Volatile Memory for Linux `_ + - `LIME - Linux Memory Extract `_ .. code-block:: shell-session diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index d732e0cdc..80bc6ddc2 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -7,9 +7,9 @@ Acquiring memory ---------------- Volatility does not provide the ability to acquire memory. - -.. tip:: - You could use `WinPmem `_ for collecting windows memory dump. - - You could also use `FTK Imager `_ +You can use any of the following tools to Acquire memory or the ones you are convenient with: + - `WinPmem `_ + - `FTK Imager `_ Listing Plugins --------------- From 9d9eb226ab64aeefaec82df217ddd478859b340d Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 16 Oct 2022 19:21:42 +0530 Subject: [PATCH 206/526] Removed the commands which were present for LIME --- doc/source/Linux.rst | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 9fb5a686e..ea6c2223c 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -11,20 +11,6 @@ You can use any of the following tools to Acquire memory or the ones you are con - `AVML - Acquire Volatile Memory for Linux `_ - `LIME - Linux Memory Extract `_ -.. code-block:: shell-session - - $ tar -xvzf lime-forensics-1.1-r14.tar.gz - $ cd lime-forensics-1.1-r14/src - $ make - .... - CC [M] lime-forensics-1.1-r14/src/tcp.o - CC [M] lime-forensics-1.1-r14/src/disk.o - .... - $ sudo insmod lime-3.2.0-23-generic.ko "path=/tmp/ubuntu.lime format=lime" - $ ls -alh /tmp/ubuntu.lime - -r--r--r-- 1 root root 2.0G Aug 17 19:37 /tmp/ubuntu.lime - -.. note:: The above command required sudo inorder to access the files which are root only. Procedure to create symbol tables for linux -------------------------------------------- From b71e367d387ed13a083bde17cb7a586a1c28cf67 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Oct 2022 15:50:22 +0100 Subject: [PATCH 207/526] Documentation: Rename, fix grammar and avoid using personal pronouns --- ...rst => getting-started-linux-tutorial.rst} | 51 +++++++++++-------- ...t => getting-started-windows-tutorial.rst} | 43 +++++++++------- doc/source/index.rst | 18 +++---- doc/source/symbol-tables.rst | 2 +- 4 files changed, 66 insertions(+), 48 deletions(-) rename doc/source/{Linux.rst => getting-started-linux-tutorial.rst} (71%) rename doc/source/{Windows.rst => getting-started-windows-tutorial.rst} (71%) diff --git a/doc/source/Linux.rst b/doc/source/getting-started-linux-tutorial.rst similarity index 71% rename from doc/source/Linux.rst rename to doc/source/getting-started-linux-tutorial.rst index ea6c2223c..15a1f0d1b 100644 --- a/doc/source/Linux.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -1,31 +1,34 @@ Linux Tutorial ============== -This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite of plugins. +This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite. Acquiring memory ---------------- -Volatility3 does not provide the ability to acquire memory. -You can use any of the following tools to Acquire memory or the ones you are convenient with: - - `AVML - Acquire Volatile Memory for Linux `_ - - `LIME - Linux Memory Extract `_ +Volatility3 does not provide the ability to acquire memory. Below are some examples of tools that can be used to acquire memory, but more are available: + +* `AVML - Acquire Volatile Memory for Linux `_ +* `LIME - Linux Memory Extract `_ Procedure to create symbol tables for linux -------------------------------------------- -To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. +To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. -.. tip:: We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. - After creating the file or downloading the file from the ISF server, please place the file under the directory ``volatility3/symbols/linux``. Make a directory linux under symbols. +.. tip:: It may be possible to locate pre-made ISF files from the `Linux ISF Server `_ , + which is built and maintained by `kevthehermit `_. + After creating the file or downloading it from the ISF server, place the file under the directory ``volatility3/symbols/linux``. + If necessary create a linux directory under the symbols directory (this will become unncessary in future versions). Listing plugins --------------- -Following are the sample of linux plugins available for volatility3. More plugins will be available on future releases. -For plugin requests, Please create an issue with description of the plugin. +The following is a sample of the linux plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. .. code-block:: shell-session @@ -36,13 +39,13 @@ For plugin requests, Please create an issue with description of the plugin. linux.check_creds.Check_creds linux.check_idt.Check_idt -.. note:: Here the the command is piped to grep and head in-order to give you sample list of linux plugins. +.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of linux plugins. Using plugins ------------- -The following is the syntax to run volatility tool. +The following is the syntax to run the volatility CLI. .. code-block:: shell-session @@ -52,11 +55,11 @@ The following is the syntax to run volatility tool. Example ------- -Example 1 -~~~~~~~~~ +banners +~~~~~~~ -In this example we will be using memory dump from Insomni'hack teaser 2020 CTF. Challenge name Getdents. We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. -I'd like to say thanks to `stuxnet `_ for providing this memory dump and `writeup `_. +In this example we will be using a memory dump from the Insomni'hack teaser 2020 CTF Challenge called Getdents. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge. +Thanks go to `stuxnet `_ for providing this memory dump and `writeup `_. .. code-block:: shell-session @@ -75,11 +78,13 @@ I'd like to say thanks to `stuxnet `_ for provid 0x7fde0010 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) -This above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for ISF file from the ISF server. -If you do not find the ISF file then, please follow the instructions on :ref:`Linux:Procedure to create symbol tables for linux`. After that place the ISF file under ``volatility3/symbols/linux`` directory. +The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server. +If ISF file cannt be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. .. tip:: Use the banner text which is most repeated to search from ISF Server. +linux.pslist +~~~~~~~~~~~~ .. code-block:: shell-session @@ -109,6 +114,9 @@ If you do not find the ISF file then, please follow the instructions on :ref:`Li ``linux.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. +linux.pstree +~~~~~~~~~~~~ + .. code-block:: shell-session $ python3 vol.py -f memory.vmem linux.pstree @@ -148,9 +156,12 @@ If you do not find the ISF file then, please follow the instructions on :ref:`Li ***** 1548 1266 gsd-keyboard ***** 1550 1266 gsd-media-keys -``linux.pstree`` helps us to display the parent child relation of processes. +``linux.pstree`` helps us to display the parent child relationships between processes. -Now to find the commands ran in bash shell. Lets use ``linux.bash``. +linux.bash +~~~~~~~~~~ + +Now to find the commands that were run in the bash shell by using ``linux.bash``. .. code-block:: shell-session diff --git a/doc/source/Windows.rst b/doc/source/getting-started-windows-tutorial.rst similarity index 71% rename from doc/source/Windows.rst rename to doc/source/getting-started-windows-tutorial.rst index 80bc6ddc2..c89b065f5 100644 --- a/doc/source/Windows.rst +++ b/doc/source/getting-started-windows-tutorial.rst @@ -1,21 +1,23 @@ Windows Tutorial ================ -This guide gives you a brief introduction to how volatility3 works and some demonstration on suite of plugins available from +This guide provides a brief introduction to how volatility3 works as a demonstration of several of the plugins available in the suite. Acquiring memory ---------------- Volatility does not provide the ability to acquire memory. -You can use any of the following tools to Acquire memory or the ones you are convenient with: - - `WinPmem `_ - - `FTK Imager `_ +Memory can be acquired using a number of tools, below are some examples but others exist: + +* `WinPmem `_ +* `FTK Imager `_ Listing Plugins --------------- -Following are the sample of linux plugins available for volatility3. More plugins will be available on future releases. -For plugin requests, Please create an issue with description of the plugin. +The following is a sample of the windows plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. .. code-block:: shell-session @@ -24,14 +26,13 @@ For plugin requests, Please create an issue with description of the plugin. windows.cmdline.CmdLine windows.crashinfo.Crashinfo windows.dlllist.DllList - Lists the loaded modules in a particular windows -.. note:: Here the the command is piped to grep and head in-order to give you sample list of windows plugins. +.. note:: Here the the command is piped to grep and head in-order to provide the start of a list of the available windows plugins. Using plugins ------------- -The following is the syntax to run volatility tool. +The following is the syntax to run the volatility CLI. .. code-block:: shell-session @@ -41,13 +42,14 @@ The following is the syntax to run volatility tool. Example ------- -Example 1 -~~~~~~~~~ +windows.pslist +~~~~~~~~~~~~~~ -In this example we will be using memory dump from PragyanCTF'22. -We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. +In this example we will be using a memory dump from the PragyanCTF'22. +We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenges. -In windows memory forensics using volatility3, most of the times we do not require creating a ISF file. +When using windows plugins in volatility 3, the required ISF file can often be generated from PDB files automatically +downloaded from Microsoft servers, and therefore does not require locating or adding specific ISF files to the volatility 3 symbols directory. .. code-block:: shell-session @@ -64,7 +66,10 @@ In windows memory forensics using volatility3, most of the times we do not requi 412 396 csrss.exe 0xfa80021c5b30 9 224 1 False 2022-02-07 16:30:13.000000 N/A Disabled 468 396 winlogon.exe 0xfa8002284060 5 113 1 False 2022-02-07 16:30:14.000000 N/A Disabled -``windows.pslist`` helps us list the processes running while the memory dump was taken. +``windows.pslist`` helps list the processes running while the memory dump was taken. + +windows.pstree +~~~~~~~~~~~~~~ .. code-block:: shell-session @@ -90,10 +95,12 @@ In windows memory forensics using volatility3, most of the times we do not requi ** 616 504 svchost.exe 0xfa8002b86ab0 13 314 0 False 2022-02-07 16:32:16.000000 N/A ** 624 504 svchost.exe 0xfa8002410630 10 350 0 False 2022-02-07 16:30:14.000000 N/A -``windows.pstree`` helps us to display the parent child relation of processes. +``windows.pstree`` helps to display the parent child relationships between processes. -.. note:: Here the the command is piped to head in-order to give you smaller output of process here top 20. +.. note:: Here the the command is piped to head in-order to provide smaller output, here listing only the first 20. +windows.hashdump +~~~~~~~~~~~~~~~~ .. code-block:: shell-session @@ -108,7 +115,7 @@ In windows memory forensics using volatility3, most of the times we do not requi HomeGroupUser$ 1002 aad3b435b51404eeaad3b435b51404ee af10ecac6ea817d2bb56e3e5c33ce1cd Dennis 1003 aad3b435b51404eeaad3b435b51404ee cf96684bbc7877920adaa9663698bf54 -``windows.hashdump`` helps us to list the hashes of the users in the system. +``windows.hashdump`` helps to list the hashes of the users in the system. diff --git a/doc/source/index.rst b/doc/source/index.rst index 0d35b02ba..9b1d05858 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -7,7 +7,7 @@ Volatility 3 is Open Source. :doc:`List of plugins ` -Here are some guidelines for using Volatility 3 effectively: +Below is the main documentation regarding volatility 3: .. toctree:: :caption: Documentation @@ -19,6 +19,14 @@ Here are some guidelines for using Volatility 3 effectively: volshell glossary +There is also some information to get you started quickly: + +.. toctree:: + :caption: Getting Started + + getting-started-linux-tutorial + getting-started-windows-tutorial + .. toctree:: :caption: Python Packages @@ -26,14 +34,6 @@ Here are some guidelines for using Volatility 3 effectively: volatility3 -.. toctree:: - :caption: Getting Started - - FAQ - Installation - Linux - Windows - Indices and tables ================== diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index d912d4906..b7c26e046 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -41,7 +41,7 @@ The :envvar:`PYTHONPATH` environment variable is not required if the Volatility or a virtual environment. Mac or Linux symbol tables ------------------------ +-------------------------- For Mac/Linux systems, both use the same mechanism for identification. The generated files contain an identifying string (the operating system banner), which Volatility's automagic can detect. Volatility caches the mapping between the strings and the symbol From 537f6a6a55b830534af5715fd8bd659111188b54 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Oct 2022 15:55:45 +0100 Subject: [PATCH 208/526] Documentation: Fix minor typo --- doc/source/getting-started-linux-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst index 15a1f0d1b..e1c671c36 100644 --- a/doc/source/getting-started-linux-tutorial.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -9,7 +9,7 @@ Acquiring memory Volatility3 does not provide the ability to acquire memory. Below are some examples of tools that can be used to acquire memory, but more are available: * `AVML - Acquire Volatile Memory for Linux `_ -* `LIME - Linux Memory Extract `_ +* `LiME - Linux Memory Extract `_ Procedure to create symbol tables for linux From 439835a61d4ba3abaec3b94048350ce85585872f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 17 Oct 2022 04:50:58 +0900 Subject: [PATCH 209/526] Fix: typo for linux tutorial --- doc/source/getting-started-linux-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst index e1c671c36..6fd06bcf9 100644 --- a/doc/source/getting-started-linux-tutorial.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -20,7 +20,7 @@ To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol .. tip:: It may be possible to locate pre-made ISF files from the `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. After creating the file or downloading it from the ISF server, place the file under the directory ``volatility3/symbols/linux``. - If necessary create a linux directory under the symbols directory (this will become unncessary in future versions). + If necessary create a linux directory under the symbols directory (this will become unnecessary in future versions). Listing plugins From 88e944192093281c833b1404c4592b51ac364c9f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 22 Oct 2022 18:20:22 +0900 Subject: [PATCH 210/526] Fix: typo for linux tutorial --- doc/source/getting-started-linux-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst index 6fd06bcf9..26ad2c2e4 100644 --- a/doc/source/getting-started-linux-tutorial.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -79,7 +79,7 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server. -If ISF file cannt be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. .. tip:: Use the banner text which is most repeated to search from ISF Server. From 94bb22d4bcc35cd355b31d873c8d54f42457f2ae Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 23 Oct 2022 22:49:06 +0100 Subject: [PATCH 211/526] Automagic: Make cache period longer and configurable --- volatility3/framework/automagic/symbol_cache.py | 2 +- volatility3/framework/constants/__init__.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 1e0bba86e..30a4068b6 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -157,10 +157,10 @@ class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) - cache_period = '-3 days' def __init__(self, filename: str): super().__init__(filename) + self.cache_period = constants.SQLITE_CACHE_PERIOD try: self._database = self._connect_storage(filename) except sqlite3.DatabaseError: diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index e0083a539..4fd53a3eb 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -63,6 +63,9 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" +SQLITE_CACHE_PERIOD = '-1 month' +"""SQLite time modifier for how long each item is valid in the cache for""" + if sys.platform == 'win32': CACHE_PATH = os.path.realpath(os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3")) os.makedirs(CACHE_PATH, exist_ok = True) From 1f185d0ee2772bfa77fb2bfc60719aca42933b2f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 28 Oct 2022 16:51:03 +1100 Subject: [PATCH 212/526] Minor fix comment typo --- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 0306bec02..29be88309 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -100,7 +100,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return sock, sock_stat, extended def _update_extended_socket_filters_info(self, sock: objects.Pointer, extended: dict) -> None: - """Get infomation from the socket and reuseport filters + """Get information from the socket and reuseport filters Args: sock: The kernel sock (sk) struct @@ -454,7 +454,7 @@ class Sockstat(plugins.PluginInterface): destination: Destination address string state: State strings (LISTEN, CONNECTED, etc) tasks: String with a list of tasks and FDs using a socket. It can also have - exteded information such as socket filters, bpf info, etc. + extended information such as socket filters, bpf info, etc. """ filter_func = lsof.pslist.PsList.create_pid_filter(pids) socket_generator = self.list_sockets(self.context, symbol_table, filter_func=filter_func) From dfadf5376a0a61a0dc7701014527303cd0d1ad63 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 28 Oct 2022 20:46:08 +1100 Subject: [PATCH 213/526] Fix issue with AF_XDP socket family, issues with older kernel versions and other fixes and improvements --- .../framework/plugins/linux/sockstat.py | 31 ++++++++++++------- .../framework/symbols/linux/__init__.py | 30 +++++++++--------- .../symbols/linux/extensions/__init__.py | 26 ++++++++++++---- 3 files changed, 53 insertions(+), 34 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 29be88309..6d50c296d 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -5,7 +5,8 @@ import logging from typing import Callable, Tuple, List, Dict -from volatility3.framework import renderers, interfaces, exceptions, constants, objects +from volatility3.framework import interfaces, exceptions, constants, objects +from volatility3.framework.renderers import TreeGrid, NotAvailableValue from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -92,7 +93,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): # Even if the sock family is not supported, or the required types # are not present in the symbols, we can still show some general # information about the socket that may be helpful. - saddr_tag = daddr_tag = "?" + saddr_tag = daddr_tag = NotAvailableValue() state = sock.get_state() sock_stat = saddr_tag, daddr_tag, state @@ -123,16 +124,22 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return bpfprog = sock_filter.prog + if bpfprog.type == 0: + # BPF_PROG_TYPE_UNSPEC = 0 + return - # BPF_PROG_TYPE_UNSPEC = 0 - if bpfprog.type > 0: - extended["bpf_filter_type"] = "eBPF" - bpfprog_aux = bpfprog.aux - if bpfprog_aux: - extended["bpf_filter_id"] = str(bpfprog_aux.id) - bpfprog_name = utility.array_to_string(bpfprog_aux.name) - if bpfprog_name: - extended["bpf_filter_name"] = bpfprog_name + extended["bpf_filter_type"] = "eBPF" + if not bpfprog.has_member("aux") or not bpfprog.aux: + return + bpfprog_aux = bpfprog.aux + if bpfprog_aux.has_member("id"): + # `id` member was added to `bpf_prog_aux` in kernels 4.13 + extended["bpf_filter_id"] = str(bpfprog_aux.id) + if bpfprog_aux.has_member("name"): + # `name` was added to `bpf_prog_aux` in kernels 4.15 + bpfprog_name = utility.array_to_string(bpfprog_aux.name) + if bpfprog_name: + extended["bpf_filter_name"] = bpfprog_name def _unix_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: """Handles the AF_UNIX socket family @@ -503,4 +510,4 @@ class Sockstat(plugins.PluginInterface): ("State", str), ("Tasks", str)] - return renderers.TreeGrid(tree_grid_args, self._generator(pids, netns_id, symbol_table)) + return TreeGrid(tree_grid_args, self._generator(pids, netns_id, symbol_table)) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index bd8748ec8..1945bb3ef 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -27,8 +27,15 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('dentry', extensions.dentry) self.set_type_class('fs_struct', extensions.fs_struct) self.set_type_class('files_struct', extensions.files_struct) - self.set_type_class('vfsmount', extensions.vfsmount) self.set_type_class('kobject', extensions.kobject) + # Might not exist in the current symbols + self.optional_set_type_class('module', extensions.module) + + # Mount + self.set_type_class('vfsmount', extensions.vfsmount) + # Might not exist in older kernels or the current symbols + self.optional_set_type_class('mount', extensions.mount) + self.optional_set_type_class('mnt_namespace', extensions.mnt_namespace) # Network self.set_type_class('net', extensions.net) @@ -36,21 +43,12 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('sock', extensions.sock) self.set_type_class('inet_sock', extensions.inet_sock) self.set_type_class('unix_sock', extensions.unix_sock) - self.set_type_class('netlink_sock', extensions.netlink_sock) - self.set_type_class('vsock_sock', extensions.vsock_sock) - self.set_type_class('packet_sock', extensions.packet_sock) - - if 'bt_sock' in self.types: - self.set_type_class('bt_sock', extensions.bt_sock) - - if 'mnt_namespace' in self.types: - self.set_type_class('mnt_namespace', extensions.mnt_namespace) - - if 'module' in self.types: - self.set_type_class('module', extensions.module) - - if 'mount' in self.types: - self.set_type_class('mount', extensions.mount) + # Might not exist in older kernels or the current symbols + self.optional_set_type_class('netlink_sock', extensions.netlink_sock) + self.optional_set_type_class('vsock_sock', extensions.vsock_sock) + self.optional_set_type_class('packet_sock', extensions.packet_sock) + self.optional_set_type_class('bt_sock', extensions.bt_sock) + self.optional_set_type_class('xdp_sock', extensions.xdp_sock) class LinuxUtilities(interfaces.configuration.VersionableInterface): diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 27fa00a8c..d193200c4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -16,8 +16,7 @@ from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_ST from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility -from volatility3.framework.symbols import generic, linux -from volatility3.framework.symbols import intermed +from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf vollog = logging.getLogger(__name__) @@ -840,9 +839,15 @@ class sock(objects.StructType): return self.sk_socket.get_inode() + def get_protocol(self): + return "" + def get_state(self): # Return the generic socket state - return self.sk.sk_socket.get_state() + if self.has_member("sk"): + return self.sk.sk_socket.get_state() + + return self.sk_socket.get_state() class unix_sock(objects.StructType): def get_name(self): @@ -989,7 +994,6 @@ class netlink_sock(objects.StructType): # Return the generic socket state return self.sk.sk_socket.get_state() - class vsock_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for vsocks @@ -1002,7 +1006,6 @@ class vsock_sock(objects.StructType): # Return the generic socket state return self.sk.sk_socket.get_state() - class packet_sock(objects.StructType): def get_protocol(self): eth_proto = socket_module.htons(self.num) @@ -1017,7 +1020,6 @@ class packet_sock(objects.StructType): # Return the generic socket state return self.sk.sk_socket.get_state() - class bt_sock(objects.StructType): def get_protocol(self): type_idx = self.sk.sk_protocol @@ -1032,3 +1034,15 @@ class bt_sock(objects.StructType): return BLUETOOTH_STATES[state_idx] else: return "UNKNOWN" + +class xdp_sock(objects.StructType): + def get_protocol(self): + # The protocol should always be 0 for xdp_sock + if self.sk.sk_protocol == 0: + return "" + else: + return "UNKNOWN" + + def get_state(self): + # Return the generic socket state + return self.sk.sk_socket.get_state() From 85dcd04961f76c095bce069102d2d6ba4b4a9411 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 28 Oct 2022 19:59:53 +0000 Subject: [PATCH 214/526] Add drivermodule plugin and wrap common access to a Drver's names in a method --- .../framework/plugins/windows/drivermodule.py | 60 +++++++++++++++++++ .../framework/plugins/windows/driverscan.py | 45 +++++++++----- 2 files changed, 90 insertions(+), 15 deletions(-) create mode 100644 volatility3/framework/plugins/windows/drivermodule.py diff --git a/volatility3/framework/plugins/windows/drivermodule.py b/volatility3/framework/plugins/windows/drivermodule.py new file mode 100644 index 000000000..54a132adb --- /dev/null +++ b/volatility3/framework/plugins/windows/drivermodule.py @@ -0,0 +1,60 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework import renderers, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import ssdt, driverscan + +# built in Windows-components that trigger false positives +KNOWN_DRIVERS = ["ACPI_HAL", + "PnpManager", + "RAW", + "WMIxWDM", + "Win32k", + "Fs_Rec"] + +class DriverModule(interfaces.plugins.PluginInterface): + """Determines if any loaded drivers were hidden by a rootkit""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'ssdt', plugin = ssdt.SSDT, version = (1, 0, 0)), + requirements.PluginRequirement(name = 'driverscan', plugin = driverscan.DriverScan, version = (1, 0, 0)), + ] + + def _generator(self): + """ + Attempt to match each driver's start code address to a known kernel module + A common rootkit technique is to register drivers from modules that are hidden, + which allows us to detect the disconnect between a malicious driver and its hidden module. + """ + kernel = self.context.modules[self.config['kernel']] + + collection = ssdt.SSDT.build_module_collection(self.context, kernel.layer_name, kernel.symbol_table_name) + + for driver in driverscan.DriverScan.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): + # we do not care about actual symbol names, we just want to know if the driver points to a known module + module_symbols = list(collection.get_module_symbols_by_absolute_location(driver.DriverStart)) + if not module_symbols: + driver_name, service_key, name = driverscan.DriverScan.get_names_for_driver(driver) + + known_exception = driver_name in KNOWN_DRIVERS + + yield (0, (format_hints.Hex(driver.vol.offset), known_exception, driver_name, service_key, name)) + + def run(self): + + return renderers.TreeGrid([ + ("Offset", format_hints.Hex), + ("Known Exception", bool), + ("Driver Name", str), + ("Serivce Key", str), + ("Alternative Name", str), + ], self._generator()) diff --git a/volatility3/framework/plugins/windows/driverscan.py b/volatility3/framework/plugins/windows/driverscan.py index 2cf309014..60ac0d67a 100644 --- a/volatility3/framework/plugins/windows/driverscan.py +++ b/volatility3/framework/plugins/windows/driverscan.py @@ -48,25 +48,40 @@ class DriverScan(interfaces.plugins.PluginInterface): _constraint, mem_object, _header = result yield mem_object + @classmethod + def get_names_for_driver(cls, driver): + """ + Convenience method for getting the commonly used + names associated with a driver + + Args: + driver: A Eriver object + + Returns: + A tuple of strings of (driver name, service key, driver alt. name) + """ + try: + driver_name = driver.get_driver_name() + except (ValueError, exceptions.InvalidAddressException): + driver_name = renderers.NotApplicableValue() + + try: + service_key = driver.DriverExtension.ServiceKeyName.String + except exceptions.InvalidAddressException: + service_key = renderers.NotApplicableValue() + + try: + name = driver.DriverName.String + except exceptions.InvalidAddressException: + name = renderers.NotApplicableValue() + + return driver_name, service_key, name + def _generator(self): kernel = self.context.modules[self.config['kernel']] for driver in self.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): - - try: - driver_name = driver.get_driver_name() - except (ValueError, exceptions.InvalidAddressException): - driver_name = renderers.NotApplicableValue() - - try: - service_key = driver.DriverExtension.ServiceKeyName.String - except exceptions.InvalidAddressException: - service_key = renderers.NotApplicableValue() - - try: - name = driver.DriverName.String - except exceptions.InvalidAddressException: - name = renderers.NotApplicableValue() + driver_name, service_key, name = self.get_names_for_driver(driver) yield (0, (format_hints.Hex(driver.vol.offset), format_hints.Hex(driver.DriverStart), format_hints.Hex(driver.DriverSize), service_key, driver_name, name)) From 017fcc05f35314e180c172504c9a176b3a66551b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 29 Oct 2022 12:55:16 +1100 Subject: [PATCH 215/526] Split address and port. Fix potential issues in xdp_sock(s) with older kernels. Postpone any kind of formatting to the generator making it more appropriate to be used as a library. --- .../framework/plugins/linux/sockstat.py | 164 +++++++++--------- .../symbols/linux/extensions/__init__.py | 49 ++---- 2 files changed, 100 insertions(+), 113 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 6d50c296d..f927c0c83 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -72,7 +72,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns a tuple with: sock: The respective kernel's *_sock object for that socket family - sock_stat: A tuple with the source, destination and state strings. + sock_stat: A tuple with the source and destination (address and port) along with its state string. extended: A dictionary with key/value extended information. """ family = sock.get_family() @@ -93,10 +93,10 @@ class SockHandlers(interfaces.configuration.VersionableInterface): # Even if the sock family is not supported, or the required types # are not present in the symbols, we can still show some general # information about the socket that may be helpful. - saddr_tag = daddr_tag = NotAvailableValue() + src_addr = src_port = dst_addr = dst_port = None state = sock.get_state() - sock_stat = saddr_tag, daddr_tag, state + sock_stat = src_addr, src_port, dst_addr, dst_port, state return sock, sock_stat, extended @@ -149,22 +149,21 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: unix_sock: The kernel's `unix_sock` object - sock_stat: A tuple with the source, destination and state strings. + sock_stat: A tuple with the source and destination (address and port) along with its state string. """ unix_sock = sock.cast("unix_sock") state = unix_sock.get_state() - saddr = unix_sock.get_name() - sinode = unix_sock.get_inode() - if unix_sock.peer != 0: - peer = unix_sock.peer.dereference().cast("unix_sock") - daddr = peer.get_name() - dinode = peer.get_inode() - else: - daddr = dinode = "" + src_addr = unix_sock.get_name() + src_port = unix_sock.get_inode() - saddr_tag = f"{saddr} {sinode}" - daddr_tag = f"{daddr} {dinode}" - sock_stat = saddr_tag, daddr_tag, state + if unix_sock.peer: + peer = unix_sock.peer.dereference().cast("unix_sock") + dst_addr = peer.get_name() + dst_port = peer.get_inode() + else: + dst_addr = dst_port = None + + sock_stat = src_addr, src_port, dst_addr, dst_port, state return unix_sock, sock_stat def _inet_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: @@ -175,21 +174,16 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: inet_sock: The kernel's `inet_sock` object - sock_stat: A tuple with the source, destination and state strings. + sock_stat: A tuple with the source and destination (address and port) along with its state string. """ inet_sock = sock.cast("inet_sock") - saddr = inet_sock.get_src_addr() - sport = inet_sock.get_src_port() - daddr = inet_sock.get_dst_addr() - dport = inet_sock.get_dst_port() + src_addr = inet_sock.get_src_addr() + src_port = inet_sock.get_src_port() + dst_addr = inet_sock.get_dst_addr() + dst_port = inet_sock.get_dst_port() state = inet_sock.get_state() - if inet_sock.get_family() == "AF_INET6": - saddr = f"[{saddr}]" - - saddr_tag = f"{saddr}:{sport}" - daddr_tag = f"{daddr}:{dport}" - sock_stat = saddr_tag, daddr_tag, state + sock_stat = src_addr, src_port, dst_addr, dst_port, state return inet_sock, sock_stat def _netlink_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: @@ -200,34 +194,26 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: netlink_sock: The kernel's `netlink_sock` object - sock_stat: A tuple with the source, destination and state strings. + sock_stat: A tuple with the source and destination (address and port) along with its state string. """ netlink_sock = sock.cast("netlink_sock") - saddr_list = [] - src_portid = f"portid:{netlink_sock.portid}" - saddr_list.append(src_portid) - if netlink_sock.groups != 0: + src_addr = None + if netlink_sock.groups: groups_bitmap = netlink_sock.groups.dereference() - groups_str = f"groups:0x{groups_bitmap:08x}" - saddr_list.append(groups_str) + src_addr = f"groups:0x{groups_bitmap:08x}" + src_port = netlink_sock.portid - daddr_list = [] - dst_portid = f"portid:{netlink_sock.dst_portid}" - daddr_list.append(dst_portid) - dst_group = f"group:0x{netlink_sock.dst_group:08x}" - daddr_list.append(dst_group) + dst_addr = f"group:0x{netlink_sock.dst_group:08x}" module = netlink_sock.module - if module and netlink_sock.module.name: - module_name_str = utility.array_to_string(netlink_sock.module.name) - module_name = f"lkm:{module_name_str}" - daddr_list.append(module_name) + if module and module.name: + module_name_str = utility.array_to_string(module.name) + dst_addr = f"{dst_addr},lkm:{module_name_str}" + dst_port = netlink_sock.dst_portid - saddr_tag = ",".join(saddr_list) - daddr_tag = ",".join(daddr_list) state = netlink_sock.get_state() - sock_stat = saddr_tag, daddr_tag, state + sock_stat = src_addr, src_port, dst_addr, dst_port, state return netlink_sock, sock_stat def _vsock_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: @@ -238,18 +224,16 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: vsock_sock: The kernel `vsock_sock` object - sock_stat: A tuple with the source, destination and state strings. + sock_stat: A tuple with the source and destination (address and port) along with its state string. """ vsock_sock = sock.cast("vsock_sock") - saddr = vsock_sock.local_addr.svm_cid - sport = vsock_sock.local_addr.svm_port - daddr = vsock_sock.remote_addr.svm_cid - dport = vsock_sock.remote_addr.svm_port + src_addr = vsock_sock.local_addr.svm_cid + src_port = vsock_sock.local_addr.svm_port + dst_addr = vsock_sock.remote_addr.svm_cid + dst_port = vsock_sock.remote_addr.svm_port state = vsock_sock.get_state() - saddr_tag = f"{saddr}:{sport}" - daddr_tag = f"{daddr}:{dport}" - sock_stat = saddr_tag, daddr_tag, state + sock_stat = src_addr, src_port, dst_addr, dst_port, state return vsock_sock, sock_stat def _packet_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: @@ -260,16 +244,17 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: packet_sock: The kernel's `packet_sock` object - sock_stat: A tuple with the source, destination and state strings. + sock_stat: A tuple with the source and destination (address and port) along with its state string. """ packet_sock = sock.cast("packet_sock") ifindex = packet_sock.ifindex - dev_name = self._netdevices.get(ifindex, "") if ifindex > 0 else "ANY" + dev_name = self._netdevices.get(ifindex) if ifindex > 0 else "ANY" - saddr_tag = f"{dev_name}" - daddr_tag = "" + src_addr = dev_name + src_port = dst_addr = dst_port = None state = packet_sock.get_state() - sock_stat = saddr_tag, daddr_tag, state + + sock_stat = src_addr, src_port, dst_addr, dst_port, state return packet_sock, sock_stat def _xdp_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: @@ -280,34 +265,39 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: xdp_sock: The kernel's `xdp_sock` object - sock_stat: A tuple with the source, destination and state strings. + sock_stat: A tuple with the source and destination (address and port) along with its state string. """ xdp_sock = sock.cast("xdp_sock") device = xdp_sock.dev if not device: return - saddr_tag = utility.array_to_string(device.name) + src_addr = utility.array_to_string(device.name) + src_port = dst_addr = dst_port = None bpfprog = device.xdp_prog if not bpfprog: return + if not bpfprog.has_member("aux") or not bpfprog.aux: + return + bpfprog_aux = bpfprog.aux - if bpfprog_aux: + if bpfprog_aux.has_member("id"): + # `id` member was added to `bpf_prog_aux` in kernels 4.13 bpfprog_id = bpfprog_aux.id - daddr_tag = f"ebpf_prog_id:{bpfprog_id}" + dst_port = f"ebpf_prog_id:{bpfprog_id}" + if bpfprog_aux.has_member("name"): + # `name` was added to `bpf_prog_aux` in kernels 4.15 bpf_name = utility.array_to_string(bpfprog_aux.name) if bpf_name: - daddr_tag += f",ebpf_prog_name:{bpf_name}" - else: - daddr_tag = "" + dst_addr = f"ebpf_prog_name:{bpf_name}" # Hallelujah, xdp_sock.state is an enum xsk_state = xdp_sock.state.lookup() state = xsk_state.replace("XSK_", "") - sock_stat = saddr_tag, daddr_tag, state + sock_stat = src_addr, src_port, dst_addr, dst_port, state return xdp_sock, sock_stat def _bluetooth_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: @@ -318,14 +308,14 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: bt_sock: The kernel's `bt_sock` object - sock_stat: A tuple with the source, destination and state strings. + sock_stat: A tuple with the source and destination (address and port) along with its state string. """ bt_sock = sock.cast("bt_sock") def bt_addr(addr): return ":".join(reversed(["%02x" % x for x in addr.b])) - saddr_tag = daddr_tag = "" + src_addr = src_port = dst_addr = dst_port = None bt_protocol = bt_sock.get_protocol() if bt_protocol == "HCI": pinfo = bt_sock.cast("hci_pinfo") @@ -333,20 +323,17 @@ class SockHandlers(interfaces.configuration.VersionableInterface): pinfo = bt_sock.cast("l2cap_pinfo") src_addr = bt_addr(pinfo.chan.src) dst_addr = bt_addr(pinfo.chan.dst) - saddr_tag = f"{src_addr}" - daddr_tag = f"{dst_addr}" elif bt_protocol == "RFCOMM": pinfo = bt_sock.cast("rfcomm_pinfo") src_addr = bt_addr(pinfo.src) dst_addr = bt_addr(pinfo.dst) - channel = pinfo.channel - saddr_tag = f"[{src_addr}]:{channel}" - daddr_tag = f"{dst_addr}" + src_port = pinfo.channel else: vollog.log(constants.LOGLEVEL_V, "Unsupported bluetooth protocol '%s'", bt_protocol) state = bt_sock.get_state() - sock_stat = saddr_tag, daddr_tag, state + + sock_stat = src_addr, src_port, dst_addr, dst_port, state return bt_sock, sock_stat class Sockstat(plugins.PluginInterface): @@ -457,8 +444,10 @@ class Sockstat(plugins.PluginInterface): family: Socket family string (AF_UNIX, AF_INET, etc) sock_type: Socket type string (STREAM, DGRAM, etc) protocol: Protocol string (UDP, TCP, etc) - source: Source address string - destination: Destination address string + source addr: Source address string + source port: Source port string (not all of them are int) + destination addr: Destination address string + destination port: Destination port (not all of them are int) state: State strings (LISTEN, CONNECTED, etc) tasks: String with a list of tasks and FDs using a socket. It can also have extended information such as socket filters, bpf info, etc. @@ -472,6 +461,7 @@ class Sockstat(plugins.PluginInterface): continue sock, sock_stat, extended = sock_fields + sock_stat, protocol = self._format_fields(sock_stat, protocol) task_comm = utility.array_to_string(task.comm) task_info = f"{task_comm},pid={task.pid},fd={fd_num}" @@ -496,6 +486,22 @@ class Sockstat(plugins.PluginInterface): fields = data['fields'] + (tasks,) yield (0, fields) + def _format_fields(self, sock_stat, protocol): + """Prepare the socket fields to be rendered + + Args: + sock_stat: A tuple with the source and destination (address and port) along with its state string. + protocol: Protocol string (UDP, TCP, etc) + + Returns: + `sock_stat` and `protocol` formatted. + """ + sock_stat = [NotAvailableValue() if field is None else str(field) for field in sock_stat] + if protocol is None: + protocol = NotAvailableValue() + + return tuple(sock_stat), protocol + def run(self): pids = self.config.get('pids') netns_id = self.config['netns'] @@ -505,8 +511,10 @@ class Sockstat(plugins.PluginInterface): ("Family", str), ("Type", str), ("Proto", str), - ("Source Addr:Port", str), - ("Destination Addr:Port", str), + ("Source Addr", str), + ("Source Port", str), + ("Destination Addr", str), + ("Destination Port", str), ("State", str), ("Tasks", str)] diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d193200c4..ab0622cf0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -819,16 +819,12 @@ class socket(objects.StructType): socket_state_idx = self.state if 0 <= socket_state_idx < len(SOCKET_STATES): return SOCKET_STATES[socket_state_idx] - else: - return "UNKNOWN" class sock(objects.StructType): def get_family(self): family_idx = self.__sk_common.skc_family if 0 <= family_idx < len(SOCK_FAMILY): return SOCK_FAMILY[family_idx] - else: - return "UNKNOWN" def get_type(self): return SOCK_TYPES.get(self.sk_type, "") @@ -840,7 +836,7 @@ class sock(objects.StructType): return self.sk_socket.get_inode() def get_protocol(self): - return "" + return def get_state(self): # Return the generic socket state @@ -851,15 +847,15 @@ class sock(objects.StructType): class unix_sock(objects.StructType): def get_name(self): - if self.addr: - sockaddr_un = self.addr.name.cast("sockaddr_un") - saddr = str(utility.array_to_string(sockaddr_un.sun_path)) - else: - saddr = "" + if not self.addr: + return + + sockaddr_un = self.addr.name.cast("sockaddr_un") + saddr = str(utility.array_to_string(sockaddr_un.sun_path)) return saddr def get_protocol(self): - return "" + return def get_state(self): """Return a string representing the sock state.""" @@ -869,8 +865,6 @@ class unix_sock(objects.StructType): state_idx = self.sk.__sk_common.skc_state if 0 <= state_idx < len(TCP_STATES): return TCP_STATES[state_idx] - else: - return "UNKNOWN" else: # Return the generic socket state return self.sk.sk_socket.get_state() @@ -883,15 +877,14 @@ class inet_sock(objects.StructType): family_idx = self.sk.__sk_common.skc_family if 0 <= family_idx < len(SOCK_FAMILY): return SOCK_FAMILY[family_idx] - else: - return "UNKNOWN" def get_protocol(self): # If INET6 family and a proto is defined, we use that specific IPv6 protocol. # Otherwise, we use the standard IP protocol. - protocol = IP_PROTOCOLS.get(self.sk.sk_protocol, "UNKNOWN") + protocol = IP_PROTOCOLS.get(self.sk.sk_protocol) if self.get_family() == "AF_INET6": protocol = IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) + return protocol def get_state(self): @@ -901,8 +894,6 @@ class inet_sock(objects.StructType): state_idx = self.sk.__sk_common.skc_state if 0 <= state_idx < len(TCP_STATES): return TCP_STATES[state_idx] - else: - return "UNKNOWN" else: # Return the generic socket state return self.sk.sk_socket.get_state() @@ -949,7 +940,7 @@ class inet_sock(objects.StructType): addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) except exceptions.InvalidAddressException: vollog.debug(f"Unable to read socket src address from {saddr.vol.offset:#x}") - return "?" + return return socket_module.inet_ntop(family, addr_bytes) @@ -978,7 +969,7 @@ class inet_sock(objects.StructType): addr_bytes = parent_layer.read(daddr.vol.offset, addr_size) except exceptions.InvalidAddressException: vollog.debug(f"Unable to read socket dst address from {daddr.vol.offset:#x}") - return "?" + return return socket_module.inet_ntop(family, addr_bytes) @@ -987,8 +978,6 @@ class netlink_sock(objects.StructType): protocol_idx = self.sk.sk_protocol if 0 <= protocol_idx < len(NETLINK_PROTOCOLS): return NETLINK_PROTOCOLS[protocol_idx] - else: - return "UNKNOWN" def get_state(self): # Return the generic socket state @@ -997,10 +986,7 @@ class netlink_sock(objects.StructType): class vsock_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for vsocks - if self.sk.sk_protocol == 0: - return "" - else: - return "UNKNOWN" + return def get_state(self): # Return the generic socket state @@ -1010,7 +996,7 @@ class packet_sock(objects.StructType): def get_protocol(self): eth_proto = socket_module.htons(self.num) if eth_proto == 0: - return "" + return elif eth_proto in ETH_PROTOCOLS: return ETH_PROTOCOLS[eth_proto] else: @@ -1025,23 +1011,16 @@ class bt_sock(objects.StructType): type_idx = self.sk.sk_protocol if 0 <= type_idx < len(BLUETOOTH_PROTOCOLS): return BLUETOOTH_PROTOCOLS[type_idx] - else: - return "UNKNOWN" def get_state(self): state_idx = self.sk.__sk_common.skc_state if 0 <= state_idx < len(BLUETOOTH_STATES): return BLUETOOTH_STATES[state_idx] - else: - return "UNKNOWN" class xdp_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for xdp_sock - if self.sk.sk_protocol == 0: - return "" - else: - return "UNKNOWN" + return def get_state(self): # Return the generic socket state From 25ceccb65b8695d60a6bfd3ecb999803ea128ab1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 31 Oct 2022 15:42:47 +1100 Subject: [PATCH 216/526] Improve and fix issues in bluetooth family. Disaggregate pid, fds, and socket address to new columns. Removed task association by socket address feature. --- .../framework/plugins/linux/sockstat.py | 175 ++++++++++-------- .../symbols/linux/extensions/__init__.py | 4 +- 2 files changed, 96 insertions(+), 83 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index f927c0c83..ad3eee01f 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -6,7 +6,7 @@ import logging from typing import Callable, Tuple, List, Dict from volatility3.framework import interfaces, exceptions, constants, objects -from volatility3.framework.renderers import TreeGrid, NotAvailableValue +from volatility3.framework.renderers import TreeGrid, NotAvailableValue, format_hints from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -72,18 +72,18 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns a tuple with: sock: The respective kernel's *_sock object for that socket family - sock_stat: A tuple with the source and destination (address and port) along with its state string. - extended: A dictionary with key/value extended information. + sock_stat: A tuple with the source and destination (address and port) along with its state string + socket_filter: A dictionary with information about the socket filter """ family = sock.get_family() - extended = {} + socket_filter = {} sock_handler = self._sock_family_handlers.get(family) if sock_handler: try: unix_sock, sock_stat = sock_handler(sock) - self._update_extended_socket_filters_info(sock, extended) + self._update_socket_filters_info(sock, socket_filter) - return unix_sock, sock_stat, extended + return unix_sock, sock_stat, socket_filter except exceptions.SymbolError as e: # Cannot finds the *_sock type in the symbols vollog.log(constants.LOGLEVEL_V, "Error processing socket family '%s': %s", family, e) @@ -98,27 +98,32 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = src_addr, src_port, dst_addr, dst_port, state - return sock, sock_stat, extended + return sock, sock_stat, socket_filter - def _update_extended_socket_filters_info(self, sock: objects.Pointer, extended: dict) -> None: + def _update_socket_filters_info(self, sock: objects.Pointer, socket_filter: dict) -> None: """Get information from the socket and reuseport filters Args: sock: The kernel sock (sk) struct - extended: Dictionary to store extended information + socket_filter: A dictionary with information about the socket filter """ if sock.has_member("sk_filter") and sock.sk_filter: sock_filter = sock.sk_filter - extended["filter_type"] = "socket_filter" - self._extract_socket_filter_info(sock_filter, extended) + socket_filter["filter_type"] = "socket_filter" + self._extract_socket_filter_info(sock_filter, socket_filter) if sock.has_member("sk_reuseport_cb") and sock.sk_reuseport_cb: sock_reuseport_cb = sock.sk_reuseport_cb - extended["filter_type"] = "reuseport_filter" - self._extract_socket_filter_info(sock_reuseport_cb, extended) + socket_filter["filter_type"] = "reuseport_filter" + self._extract_socket_filter_info(sock_reuseport_cb, socket_filter) - def _extract_socket_filter_info(self, sock_filter: objects.Pointer, extended: dict) -> None: - extended["bpf_filter_type"] = "cBPF" + def _extract_socket_filter_info(self, sock_filter: objects.Pointer, socket_filter: dict) -> None: + """Get specific information for each type of filter + + Args: + socket_filter: A dictionary with information about the socket filter + """ + socket_filter["bpf_filter_type"] = "cBPF" if not sock_filter.has_member("prog") or not sock_filter.prog: return @@ -128,18 +133,18 @@ class SockHandlers(interfaces.configuration.VersionableInterface): # BPF_PROG_TYPE_UNSPEC = 0 return - extended["bpf_filter_type"] = "eBPF" + socket_filter["bpf_filter_type"] = "eBPF" if not bpfprog.has_member("aux") or not bpfprog.aux: return bpfprog_aux = bpfprog.aux if bpfprog_aux.has_member("id"): # `id` member was added to `bpf_prog_aux` in kernels 4.13 - extended["bpf_filter_id"] = str(bpfprog_aux.id) + socket_filter["bpf_filter_id"] = str(bpfprog_aux.id) if bpfprog_aux.has_member("name"): # `name` was added to `bpf_prog_aux` in kernels 4.15 bpfprog_name = utility.array_to_string(bpfprog_aux.name) if bpfprog_name: - extended["bpf_filter_name"] = bpfprog_name + socket_filter["bpf_filter_name"] = bpfprog_name def _unix_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: """Handles the AF_UNIX socket family @@ -149,7 +154,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: unix_sock: The kernel's `unix_sock` object - sock_stat: A tuple with the source and destination (address and port) along with its state string. + sock_stat: A tuple with the source and destination (address and port) along with its state string """ unix_sock = sock.cast("unix_sock") state = unix_sock.get_state() @@ -174,7 +179,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: inet_sock: The kernel's `inet_sock` object - sock_stat: A tuple with the source and destination (address and port) along with its state string. + sock_stat: A tuple with the source and destination (address and port) along with its state string """ inet_sock = sock.cast("inet_sock") src_addr = inet_sock.get_src_addr() @@ -194,7 +199,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: netlink_sock: The kernel's `netlink_sock` object - sock_stat: A tuple with the source and destination (address and port) along with its state string. + sock_stat: A tuple with the source and destination (address and port) along with its state string """ netlink_sock = sock.cast("netlink_sock") @@ -224,7 +229,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: vsock_sock: The kernel `vsock_sock` object - sock_stat: A tuple with the source and destination (address and port) along with its state string. + sock_stat: A tuple with the source and destination (address and port) along with its state string """ vsock_sock = sock.cast("vsock_sock") src_addr = vsock_sock.local_addr.svm_cid @@ -244,7 +249,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: packet_sock: The kernel's `packet_sock` object - sock_stat: A tuple with the source and destination (address and port) along with its state string. + sock_stat: A tuple with the source and destination (address and port) along with its state string """ packet_sock = sock.cast("packet_sock") ifindex = packet_sock.ifindex @@ -265,7 +270,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: xdp_sock: The kernel's `xdp_sock` object - sock_stat: A tuple with the source and destination (address and port) along with its state string. + sock_stat: A tuple with the source and destination (address and port) along with its state string """ xdp_sock = sock.cast("xdp_sock") device = xdp_sock.dev @@ -293,8 +298,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if bpf_name: dst_addr = f"ebpf_prog_name:{bpf_name}" - # Hallelujah, xdp_sock.state is an enum - xsk_state = xdp_sock.state.lookup() + xsk_state = xdp_sock.get_state() state = xsk_state.replace("XSK_", "") sock_stat = src_addr, src_port, dst_addr, dst_port, state @@ -308,7 +312,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): Returns: bt_sock: The kernel's `bt_sock` object - sock_stat: A tuple with the source and destination (address and port) along with its state string. + sock_stat: A tuple with the source and destination (address and port) along with its state string """ bt_sock = sock.cast("bt_sock") @@ -318,16 +322,37 @@ class SockHandlers(interfaces.configuration.VersionableInterface): src_addr = src_port = dst_addr = dst_port = None bt_protocol = bt_sock.get_protocol() if bt_protocol == "HCI": - pinfo = bt_sock.cast("hci_pinfo") + if self._vmlinux.has_type("hci_pinfo"): + pinfo = bt_sock.cast("hci_pinfo") + if pinfo.has_member("hdev") and self._vmlinux.has_type("hci_dev") \ + and pinfo.hdev.has_member("dev_name"): + src_addr = utility.array_to_string(pinfo.hdev.dev_name) + else: + vollog.log(constants.LOGLEVEL_V, "Type definition for 'hci_pinfo' is not available in the symbols") elif bt_protocol == "L2CAP": - pinfo = bt_sock.cast("l2cap_pinfo") - src_addr = bt_addr(pinfo.chan.src) - dst_addr = bt_addr(pinfo.chan.dst) + if self._vmlinux.has_type("l2cap_pinfo"): + pinfo = bt_sock.cast("l2cap_pinfo") + src_addr = bt_addr(pinfo.chan.src) + dst_addr = bt_addr(pinfo.chan.dst) + src_port = pinfo.chan.sport + dst_port = pinfo.chan.psm + else: + vollog.log(constants.LOGLEVEL_V, "Type definition for 'l2cap_pinfo' is not available in the symbols") elif bt_protocol == "RFCOMM": - pinfo = bt_sock.cast("rfcomm_pinfo") - src_addr = bt_addr(pinfo.src) - dst_addr = bt_addr(pinfo.dst) - src_port = pinfo.channel + if self._vmlinux.has_type("rfcomm_pinfo"): + pinfo = bt_sock.cast("rfcomm_pinfo") + src_addr = bt_addr(pinfo.src) + dst_addr = bt_addr(pinfo.dst) + src_port = pinfo.channel + else: + vollog.log(constants.LOGLEVEL_V, "Type definition for 'rfcomm_pinfo' is not available in the symbols") + elif bt_protocol == "SCO": + if self._vmlinux.has_type("sco_pinfo"): + pinfo = bt_sock.cast("sco_pinfo") + src_addr = bt_addr(pinfo.src) + dst_addr = bt_addr(pinfo.dst) + else: + vollog.log(constants.LOGLEVEL_V, "Type definition for 'sco_pinfo' is not available in the symbols") else: vollog.log(constants.LOGLEVEL_V, "Unsupported bluetooth protocol '%s'", bt_protocol) @@ -431,6 +456,22 @@ class Sockstat(plugins.PluginInterface): netns_id = net.get_inode() yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields + def _format_fields(self, sock_stat, protocol): + """Prepare the socket fields to be rendered + + Args: + sock_stat: A tuple with the source and destination (address and port) along with its state string + protocol: Protocol string (UDP, TCP, etc) + + Returns: + `sock_stat` and `protocol` formatted. + """ + sock_stat = [NotAvailableValue() if field is None else str(field) for field in sock_stat] + if protocol is None: + protocol = NotAvailableValue() + + return tuple(sock_stat), protocol + def _generator(self, pids: List[int], netns_id_arg: int, symbol_table: str): """Enumerate tasks sockets. Each row represents a kernel socket. @@ -455,7 +496,6 @@ class Sockstat(plugins.PluginInterface): filter_func = lsof.pslist.PsList.create_pid_filter(pids) socket_generator = self.list_sockets(self.context, symbol_table, filter_func=filter_func) - tasks_per_sock = {} for task, netns_id, fd_num, family, sock_type, protocol, sock_fields in socket_generator: if netns_id_arg and netns_id_arg != netns_id: continue @@ -463,59 +503,32 @@ class Sockstat(plugins.PluginInterface): sock, sock_stat, extended = sock_fields sock_stat, protocol = self._format_fields(sock_stat, protocol) - task_comm = utility.array_to_string(task.comm) - task_info = f"{task_comm},pid={task.pid},fd={fd_num}" - if extended: - extended_str = ",".join(f"{k}={v}" for k, v in extended.items()) - task_info = f"{task_info},{extended_str}" + socket_filter_str = ",".join(f"{k}={v}" for k, v in extended.items()) if extended else NotAvailableValue() - fields = netns_id, family, sock_type, protocol, *sock_stat + fields = (netns_id, task.pid, fd_num, format_hints.Hex(sock.vol.offset), + family, sock_type, protocol, *sock_stat, socket_filter_str) - # Each row represents a kernel socket, so let's group the task FDs - # by socket using the socket address - sock_addr = sock.vol.offset - tasks_per_sock.setdefault(sock_addr, {}) - tasks_per_sock[sock_addr].setdefault('tasks', []) - tasks_per_sock[sock_addr]['tasks'].append(task_info) - tasks_per_sock[sock_addr]['fields'] = fields - - for data in tasks_per_sock.values(): - task_list = [f"({task})" for task in data['tasks']] - tasks = ",".join(task_list) - - fields = data['fields'] + (tasks,) yield (0, fields) - def _format_fields(self, sock_stat, protocol): - """Prepare the socket fields to be rendered - - Args: - sock_stat: A tuple with the source and destination (address and port) along with its state string. - protocol: Protocol string (UDP, TCP, etc) - - Returns: - `sock_stat` and `protocol` formatted. - """ - sock_stat = [NotAvailableValue() if field is None else str(field) for field in sock_stat] - if protocol is None: - protocol = NotAvailableValue() - - return tuple(sock_stat), protocol - def run(self): pids = self.config.get('pids') netns_id = self.config['netns'] symbol_table = self.config['kernel'] - tree_grid_args = [("NetNS", int), - ("Family", str), - ("Type", str), - ("Proto", str), - ("Source Addr", str), - ("Source Port", str), - ("Destination Addr", str), - ("Destination Port", str), - ("State", str), - ("Tasks", str)] + tree_grid_args = [ + ("NetNS", int), + ("Pid", int), + ("FD", int), + ("Sock Offset", format_hints.Hex), + ("Family", str), + ("Type", str), + ("Proto", str), + ("Source Addr", str), + ("Source Port", str), + ("Destination Addr", str), + ("Destination Port", str), + ("State", str), + ("Filter", str), + ] return TreeGrid(tree_grid_args, self._generator(pids, netns_id, symbol_table)) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index ab0622cf0..c26d68b0e 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1023,5 +1023,5 @@ class xdp_sock(objects.StructType): return def get_state(self): - # Return the generic socket state - return self.sk.sk_socket.get_state() + # xdp_sock.state is an enum + return self.state.lookup() From aa0c2b6c744486bbb7135e754b47bf1dc60e7360 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 2 Nov 2022 20:53:00 +0000 Subject: [PATCH 217/526] Mac: Fix bug found by buildbot/npetroni due refactoring --- volatility3/framework/symbols/mac/extensions/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index a66bfb534..92410704f 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -28,8 +28,11 @@ class proc(generic.GenericIntelProcess): if not isinstance(parent_layer, interfaces.layers.TranslationLayerInterface): raise TypeError("Parent layer is not a translation layer, unable to construct process layer") - with contextlib.suppress(exceptions.InvalidAddressException): + try: dtb = self.get_task().map.pmap.pm_cr3 + except exceptions.InvalidAddressException: + # Bail out because we couldn't find the DTB + return None if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.p_pid}" From d09f23a7d7a791c6e846f401de7f1168326e34ee Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 2 Nov 2022 20:55:24 +0000 Subject: [PATCH 218/526] Mac: Fix additional possibility of failure from refactoring --- volatility3/framework/symbols/mac/extensions/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 92410704f..45dc1db70 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -41,10 +41,8 @@ class proc(generic.GenericIntelProcess): return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - with contextlib.suppress(exceptions.InvalidAddressException): - task = self.get_task() - try: + task = self.get_task() current_map = task.map.hdr.links.next except exceptions.InvalidAddressException: return From 0c80ae4f816281541e177017f9e2e1e518a78b3e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 2 Nov 2022 21:39:27 +0000 Subject: [PATCH 219/526] Automagic: Check file datetime to determine whether to recache --- .../framework/automagic/symbol_cache.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 30a4068b6..fe5dfac52 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import base64 +import datetime import json import logging import os @@ -170,6 +171,7 @@ class SqliteCache(CacheManagerInterface): def _connect_storage(self, path: str) -> sqlite3.Connection: database = sqlite3.connect(path) database.row_factory = sqlite3.Row + database.cursor().execute( f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})') schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() @@ -259,10 +261,31 @@ class SqliteCache(CacheManagerInterface): cache_update = set() files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: - result = self._database.cursor().execute("SELECT location FROM cache WHERE local = 1 " + result = self._database.cursor().execute("SELECT location, cached FROM cache WHERE local = 1 " f"AND cached < date('now', '{self.cache_period}');") for row in result: - if row['location'] in files_to_timestamp: + location = row['location'] + stored_timestamp = datetime.datetime.fromisoformat(row['cached']) + timestamp = stored_timestamp # Default to requiring update + + # See if the file is a local URL type we can handle: + parsed = urllib.parse.urlparse(location) + pathname = None + if parsed.scheme == 'file': + pathname = urllib.request.url2pathname(parsed.path) + if parsed.scheme == 'jar': + inner_url = urllib.parse.urlparse(parsed.path) + if inner_url.scheme == 'file': + pathname = inner_url.path.split('!')[0] + + if pathname: + timestamp = datetime.datetime.fromtimestamp(os.stat(pathname).st_mtime) + else: + vollog.log(constants.LOGLEVEL_VVVV, + "File location in database classed as local but not file/jar URL") + + # If we're supposed to include it, and our last check is older than (or equal to) the file timestamp + if row['location'] in files_to_timestamp and stored_timestamp < timestamp: cache_update.add(row['location']) idextractors = list(framework.class_subclasses(IdentifierProcessor)) From 5ac191b31008a1e678a77839cb2aed489310691a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 2 Nov 2022 21:43:18 +0000 Subject: [PATCH 220/526] Automagic: Set the cache period back to 3 days --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 4fd53a3eb..b19e80472 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -63,7 +63,7 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" -SQLITE_CACHE_PERIOD = '-1 month' +SQLITE_CACHE_PERIOD = '-3 days' """SQLite time modifier for how long each item is valid in the cache for""" if sys.platform == 'win32': From 380b76a90d7012937749b206163668abdc13e53c Mon Sep 17 00:00:00 2001 From: ikelos Date: Thu, 3 Nov 2022 01:05:40 +0000 Subject: [PATCH 221/526] Apply suggestions from code review Co-authored-by: Donghyun Kim --- volatility3/framework/plugins/windows/drivermodule.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/drivermodule.py b/volatility3/framework/plugins/windows/drivermodule.py index 54a132adb..1a97edb80 100644 --- a/volatility3/framework/plugins/windows/drivermodule.py +++ b/volatility3/framework/plugins/windows/drivermodule.py @@ -19,9 +19,10 @@ class DriverModule(interfaces.plugins.PluginInterface): """Determines if any loaded drivers were hidden by a rootkit""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod - def get_requirements(cls): + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), @@ -29,7 +30,7 @@ class DriverModule(interfaces.plugins.PluginInterface): requirements.PluginRequirement(name = 'driverscan', plugin = driverscan.DriverScan, version = (1, 0, 0)), ] - def _generator(self): + def _generator(self) -> Iterator[Tuple]: """ Attempt to match each driver's start code address to a known kernel module A common rootkit technique is to register drivers from modules that are hidden, @@ -49,12 +50,12 @@ class DriverModule(interfaces.plugins.PluginInterface): yield (0, (format_hints.Hex(driver.vol.offset), known_exception, driver_name, service_key, name)) - def run(self): + def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([ ("Offset", format_hints.Hex), ("Known Exception", bool), ("Driver Name", str), - ("Serivce Key", str), + ("Service Key", str), ("Alternative Name", str), ], self._generator()) From c1dcfe8b570bcaad184721b9b244ccfe5a6e2c3f Mon Sep 17 00:00:00 2001 From: ikelos Date: Thu, 3 Nov 2022 01:11:18 +0000 Subject: [PATCH 222/526] Update volatility3/framework/plugins/windows/drivermodule.py --- volatility3/framework/plugins/windows/drivermodule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/drivermodule.py b/volatility3/framework/plugins/windows/drivermodule.py index 1a97edb80..31f4711b1 100644 --- a/volatility3/framework/plugins/windows/drivermodule.py +++ b/volatility3/framework/plugins/windows/drivermodule.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +from typing import Iterator, List, Tuple from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints From 5bbec2d7c4e7330d065e2b3a6e08df964856945b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 7 Nov 2022 20:46:30 +0000 Subject: [PATCH 223/526] Core: Bump to 2.4.1 in preparation for 2.4.0 release --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index b19e80472..95b365609 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 4 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From e3c548686c59abadddfc135ba5c9f3ecd32adc9d Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 13 Nov 2022 12:16:27 +0000 Subject: [PATCH 224/526] Create codeql.yml Shift from LGTM.com over to built-in github codeql analysis. --- .github/workflows/codeql.yml | 74 ++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..fcefcfa96 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,74 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "develop" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "develop" ] + schedule: + - cron: '16 8 * * 0' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" From 297e1c9e81b8e7f415ce370e2fff148e281955a6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 12:56:30 +0000 Subject: [PATCH 225/526] Include code quality alerts in CodeQL scans --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index fcefcfa96..72bba07aa 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -50,7 +50,7 @@ jobs: # Prefix the list here with "+" to use these queries and those in the config file. # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality + queries: security-and-quality # ,security-extended # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). From 3f5fd3502d8cfcd97816cd058049b04a9951a54a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 13:34:39 +0000 Subject: [PATCH 226/526] Infra: Update the bug_report template to favour text over screenshots --- .github/ISSUE_TEMPLATE/bug_report.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 3a0cce8cc..2ccd4713c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -23,8 +23,10 @@ Steps to reproduce the behavior: **Expected behavior** A clear and concise description of what you expected to happen. -**Screenshots** -If applicable, add screenshots to help explain your problem. +**Example output** +Please copy and paste the text demonstrating the issue, ideally with verbose output turned on (`vol.py -vvv ...`). + +Text is preferred to screenshots for searching and to talk about specific parts of the output. **Additional information** Add any other information about the problem here. From 324df0927534fd48fa61835c499bd0583b60c1cf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 13:46:59 +0000 Subject: [PATCH 227/526] Core: Fix code scanning warnings notes --- volatility3/framework/objects/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index eedd22bb2..4334f9d74 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -611,12 +611,10 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): @overload - def __getitem__(self, i: int) -> interfaces.objects.Template: - ... + def __getitem__(self, i: int) -> interfaces.objects.Template: ... @overload - def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: - ... + def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: ... def __getitem__(self, i): """Returns the i-th item from the array.""" From 129b92e3eedfbc71b1b1570e0d01c4c22e4a6fc1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 14:48:33 +0000 Subject: [PATCH 228/526] Core: Fix code scanning warnings/notes --- development/mac-kdk/parse_pbzx2.py | 84 +++++++++---------- doc/source/conf.py | 1 + volatility3/cli/volshell/__init__.py | 1 - volatility3/cli/volshell/generic.py | 8 +- volatility3/framework/automagic/pdbscan.py | 1 + .../framework/automagic/symbol_cache.py | 4 +- volatility3/framework/interfaces/automagic.py | 2 +- volatility3/framework/layers/resources.py | 1 + .../framework/plugins/linux/check_modules.py | 2 +- volatility3/framework/plugins/mac/lsmod.py | 5 +- .../framework/plugins/windows/cachedump.py | 14 ++-- .../framework/plugins/windows/netstat.py | 2 +- .../framework/plugins/windows/pslist.py | 4 +- .../framework/plugins/windows/psscan.py | 3 +- volatility3/framework/symbols/intermed.py | 7 +- .../symbols/linux/extensions/__init__.py | 1 + volatility3/framework/symbols/metadata.py | 4 +- .../symbols/windows/extensions/__init__.py | 3 +- .../plugins/windows/registry/certificates.py | 22 ++--- 19 files changed, 86 insertions(+), 83 deletions(-) diff --git a/development/mac-kdk/parse_pbzx2.py b/development/mac-kdk/parse_pbzx2.py index 7ce9090d4..5e56c9933 100644 --- a/development/mac-kdk/parse_pbzx2.py +++ b/development/mac-kdk/parse_pbzx2.py @@ -22,53 +22,49 @@ def seekread(f, offset = None, length = 0, relative = True): def parse_pbzx(pbzx_path): section = 0 xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) - f = open(pbzx_path, 'rb') - # pbzx = f.read() - # f.close() - magic = seekread(f, length = 4) - if magic != 'pbzx': - raise RuntimeError("Error: Not a pbzx file") - # Read 8 bytes for initial flags - flags = seekread(f, length = 8) - # Interpret the flags as a 64-bit big-endian unsigned int - flags = struct.unpack('>Q', flags)[0] - xar_f = open(xar_out_path, 'wb') - while flags & (1 << 24): - # Read in more flags + with open(pbzx_path, 'rb') as f: + # pbzx = f.read() + # f.close() + magic = seekread(f, length = 4) + if magic != 'pbzx': + raise RuntimeError("Error: Not a pbzx file") + # Read 8 bytes for initial flags flags = seekread(f, length = 8) + # Interpret the flags as a 64-bit big-endian unsigned int flags = struct.unpack('>Q', flags)[0] - # Read in length - f_length = seekread(f, length = 8) - f_length = struct.unpack('>Q', f_length)[0] - xzmagic = seekread(f, length = 6) - if xzmagic != '\xfd7zXZ\x00': - # This isn't xz content, this is actually _raw decompressed cpio_ chunk of 16MB in size... - # Let's back up ... - seekread(f, offset = -6, length = 0) - # ... and split it out ... - f_content = seekread(f, length = f_length) - section += 1 - decomp_out = '%s.part%02d.cpio' % (pbzx_path, section) - g = open(decomp_out, 'wb') - g.write(f_content) - g.close() - # Now to start the next section, which should hopefully be .xz (we'll just assume it is ...) - xar_f.close() - section += 1 - new_out = '%s.part%02d.cpio.xz' % (pbzx_path, section) - xar_f = open(new_out, 'wb') - else: - f_length -= 6 - # This part needs buffering - f_content = seekread(f, length = f_length) - tail = seekread(f, offset = -2, length = 2) - xar_f.write(xzmagic) - xar_f.write(f_content) - if tail != 'YZ': - xar_f.close() - raise RuntimeError("Error: Footer is not xar file footer") + while flags & (1 << 24): + with open(xar_out_path, 'wb') as xar_f: + xar_f.seek(0, os.SEEK_END) + # Read in more flags + flags = seekread(f, length = 8) + flags = struct.unpack('>Q', flags)[0] + # Read in length + f_length = seekread(f, length = 8) + f_length = struct.unpack('>Q', f_length)[0] + xzmagic = seekread(f, length = 6) + if xzmagic != '\xfd7zXZ\x00': + # This isn't xz content, this is actually _raw decompressed cpio_ chunk of 16MB in size... + # Let's back up ... + seekread(f, offset = -6, length = 0) + # ... and split it out ... + f_content = seekread(f, length = f_length) + section += 1 + decomp_out = '%s.part%02d.cpio' % (pbzx_path, section) + with open(decomp_out, 'wb') as g: + g.write(f_content) + # Now to start the next section, which should hopefully be .xz (we'll just assume it is ...) + section += 1 + xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) + else: + f_length -= 6 + # This part needs buffering + f_content = seekread(f, length = f_length) + tail = seekread(f, offset = -2, length = 2) + xar_f.write(xzmagic) + xar_f.write(f_content) + if tail != 'YZ': + raise RuntimeError("Error: Footer is not xar file footer") try: - f.close() xar_f.close() except IOError: pass diff --git a/doc/source/conf.py b/doc/source/conf.py index cadf6d3f2..895219b25 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -121,6 +121,7 @@ try: extensions.append('sphinx_autodoc_typehints') except ImportError: + # If the autodoc typehints extension isn't available, carry on regardless pass # Add any paths that contain templates here, relative to this directory. diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 5eeef77cf..f32a587e0 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -36,7 +36,6 @@ class VolShell(cli.CommandLine): def __init__(self): super().__init__() - self.output_dir = None def run(self): """Executes the command line module, taking the system arguments, diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 19e263a03..274b6ca17 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -324,11 +324,11 @@ class Volshell(interfaces.plugins.PluginInterface): " " * (longest_member - len_member), " ", member_type.vol.type_name) @classmethod - def _display_value(self, value: Any) -> str: + def _display_value(cls, value: Any) -> str: if isinstance(value, objects.PrimitiveObject): return repr(value) elif isinstance(value, objects.Array): - return repr([self._display_value(val) for val in value]) + return repr([cls._display_value(val) for val in value]) else: return hex(value.vol.offset) @@ -390,8 +390,8 @@ class Volshell(interfaces.plugins.PluginInterface): location = "file:" + request.pathname2url(location) print(f"Running code from {location}\n") accessor = resources.ResourceAccessor() - with io.TextIOWrapper(accessor.open(url = location), encoding = 'utf-8') as fp: - self.__console.runsource(fp.read(), symbol = 'exec') + with accessor.open(url = location) as fp: + self.__console.runsource(io.TextIOWrapper(fp.read(), encoding = 'utf-8'), symbol = 'exec') print("\nCode complete") def load_file(self, location: str): diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 5cbdbfe0e..36288ef90 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -181,6 +181,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): hex(kvo))) except exceptions.InvalidAddressException: vollog.debug(f"Potential kernel_virtual_offset caused a page fault: {hex(kvo)}") + return None vollog.debug("Kernel base determination - testing fixed base address") return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, False, True, progress_callback) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index fe5dfac52..164021340 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -223,8 +223,7 @@ class SqliteCache(CacheManagerInterface): def is_url_local(self, url: str) -> bool: """Determines whether an url is local or not""" parsed = urllib.parse.urlparse(url) - if parsed.scheme in ['file', 'jar']: - return True + return parsed.scheme in ['file', 'jar'] def get_identifier(self, location: str) -> Optional[bytes]: results = self._database.cursor().execute('SELECT identifier FROM cache WHERE location = ?', @@ -246,6 +245,7 @@ class SqliteCache(CacheManagerInterface): (location,)).fetchall() for row in results: return row['hash'] + return None def update(self, progress_callback = None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index 713f91da0..4885645c3 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -113,7 +113,7 @@ class StackerLayerInterface(metaclass = ABCMeta): """The list operating systems/first-level plugin hierarchy that should exclude this stacker""" @classmethod - def stack(self, + def stack(cls, context: interfaces.context.ContextInterface, layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index dca215c85..73f59bdbd 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -31,6 +31,7 @@ try: # Import so that the handler is found by the framework.class_subclasses callc import smb.SMBHandler # lgtm [py/unused-import] except ImportError: + # If we fail to import this, it means that SMB handling won't be available pass vollog = logging.getLogger(__name__) diff --git a/volatility3/framework/plugins/linux/check_modules.py b/volatility3/framework/plugins/linux/check_modules.py index 6af8dec96..2c478cebf 100644 --- a/volatility3/framework/plugins/linux/check_modules.py +++ b/volatility3/framework/plugins/linux/check_modules.py @@ -29,7 +29,7 @@ class Check_modules(plugins.PluginInterface): ] @classmethod - def get_kset_modules(self, context: interfaces.context.ContextInterface, vmlinux_name: str): + def get_kset_modules(cls, context: interfaces.context.ContextInterface, vmlinux_name: str): vmlinux = context.modules[vmlinux_name] diff --git a/volatility3/framework/plugins/mac/lsmod.py b/volatility3/framework/plugins/mac/lsmod.py index 095fbc663..345267fea 100644 --- a/volatility3/framework/plugins/mac/lsmod.py +++ b/volatility3/framework/plugins/mac/lsmod.py @@ -46,14 +46,14 @@ class Lsmod(plugins.PluginInterface): try: kmod = kmod_ptr.dereference().cast("kmod_info") except exceptions.InvalidAddressException: - return [] + return # Generation finished yield kmod try: kmod = kmod.next except exceptions.InvalidAddressException: - return [] + return # Generation finished seen: Set = set() @@ -74,6 +74,7 @@ class Lsmod(plugins.PluginInterface): kmod = kmod.next except exceptions.InvalidAddressException: return + return # Generation finished def _generator(self): for module in self.list_modules(self.context, self.config['kernel']): diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index f77c6257b..59ea656f3 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -83,6 +83,13 @@ class Cachedump(interfaces.plugins.PluginInterface): return (username, domain, domain_name, hashh) def _generator(self, syshive, sechive): + if not syshive or not sechive: + if syshive is None: + vollog.warning('Unable to locate SYSTEM hive') + if sechive is None: + vollog.warning('Unable to locate SECURITY hive') + return + bootkey = hashdump.Hashdump.get_bootkey(syshive) if not bootkey: vollog.warning('Unable to find bootkey') @@ -142,12 +149,5 @@ class Cachedump(interfaces.plugins.PluginInterface): if hive.get_name().split('\\')[-1].upper() == 'SECURITY': sechive = hive - if syshive is None or sechive is None: - if syshive is None: - vollog.warning('Unable to locate SYSTEM hive') - if sechive is None: - vollog.warning('Unable to locate SECURITY hive') - return - return renderers.TreeGrid([("Username", str), ("Domain", str), ("Domain name", str), ('Hash', bytes)], self._generator(syshive, sechive)) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 93ac3af93..f7b285f05 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -42,7 +42,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ] @classmethod - def _decode_pointer(self, value): + def _decode_pointer(cls, value): """Copied from `windows.handles`. Windows encodes pointers to objects and decodes them on the fly diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index cadccc5f1..6023d5f04 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -62,7 +62,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ file_handle = None + proc_id = 'Invalid process object' try: + proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() peb = context.object(kernel_table_name + constants.BANG + "_PEB", layer_name = proc_layer_name, @@ -76,7 +78,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_handle.seek(offset) file_handle.write(data) except Exception as excp: - vollog.debug(f"Unable to dump PE with pid {proc.UniqueProcessId}: {excp}") + vollog.debug(f"Unable to dump PE with pid {proc_id}: {excp}") return file_handle diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 335624672..9e3366dff 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -78,7 +78,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): layer_name: str, symbol_table: str, proc: interfaces.objects.ObjectInterface) -> \ - Iterable[interfaces.objects.ObjectInterface]: + Optional[interfaces.objects.ObjectInterface]: """ Returns a virtual process from a physical addressed one Args: @@ -124,6 +124,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if virtual_process and \ proc.vol.offset == ph_offset: return virtual_process + return None @classmethod def get_osversion(cls, context: interfaces.context.ContextInterface, layer_name: str, diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 1fceb1bcc..3fde0978d 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -102,10 +102,9 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): # Check there are no obvious errors # Open the file and test the version self._versions = dict([(x.version, x) for x in class_subclasses(ISFormatTable)]) - fp = resources.ResourceAccessor().open(isf_url) - reader = codecs.getreader("utf-8") - json_object = json.load(reader(fp)) # type: ignore - fp.close() + with resources.ResourceAccessor().open(isf_url) as fp: + reader = codecs.getreader("utf-8") + json_object = json.load(reader(fp)) # type: ignore # Validation is expensive, but we cache to store the hashes of successfully validated json objects if validate and not schemas.validate(json_object): diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 73f31115a..b47013c5d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -128,6 +128,7 @@ class module(generic.GenericIntelProcess): sym_addr = sym.st_value if wanted_sym_name == sym_name: return sym_addr + return # Generation finished @property def section_symtab(self): diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 350bb0a53..f42ac78fe 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Optional, Tuple +from typing import Optional, Tuple, Union from volatility3.framework import interfaces @@ -11,7 +11,7 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): """Class to handle the metadata from a Windows symbol table.""" @property - def pe_version(self) -> Optional[Tuple]: + def pe_version(self) -> Optional[Union[Tuple[int, int, int], Tuple[int, int, int, int]]]: build = self._json_data.get('pe', {}).get('build', None) revision = self._json_data.get('pe', {}).get('revision', None) minor = self._json_data.get('pe', {}).get('minor', None) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index a86f5b3cb..87e8e0f45 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -719,7 +719,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): envars = context.layers[process_space].read(block, block_size).decode("utf-16-le", errors = 'replace').split('\x00')[:-1] except exceptions.InvalidAddressException: - return renderers.UnreadableValue() + return # Generation finished for envar in envars: split_index = envar.find('=') @@ -729,6 +729,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): # Exclude parse problem with some types of env if env and var: yield env, var + return # Generation finished class LIST_ENTRY(objects.StructType, collections.abc.Iterable): diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 6029c0a5c..94962bc81 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -3,7 +3,7 @@ import logging import struct from typing import List, Iterator, Optional, Tuple, Type -from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey @@ -46,14 +46,13 @@ class Certificates(interfaces.plugins.PluginInterface): open_method: Type[interfaces.plugins.FileHandlerInterface]) -> \ Optional[interfaces.plugins.FileHandlerInterface]: try: - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) - file_handle = open_method(dump_name) - file_handle.write(certificate_data) - return file_handle + dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) + file_handle = open_method(dump_name) + file_handle.write(certificate_data) + return file_handle except exceptions.InvalidAddressException: - vollog.debug(f"Unable to certificate file at {hive_offset:#x}") - return None + vollog.debug(f"Unable to dump certificate file at {hive_offset:#x}") + return None def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: @@ -79,9 +78,10 @@ class Certificates(interfaces.plugins.PluginInterface): key_hash = key_path[key_path.rindex("\\") + 1:] if self.config['dump']: - file_handle = self.dump_certificate(certificate_data, hive.hive_offset, reg_section, key_hash, self.open) - if file_handle: - file_handle.close() + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + file_handle = self.dump_certificate(certificate_data, hive.hive_offset, reg_section, key_hash, self.open) + if file_handle: + file_handle.close() yield (0, (top_key, reg_section, key_hash, name)) From 374960f6db64109971964a3e18381016650a3087 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 14:53:39 +0000 Subject: [PATCH 229/526] Windows: Fix bad use of strip Close #867 --- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index d74b21b60..25911e376 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -359,7 +359,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - module_name = guid["pdb_name"].strip('.pdb') + module_name = guid["pdb_name"].replace('.pdb', '') symbol_table_name = cls.load_windows_symbol_table(context, guid["GUID"], From b98a311688741d343a8711d2df657fd87b989352 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 15:00:40 +0000 Subject: [PATCH 230/526] Core: Fix up recent typing changes --- volatility3/framework/plugins/windows/psscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 9e3366dff..00f96ff63 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Iterable, Callable, Tuple +from typing import Iterable, Callable, Optional, Tuple from volatility3.framework import renderers, interfaces, layers, exceptions from volatility3.framework.configuration import requirements From 9b537678a5e1c252852f4aafe6dc669582c047f9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 15:52:09 +0000 Subject: [PATCH 231/526] Core: Fix up more scanning notes/warnings --- development/mac-kdk/parse_pbzx2.py | 5 +---- volatility3/framework/layers/vmware.py | 3 ++- volatility3/framework/objects/__init__.py | 6 ++---- .../framework/plugins/windows/registry/userassist.py | 6 +++++- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- volatility3/framework/symbols/windows/pdbconv.py | 1 + 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/development/mac-kdk/parse_pbzx2.py b/development/mac-kdk/parse_pbzx2.py index 5e56c9933..173a4d648 100644 --- a/development/mac-kdk/parse_pbzx2.py +++ b/development/mac-kdk/parse_pbzx2.py @@ -17,6 +17,7 @@ def seekread(f, offset = None, length = 0, relative = True): f.seek(offset, [0, 1, 2][relative]) if length: return f.read(length) + return None def parse_pbzx(pbzx_path): @@ -64,10 +65,6 @@ def parse_pbzx(pbzx_path): xar_f.write(f_content) if tail != 'YZ': raise RuntimeError("Error: Footer is not xar file footer") - try: - xar_f.close() - except IOError: - pass def main(): diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index ae4a7d55e..61b13eb88 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -154,7 +154,8 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): vmss_success = False with contextlib.suppress(IOError): - _ = resources.ResourceAccessor().open(vmss).read(10) + with resources.ResourceAccessor().open(vmss) as fp: + _ = fp.read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmss_success = True diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 4334f9d74..2b026ccd1 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -747,10 +747,8 @@ class AggregateType(interfaces.objects.ObjectInterface): if isinstance(cls, agg_type): agg_name = agg_type.__name__ - assert isinstance(members, collections.abc.Mapping) - f"{agg_name} members parameter must be a mapping: {type(members)}" - assert all([(isinstance(member, tuple) and len(member) == 2) for member in members.values()]) - f"{agg_name} members must be a tuple of relative_offsets and templates" + assert isinstance(members, collections.abc.Mapping), f"{agg_name} members parameter must be a mapping: {type(members)}" + assert all([(isinstance(member, tuple) and len(member) == 2) for member in members.values()]), f"{agg_name} members must be a tuple of relative_offsets and templates" def member(self, attr: str = 'member') -> object: """Specifically named method for retrieving members.""" diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 30b5db695..f31b7832e 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -33,7 +33,11 @@ class UserAssist(interfaces.plugins.PluginInterface): self._reg_table_name = None self._win7 = None # taken from http://msdn.microsoft.com/en-us/library/dd378457%28v=vs.85%29.aspx - self._folder_guids = json.load(open(os.path.join(os.path.dirname(__file__), "userassist.json"), "rb")) + try: + with open(os.path.join(os.path.dirname(__file__), "userassist.json"), "rb") as fp: + self._folder_guids = json.load(fp) + except IOError: + vollog.error("Usersassist data file not found") @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b47013c5d..ce002b905 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -128,7 +128,7 @@ class module(generic.GenericIntelProcess): sym_addr = sym.st_value if wanted_sym_name == sym_name: return sym_addr - return # Generation finished + return None @property def section_symtab(self): diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index 15b5c733a..d7d171ce1 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -934,6 +934,7 @@ class PdbRetreiver: if progress_callback is not None: progress_callback(100, f"Downloading {url + suffix}") if result is None: + result.close() return None return url + suffix From 8918b385a033c6d3adc80b77f1df1db0540395f7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 15:56:50 +0000 Subject: [PATCH 232/526] Windows: Improve nestat error checking Should partially solve #863 --- volatility3/framework/plugins/windows/netstat.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index f7b285f05..651ca7696 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -427,6 +427,8 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self.config_path) tcpip_module = self.get_tcpip_module(self.context, kernel.layer_name, kernel.symbol_table_name) + if not tcpip_module: + vollog.error("Unable to locate symbols for the memory image's tcpip module") try: tcpip_symbol_table = pdbutil.PDBUtility.symbol_table_from_pdb( From dd876ae18c376e9091d72679272974cfa19d6a53 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 16:27:48 +0000 Subject: [PATCH 233/526] Core: Put the dev requirement back in the dev requirements file --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3ff7c50b8..7c372da2a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -16,7 +16,7 @@ pycryptodome # This can improve error messages regarding improperly configured ISF files, # but is only recommended for development -# jsonschema>=2.3.0 +jsonschema>=2.3.0 # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 From 60ec8a39fff6bc53da653adf2c54f87c3a0d20f9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 18:37:19 +0000 Subject: [PATCH 234/526] Core: Fix another github scanning issue. --- volatility3/framework/symbols/windows/pdbconv.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index d7d171ce1..4809aafcf 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -926,15 +926,16 @@ class PdbRetreiver: try: vollog.debug(f"Attempting to retrieve {url + suffix}") # We have to cache this because the file is opened by a layer and we can't control whether that caches - result = resources.ResourceAccessor(progress_callback).open(url + suffix) + with resources.ResourceAccessor(progress_callback).open(url + suffix) as fp: + fp.read(10) + result = True except (error.HTTPError, error.URLError) as excp: vollog.debug(f"Failed with {excp}") if result: break if progress_callback is not None: progress_callback(100, f"Downloading {url + suffix}") - if result is None: - result.close() + if not result: return None return url + suffix From 364a6a75f94c6c175ee03ea03bc40559d7ae4c5c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 14:53:39 +0000 Subject: [PATCH 235/526] Windows: Fix bad use of strip Close #867 --- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 2fa9bd591..569af6276 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -345,7 +345,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - module_name = guid["pdb_name"].strip('.pdb') + module_name = guid["pdb_name"].replace('.pdb', '') symbol_table_name = cls.load_windows_symbol_table(context, guid["GUID"], From b5c3ab171e4d3554e39efd8a07dbd9f5dceafccc Mon Sep 17 00:00:00 2001 From: cpuu Date: Sat, 19 Nov 2022 15:01:39 +0900 Subject: [PATCH 236/526] Add macos tutorial --- doc/source/getting-started-macos-tutorial.rst | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 doc/source/getting-started-macos-tutorial.rst diff --git a/doc/source/getting-started-macos-tutorial.rst b/doc/source/getting-started-macos-tutorial.rst new file mode 100644 index 000000000..6dd144ac3 --- /dev/null +++ b/doc/source/getting-started-macos-tutorial.rst @@ -0,0 +1,152 @@ +macOS Tutorial +============== + +This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite. + +Acquiring memory +---------------- + +Volatility3 does not provide the ability to acquire memory. The example below is an open source tool. Other commercial tools are also available. + +* `osxpmem `_ + + + +Procedure to create symbol tables for macOS +-------------------------------------------- + +To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. + +.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , + which is built and maintained by `volatilityfoundation `_. + After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/mac``. + If necessary create a mac directory under the symbols directory (this will become unnecessary in future versions). + + +Listing plugins +--------------- + +The following is a sample of the macOS plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. + +.. code-block:: shell-session + + $ python3 vol.py --help | grep -i mac. | head -n 5 + mac.bash.Bash Recovers bash command history from memory. + mac.check_syscall.Check_syscall + mac.check_sysctl.Check_sysctl + mac.check_trap_table.Check_trap_table + +.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of macOS plugins. + + +Using plugins +------------- + +The following is the syntax to run the volatility CLI. + +.. code-block:: shell-session + + $ python3 vol.py -f + + +Example +------- + +banners +~~~~~~~ + +In this example we will be using a memory dump from the Securinets CTF Quals 2019 Challenge called Contact_me. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge. +Thanks go to `stuxnet `_ for providing this memory dump and `writeup `_. + + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me banners.Banners + + Volatility 3 Framework 2.1.0 + + Progress: 100.00 PDB scanning finished + Offset Banner + + 0x4d2c7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0xb42b180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0xcda9100 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x1275e7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x1284fba4 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x34ad0180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + + +The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols/mac`` directory. + +linux.pslist +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.pslist + + Volatility 3 Framework 2.1.0 Stacking attempts finished + + PID PPID COMM + + 0 0 kernel_task + 1 0 launchd + 35 1 UserEventAgent + 38 1 kextd + 39 1 fseventsd + 37 1 uninstalld + 45 1 configd + 46 1 powerd + 52 1 logd + 58 1 warmd + ..... + +``mac.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. + +mac.pstree +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.pstree + Volatility 3 Framework 2.1.0 + Progress: 100.00 Stacking attempts finished + PID PPID COMM + + 35 1 UserEventAgent + 38 1 kextd + 39 1 fseventsd + 37 1 uninstalld + 204 1 softwareupdated + * 449 204 SoftwareUpdateCo + 337 1 system_installd + * 455 337 update_dyld_shar + +``mac.pstree`` helps us to display the parent child relationships between processes. + +mac.ifconfig +~~~~~~~~~~ + +we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.ifconfig + + Volatility 3 Framework 2.1.0 + Progress: 100.00 Stacking attempts finished + Interface IP Address Mac Address Promiscuous + + lo0 False + lo0 127.0.0.1 False + lo0 ::1 False + lo0 fe80:1::1 False + gif0 False + stf0 False + en0 00:0C:29:89:8B:F0 00:0C:29:89:8B:F0 False + en0 fe80:4::10fb:c89d:217f:52ae 00:0C:29:89:8B:F0 False + en0 192.168.140.128 00:0C:29:89:8B:F0 False + utun0 False + utun0 fe80:5::2a95:bb15:87e3:977c False From 17bcc8d47e372e5066d07b98b0059d1d2b7ee548 Mon Sep 17 00:00:00 2001 From: cpuu Date: Sat, 19 Nov 2022 23:11:45 +0900 Subject: [PATCH 237/526] typo --- doc/source/getting-started-macos-tutorial.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/getting-started-macos-tutorial.rst b/doc/source/getting-started-macos-tutorial.rst index 6dd144ac3..0d95a1700 100644 --- a/doc/source/getting-started-macos-tutorial.rst +++ b/doc/source/getting-started-macos-tutorial.rst @@ -131,6 +131,8 @@ mac.ifconfig ~~~~~~~~~~ we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. + + .. code-block:: shell-session $ python3 vol.py -f contact_me mac.ifconfig From 53870c64d1553e43e283bff3fbc08cc0249d4d0e Mon Sep 17 00:00:00 2001 From: cpuu Date: Sat, 19 Nov 2022 23:15:29 +0900 Subject: [PATCH 238/526] Add macos tutorial --- doc/source/getting-started-macos-tutorial.rst | 2 +- doc/source/index.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/getting-started-macos-tutorial.rst b/doc/source/getting-started-macos-tutorial.rst index 0d95a1700..76f77d0f3 100644 --- a/doc/source/getting-started-macos-tutorial.rst +++ b/doc/source/getting-started-macos-tutorial.rst @@ -17,7 +17,7 @@ Procedure to create symbol tables for macOS To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. -.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , +.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , which is built and maintained by `volatilityfoundation `_. After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/mac``. If necessary create a mac directory under the symbols directory (this will become unnecessary in future versions). diff --git a/doc/source/index.rst b/doc/source/index.rst index 9b1d05858..e096731c7 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -26,6 +26,7 @@ There is also some information to get you started quickly: getting-started-linux-tutorial getting-started-windows-tutorial + getting-started-macos-tutorial .. toctree:: From fc0fa30f9e3fda0479f0fd9761b33550d705ecc2 Mon Sep 17 00:00:00 2001 From: cpuu Date: Sun, 20 Nov 2022 10:18:47 +0900 Subject: [PATCH 239/526] Update doc/source/getting-started-macos-tutorial.rst Co-authored-by: Donghyun Kim --- doc/source/getting-started-macos-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting-started-macos-tutorial.rst b/doc/source/getting-started-macos-tutorial.rst index 76f77d0f3..bc0cb1b92 100644 --- a/doc/source/getting-started-macos-tutorial.rst +++ b/doc/source/getting-started-macos-tutorial.rst @@ -81,7 +81,7 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols/mac`` directory. -linux.pslist +mac.pslist ~~~~~~~~~~~~ .. code-block:: shell-session From 8c41007f0b72e80debce6ad2159ff368ae0dd719 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 29 Nov 2022 21:16:33 +0900 Subject: [PATCH 240/526] Fix: setup python method --- .github/workflows/build-pypi.yml | 12 +++++++----- .github/workflows/test.yaml | 13 ++++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 77fe26931..ce69d4768 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -16,13 +16,15 @@ jobs: build: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.6"] steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.x - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cf70b66cd..17117bd79 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,13 +4,16 @@ jobs: build: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.6"] steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.6 - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: '3.6' + python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | From f7a4d4f1ef0cdc408bcf32bb902c3b3e67c9642c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 29 Nov 2022 21:18:20 +0900 Subject: [PATCH 241/526] Fix: 3.6.7 versions --- .github/workflows/build-pypi.yml | 2 +- .github/workflows/test.yaml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index ce69d4768..7433269bf 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6"] + python-version: ["3.6.7"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 17117bd79..bcc3b79b0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6"] + python-version: ["3.6.7"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -14,7 +14,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies run: | python -m pip install --upgrade pip From 6e056a7819ba4694b9e643dcc8d73e6779c1a9a2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 29 Nov 2022 21:20:25 +0900 Subject: [PATCH 242/526] Add: env value --- .github/workflows/build-pypi.yml | 2 ++ .github/workflows/test.yaml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 7433269bf..95145f329 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -25,6 +25,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + env: + AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache - name: Install dependencies run: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bcc3b79b0..5d712ce25 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,13 +6,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6.7"] + python-version: ["3.6"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + env: + AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache - name: Install dependencies run: | From 3daba5dabbdba1670262b48ca466a19328721b65 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 30 Nov 2022 04:04:07 +0900 Subject: [PATCH 243/526] Fix: more detail version --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5d712ce25..0f3f10228 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6"] + python-version: ["3.6.7"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 372003b0e48224fcafa3f3bf0620b04d3ca9cb1e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 30 Nov 2022 04:07:18 +0900 Subject: [PATCH 244/526] Fix: more detail ubuntu version --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0f3f10228..5f3c996f3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,10 +3,10 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.6.7"] + python-version: ["3.6"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 7aaf157e07480ccdb88bab33053a045fd8c16277 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 30 Nov 2022 04:09:28 +0900 Subject: [PATCH 245/526] Remove: env --- .github/workflows/test.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5f3c996f3..8f044a365 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -13,8 +13,6 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - env: - AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache - name: Install dependencies run: | From 130baa7f34e5430d3dc081f0d8d9e936fe59ffb6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 30 Nov 2022 04:10:40 +0900 Subject: [PATCH 246/526] Remove: env for build --- .github/workflows/build-pypi.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 95145f329..061c8359a 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -15,18 +15,16 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.6.7"] + python-version: ["3.6"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - env: - AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache - name: Install dependencies run: | From e694713b20f23d38564a13b38a9898614945fa8b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 30 Nov 2022 01:05:25 +0000 Subject: [PATCH 247/526] Github: Backport action changes to 2.4.0 release --- .github/workflows/build-pypi.yml | 14 ++++++++------ .github/workflows/test.yaml | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 77fe26931..061c8359a 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -15,14 +15,16 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.6"] steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.x - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cf70b66cd..8f044a365 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,14 +3,16 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.6"] steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.6 - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: '3.6' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | From bd402772c737f24f86b79ae5edd9485c228a83db Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 7 Dec 2022 20:48:14 +0000 Subject: [PATCH 248/526] Core: Apply black to the entire codebase and add workflow --- .github/workflows/black.yml | 4 + setup.py | 63 +- test/conftest.py | 51 +- test/test_volatility.py | 134 ++- vol.py | 2 +- volatility3/__init__.py | 6 +- volatility3/cli/__init__.py | 519 ++++++---- volatility3/cli/text_renderer.py | 167 ++-- volatility3/cli/volargparse.py | 34 +- volatility3/cli/volshell/__init__.py | 286 ++++-- volatility3/cli/volshell/generic.py | 313 +++++-- volatility3/cli/volshell/linux.py | 38 +- volatility3/cli/volshell/mac.py | 44 +- volatility3/cli/volshell/windows.py | 42 +- volatility3/framework/__init__.py | 100 +- volatility3/framework/automagic/__init__.py | 52 +- .../framework/automagic/construct_layers.py | 47 +- volatility3/framework/automagic/linux.py | 157 ++-- volatility3/framework/automagic/mac.py | 190 ++-- volatility3/framework/automagic/module.py | 47 +- volatility3/framework/automagic/pdbscan.py | 342 ++++--- volatility3/framework/automagic/stacker.py | 171 +++- .../framework/automagic/symbol_cache.py | 293 ++++-- .../framework/automagic/symbol_finder.py | 105 ++- volatility3/framework/automagic/windows.py | 307 +++--- .../framework/configuration/requirements.py | 414 +++++--- volatility3/framework/constants/__init__.py | 36 +- .../framework/constants/linux/__init__.py | 2 +- volatility3/framework/contexts/__init__.py | 248 +++-- volatility3/framework/exceptions.py | 32 +- volatility3/framework/interfaces/__init__.py | 12 +- volatility3/framework/interfaces/automagic.py | 80 +- .../framework/interfaces/configuration.py | 208 ++-- volatility3/framework/interfaces/context.py | 113 ++- volatility3/framework/interfaces/layers.py | 239 +++-- volatility3/framework/interfaces/objects.py | 137 ++- volatility3/framework/interfaces/plugins.py | 26 +- volatility3/framework/interfaces/renderers.py | 80 +- volatility3/framework/interfaces/symbols.py | 120 ++- volatility3/framework/layers/avml.py | 105 ++- volatility3/framework/layers/crash.py | 155 ++- volatility3/framework/layers/elf.py | 82 +- volatility3/framework/layers/intel.py | 235 +++-- volatility3/framework/layers/leechcore.py | 30 +- volatility3/framework/layers/lime.py | 54 +- volatility3/framework/layers/linear.py | 52 +- volatility3/framework/layers/msf.py | 175 ++-- volatility3/framework/layers/physical.py | 87 +- volatility3/framework/layers/qemu.py | 375 +++++--- volatility3/framework/layers/registry.py | 200 ++-- volatility3/framework/layers/resources.py | 130 ++- .../framework/layers/scanners/__init__.py | 17 +- .../framework/layers/scanners/multiregexp.py | 8 +- volatility3/framework/layers/segmented.py | 76 +- volatility3/framework/layers/vmware.py | 163 +++- volatility3/framework/objects/__init__.py | 666 ++++++++----- volatility3/framework/objects/templates.py | 54 +- volatility3/framework/objects/utility.py | 33 +- volatility3/framework/plugins/__init__.py | 31 +- volatility3/framework/plugins/banners.py | 39 +- volatility3/framework/plugins/configwriter.py | 31 +- .../framework/plugins/frameworkinfo.py | 18 +- volatility3/framework/plugins/isfinfo.py | 150 ++- volatility3/framework/plugins/layerwriter.py | 100 +- volatility3/framework/plugins/linux/bash.py | 97 +- .../framework/plugins/linux/check_afinfo.py | 51 +- .../framework/plugins/linux/check_creds.py | 13 +- .../framework/plugins/linux/check_idt.py | 64 +- .../framework/plugins/linux/check_modules.py | 38 +- .../framework/plugins/linux/check_syscall.py | 63 +- volatility3/framework/plugins/linux/elfs.py | 63 +- .../plugins/linux/keyboard_notifiers.py | 40 +- volatility3/framework/plugins/linux/kmsg.py | 139 ++- volatility3/framework/plugins/linux/lsmod.py | 21 +- volatility3/framework/plugins/linux/lsof.py | 49 +- .../framework/plugins/linux/malfind.py | 75 +- .../framework/plugins/linux/mountinfo.py | 170 +++- volatility3/framework/plugins/linux/proc.py | 67 +- volatility3/framework/plugins/linux/psaux.py | 47 +- volatility3/framework/plugins/linux/pslist.py | 89 +- volatility3/framework/plugins/linux/pstree.py | 18 +- .../framework/plugins/linux/tty_check.py | 50 +- volatility3/framework/plugins/mac/bash.py | 116 ++- .../framework/plugins/mac/check_syscall.py | 57 +- .../framework/plugins/mac/check_sysctl.py | 73 +- .../framework/plugins/mac/check_trap_table.py | 57 +- volatility3/framework/plugins/mac/ifconfig.py | 28 +- .../framework/plugins/mac/kauth_listeners.py | 64 +- .../framework/plugins/mac/kauth_scopes.py | 70 +- volatility3/framework/plugins/mac/kevents.py | 108 ++- .../framework/plugins/mac/list_files.py | 43 +- volatility3/framework/plugins/mac/lsmod.py | 30 +- volatility3/framework/plugins/mac/lsof.py | 50 +- volatility3/framework/plugins/mac/malfind.py | 75 +- volatility3/framework/plugins/mac/mount.py | 23 +- volatility3/framework/plugins/mac/netstat.py | 110 ++- .../framework/plugins/mac/proc_maps.py | 64 +- volatility3/framework/plugins/mac/psaux.py | 52 +- volatility3/framework/plugins/mac/pslist.py | 194 ++-- volatility3/framework/plugins/mac/pstree.py | 28 +- .../framework/plugins/mac/socket_filters.py | 84 +- volatility3/framework/plugins/mac/timers.py | 79 +- .../framework/plugins/mac/trustedbsd.py | 69 +- .../framework/plugins/mac/vfsevents.py | 42 +- volatility3/framework/plugins/timeliner.py | 197 ++-- .../framework/plugins/windows/bigpools.py | 132 ++- .../framework/plugins/windows/cachedump.py | 108 ++- .../framework/plugins/windows/callbacks.py | 302 ++++-- .../framework/plugins/windows/cmdline.py | 63 +- .../framework/plugins/windows/crashinfo.py | 96 +- .../framework/plugins/windows/devicetree.py | 254 ++--- .../framework/plugins/windows/dlllist.py | 195 ++-- .../framework/plugins/windows/driverirp.py | 109 ++- .../framework/plugins/windows/drivermodule.py | 71 +- .../framework/plugins/windows/driverscan.py | 63 +- .../framework/plugins/windows/dumpfiles.py | 217 +++-- .../framework/plugins/windows/envars.py | 226 +++-- .../framework/plugins/windows/filescan.py | 41 +- .../plugins/windows/getservicesids.py | 56 +- .../framework/plugins/windows/getsids.py | 162 +++- .../framework/plugins/windows/handles.py | 223 +++-- .../framework/plugins/windows/hashdump.py | 468 +++++++-- volatility3/framework/plugins/windows/info.py | 179 +++- .../framework/plugins/windows/joblinks.py | 113 ++- .../framework/plugins/windows/ldrmodules.py | 124 ++- .../framework/plugins/windows/lsadump.py | 114 ++- .../framework/plugins/windows/malfind.py | 163 +++- .../framework/plugins/windows/mbrscan.py | 321 ++++--- .../framework/plugins/windows/memmap.py | 98 +- .../framework/plugins/windows/mftscan.py | 134 ++- .../framework/plugins/windows/modscan.py | 156 ++- .../framework/plugins/windows/modules.py | 131 ++- .../framework/plugins/windows/mutantscan.py | 47 +- .../framework/plugins/windows/netscan.py | 347 ++++--- .../framework/plugins/windows/netstat.py | 545 +++++++---- .../framework/plugins/windows/poolscanner.py | 377 +++++--- .../framework/plugins/windows/privileges.py | 95 +- .../framework/plugins/windows/pslist.py | 210 +++-- .../framework/plugins/windows/psscan.py | 214 +++-- .../framework/plugins/windows/pstree.py | 94 +- .../plugins/windows/registry/hivelist.py | 254 +++-- .../plugins/windows/registry/hivescan.py | 61 +- .../plugins/windows/registry/printkey.py | 253 +++-- .../plugins/windows/registry/userassist.py | 272 ++++-- .../framework/plugins/windows/sessions.py | 75 +- .../plugins/windows/skeleton_key_check.py | 384 +++++--- volatility3/framework/plugins/windows/ssdt.py | 96 +- .../framework/plugins/windows/strings.py | 114 ++- .../framework/plugins/windows/svcscan.py | 196 ++-- .../framework/plugins/windows/symlinkscan.py | 57 +- .../framework/plugins/windows/vadinfo.py | 192 ++-- .../framework/plugins/windows/vadwalk.py | 97 +- .../framework/plugins/windows/vadyarascan.py | 110 ++- .../framework/plugins/windows/verinfo.py | 202 ++-- .../framework/plugins/windows/virtmap.py | 123 ++- volatility3/framework/plugins/yarascan.py | 115 ++- volatility3/framework/renderers/__init__.py | 121 ++- volatility3/framework/renderers/conversion.py | 34 +- .../framework/renderers/format_hints.py | 36 +- volatility3/framework/symbols/__init__.py | 96 +- .../framework/symbols/generic/__init__.py | 37 +- volatility3/framework/symbols/intermed.py | 551 ++++++----- .../framework/symbols/linux/__init__.py | 116 ++- volatility3/framework/symbols/linux/bash.py | 3 +- .../symbols/linux/extensions/__init__.py | 247 +++-- .../symbols/linux/extensions/bash.py | 1 - .../framework/symbols/linux/extensions/elf.py | 142 ++- volatility3/framework/symbols/mac/__init__.py | 159 ++-- .../symbols/mac/extensions/__init__.py | 146 ++- volatility3/framework/symbols/metadata.py | 16 +- volatility3/framework/symbols/native.py | 80 +- .../framework/symbols/windows/__init__.py | 67 +- .../symbols/windows/extensions/__init__.py | 579 +++++++----- .../symbols/windows/extensions/crash.py | 31 +- .../symbols/windows/extensions/kdbg.py | 25 +- .../symbols/windows/extensions/mbr.py | 28 +- .../symbols/windows/extensions/mft.py | 6 +- .../symbols/windows/extensions/network.py | 91 +- .../symbols/windows/extensions/pe.py | 123 ++- .../symbols/windows/extensions/pool.py | 242 +++-- .../symbols/windows/extensions/registry.py | 138 ++- .../symbols/windows/extensions/services.py | 51 +- .../framework/symbols/windows/pdbconv.py | 885 ++++++++++-------- .../framework/symbols/windows/pdbutil.py | 392 +++++--- .../framework/symbols/windows/versions.py | 138 ++- .../plugins/windows/registry/certificates.py | 116 ++- volatility3/plugins/windows/statistics.py | 69 +- volatility3/schemas/__init__.py | 26 +- volshell.py | 2 +- 189 files changed, 16494 insertions(+), 8343 deletions(-) create mode 100644 .github/workflows/black.yml diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml new file mode 100644 index 000000000..6bd6fd1af --- /dev/null +++ b/.github/workflows/black.yml @@ -0,0 +1,4 @@ +- uses: psf/black@stable + with: + options: "--check --diff --verbose" + src: "./volatility3" diff --git a/setup.py b/setup.py index a4bd3fffe..ae1b3f6dc 100644 --- a/setup.py +++ b/setup.py @@ -6,9 +6,10 @@ import setuptools from volatility3.framework import constants -with open("README.md", "r", encoding = "utf-8") as fh: +with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() + def get_install_requires(): requirements = [] with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: @@ -19,32 +20,34 @@ def get_install_requires(): requirements.append(stripped_line) return requirements -setuptools.setup(name = "volatility3", - description = "Memory forensics framework", - version = constants.PACKAGE_VERSION, - license = "VSL", - keywords = "volatility memory forensics framework windows linux volshell", - author = "Volatility Foundation", - long_description = long_description, - long_description_content_type = "text/markdown", - author_email = "volatility@volatilityfoundation.org", - url = "https://github.com/volatilityfoundation/volatility3/", - project_urls = { - "Bug Tracker": "https://github.com/volatilityfoundation/volatility3/issues", - "Documentation": "https://volatility3.readthedocs.io/", - "Source Code": "https://github.com/volatilityfoundation/volatility3", - }, - python_requires = '>=3.6.0', - include_package_data = True, - exclude_package_data = { - '': ['development', 'development.*'], - 'development': ['*'] - }, - packages = setuptools.find_namespace_packages(exclude = ["development", "development.*"]), - entry_points = { - 'console_scripts': [ - 'vol = volatility3.cli:main', - 'volshell = volatility3.cli.volshell:main', - ], - }, - install_requires = get_install_requires()) + +setuptools.setup( + name="volatility3", + description="Memory forensics framework", + version=constants.PACKAGE_VERSION, + license="VSL", + keywords="volatility memory forensics framework windows linux volshell", + author="Volatility Foundation", + long_description=long_description, + long_description_content_type="text/markdown", + author_email="volatility@volatilityfoundation.org", + url="https://github.com/volatilityfoundation/volatility3/", + project_urls={ + "Bug Tracker": "https://github.com/volatilityfoundation/volatility3/issues", + "Documentation": "https://volatility3.readthedocs.io/", + "Source Code": "https://github.com/volatilityfoundation/volatility3", + }, + python_requires=">=3.6.0", + include_package_data=True, + exclude_package_data={"": ["development", "development.*"], "development": ["*"]}, + packages=setuptools.find_namespace_packages( + exclude=["development", "development.*"] + ), + entry_points={ + "console_scripts": [ + "vol = volatility3.cli:main", + "volshell = volatility3.cli.volshell:main", + ], + }, + install_requires=get_install_requires(), +) diff --git a/test/conftest.py b/test/conftest.py index 9057e1676..4ad63065b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -3,38 +3,57 @@ import os import pytest + def pytest_addoption(parser): - parser.addoption("--volatility", action="store", default=None, + parser.addoption( + "--volatility", + action="store", + default=None, required=True, - help="path to the volatility script") + help="path to the volatility script", + ) - parser.addoption("--python", action="store", default="python3", - help="The name of the interpreter to use when running the volatility script") + parser.addoption( + "--python", + action="store", + default="python3", + help="The name of the interpreter to use when running the volatility script", + ) - parser.addoption("--image", action="append", default=[], - help="path to an image to test") + parser.addoption( + "--image", action="append", default=[], help="path to an image to test" + ) + + parser.addoption( + "--image-dir", + action="append", + default=[], + help="path to a directory containing images to test", + ) - parser.addoption("--image-dir", action="append", default=[], - help="path to a directory containing images to test") def pytest_generate_tests(metafunc): """Parameterize tests based on image names""" - images = metafunc.config.getoption('image') - for image_dir in metafunc.config.getoption('image_dir'): - images = images + [os.path.join(image_dir, dir) for dir in os.listdir(image_dir)] + images = metafunc.config.getoption("image") + for image_dir in metafunc.config.getoption("image_dir"): + images = images + [ + os.path.join(image_dir, dir) for dir in os.listdir(image_dir) + ] # tests with "image" parameter are run against images - if 'image' in metafunc.fixturenames: - metafunc.parametrize("image", - images, - ids=[os.path.basename(image) for image in images]) + if "image" in metafunc.fixturenames: + metafunc.parametrize( + "image", images, ids=[os.path.basename(image) for image in images] + ) + # Fixtures @pytest.fixture def volatility(request): return request.config.getoption("--volatility") + @pytest.fixture def python(request): - return request.config.getoption("--python") + return request.config.getoption("--python") diff --git a/test/test_volatility.py b/test/test_volatility.py index 0f50cbb8d..fb4fece12 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -18,6 +18,7 @@ import json # HELPER FUNCTIONS # + def runvol(args, volatility, python): volpy = volatility python_cmd = python @@ -35,22 +36,29 @@ def runvol(args, volatility, python): return p.returncode, stdout, stderr + def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): - args = globalargs + [ - "--single-location", - img, - "-q", - plugin, - ] + pluginargs + args = ( + globalargs + + [ + "--single-location", + img, + "-q", + plugin, + ] + + pluginargs + ) return runvol(args, volatility, python) + # # TESTS # # WINDOWS + def test_windows_pslist(image, volatility, python): rc, out, err = runvol_plugin("windows.pslist.PsList", image, volatility, python) out = out.lower() @@ -61,12 +69,14 @@ def test_windows_pslist(image, volatility, python): assert rc == 0 rc, out, err = runvol_plugin( - "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"]) + "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"] + ) out = out.lower() assert out.find(b"system") != -1 assert out.count(b"\n") < 10 assert rc == 0 + def test_windows_psscan(image, volatility, python): rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) out = out.lower() @@ -76,38 +86,47 @@ def test_windows_psscan(image, volatility, python): assert out.count(b"\n") > 10 assert rc == 0 + def test_windows_dlllist(image, volatility, python): rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 + def test_windows_modules(image, volatility, python): rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 + def test_windows_hivelist(image, volatility, python): - rc, out, err = runvol_plugin("windows.registry.hivelist.HiveList", image, volatility, python) + rc, out, err = runvol_plugin( + "windows.registry.hivelist.HiveList", image, volatility, python + ) out = out.lower() not_xp = out.find(b"\\systemroot\\system32\\config\\software") if not_xp == -1: - assert out.find(b"\\device\\harddiskvolume1\\windows\\system32\\config\\software") != -1 + assert ( + out.find(b"\\device\\harddiskvolume1\\windows\\system32\\config\\software") + != -1 + ) assert out.count(b"\n") > 10 assert rc == 0 + def test_windows_dumpfiles(image, volatility, python): - json_file = open('./test/known_files.json') + json_file = open("./test/known_files.json") known_files = json.load(json_file) failed_chksms = 0 - if sys.platform == 'win32': + if sys.platform == "win32": file_name = ntpath.basename(image) else: file_name = os.path.basename(image) @@ -117,11 +136,21 @@ def test_windows_dumpfiles(image, volatility, python): path = tempfile.mkdtemp() - rc, out, err = runvol_plugin("windows.dumpfiles.DumpFiles", image, volatility, python, globalargs=["-o", path], pluginargs=["--virtaddr", addr]) + rc, out, err = runvol_plugin( + "windows.dumpfiles.DumpFiles", + image, + volatility, + python, + globalargs=["-o", path], + pluginargs=["--virtaddr", addr], + ) for file in os.listdir(path): with open(os.path.join(path, file), "rb") as fp: - if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: + if ( + hashlib.md5(fp.read()).hexdigest() + not in known_files["windows_dumpfiles"][file_name][addr] + ): failed_chksms += 1 shutil.rmtree(path) @@ -135,16 +164,24 @@ def test_windows_dumpfiles(image, volatility, python): print("Key Error raised on " + str(e)) assert False + def test_windows_handles(image, volatility, python): rc, out, err = runvol_plugin( - "windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"]) + "windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"] + ) assert out.find(b"System Pid 4") != -1 - assert out.find(b"MACHINE\\SYSTEM\\CONTROLSET001\\CONTROL\\SESSION MANAGER\\MEMORY MANAGEMENT\\PREFETCHPARAMETERS") != -1 + assert ( + out.find( + b"MACHINE\\SYSTEM\\CONTROLSET001\\CONTROL\\SESSION MANAGER\\MEMORY MANAGEMENT\\PREFETCHPARAMETERS" + ) + != -1 + ) assert out.find(b"MACHINE\\SYSTEM\\SETUP") != -1 assert out.count(b"\n") > 500 assert rc == 0 + def test_windows_svcscan(image, volatility, python): rc, out, err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) @@ -152,9 +189,11 @@ def test_windows_svcscan(image, volatility, python): assert out.count(b"\n") > 250 assert rc == 0 + def test_windows_privileges(image, volatility, python): rc, out, err = runvol_plugin( - "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"]) + "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"] + ) assert out.find(b"SeCreateTokenPrivilege") != -1 assert out.find(b"SeCreateGlobalPrivilege") != -1 @@ -162,9 +201,11 @@ def test_windows_privileges(image, volatility, python): assert out.count(b"\n") > 20 assert rc == 0 + def test_windows_getsids(image, volatility, python): rc, out, err = runvol_plugin( - "windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"]) + "windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"] + ) assert out.find(b"Local System") != -1 assert out.find(b"Administrators") != -1 @@ -172,6 +213,7 @@ def test_windows_getsids(image, volatility, python): assert out.find(b"Authenticated Users") != -1 assert rc == 0 + def test_windows_envars(image, volatility, python): rc, out, err = runvol_plugin("windows.envars.Envars", image, volatility, python) @@ -183,8 +225,11 @@ def test_windows_envars(image, volatility, python): assert out.count(b"\n") > 500 assert rc == 0 + def test_windows_callbacks(image, volatility, python): - rc, out, err = runvol_plugin("windows.callbacks.Callbacks", image, volatility, python) + rc, out, err = runvol_plugin( + "windows.callbacks.Callbacks", image, volatility, python + ) assert out.find(b"PspCreateProcessNotifyRoutine") != -1 assert out.find(b"KeBugCheckCallbackListHead") != -1 @@ -203,8 +248,11 @@ def test_windows_vadwalk(image, volatility, python): assert out.find(b"0x0") != -1 assert rc == 0 + def test_windows_devicetree(image, volatility, python): - rc, out, err = runvol_plugin("windows.devicetree.DeviceTree", image, volatility, python) + rc, out, err = runvol_plugin( + "windows.devicetree.DeviceTree", image, volatility, python + ) assert out.find(b"DEV") != -1 assert out.find(b"DRV") != -1 @@ -214,17 +262,20 @@ def test_windows_devicetree(image, volatility, python): assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 assert rc == 0 + # LINUX + def test_linux_pslist(image, volatility, python): rc, out, err = runvol_plugin("linux.pslist.PsList", image, volatility, python) out = out.lower() - assert ((out.find(b"init") != -1) or (out.find(b"systemd") != -1)) + assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.find(b"watchdog") != -1 assert out.count(b"\n") > 10 assert rc == 0 + def test_linux_check_idt(image, volatility, python): rc, out, err = runvol_plugin("linux.check_idt.Check_idt", image, volatility, python) out = out.lower() @@ -233,8 +284,11 @@ def test_linux_check_idt(image, volatility, python): assert out.count(b"\n") > 10 assert rc == 0 + def test_linux_check_syscall(image, volatility, python): - rc, out, err = runvol_plugin("linux.check_syscall.Check_syscall", image, volatility, python) + rc, out, err = runvol_plugin( + "linux.check_syscall.Check_syscall", image, volatility, python + ) out = out.lower() assert out.find(b"sys_close") != -1 @@ -242,6 +296,7 @@ def test_linux_check_syscall(image, volatility, python): assert out.count(b"\n") > 100 assert rc == 0 + def test_linux_lsmod(image, volatility, python): rc, out, err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) out = out.lower() @@ -249,6 +304,7 @@ def test_linux_lsmod(image, volatility, python): assert out.count(b"\n") > 10 assert rc == 0 + def test_linux_lsof(image, volatility, python): rc, out, err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) out = out.lower() @@ -257,6 +313,7 @@ def test_linux_lsof(image, volatility, python): assert out.count(b"\n") > 35 assert rc == 0 + def test_linux_proc_maps(image, volatility, python): rc, out, err = runvol_plugin("linux.proc.Maps", image, volatility, python) out = out.lower() @@ -265,6 +322,7 @@ def test_linux_proc_maps(image, volatility, python): assert out.count(b"\n") > 100 assert rc == 0 + def test_linux_tty_check(image, volatility, python): rc, out, err = runvol_plugin("linux.tty_check.tty_check", image, volatility, python) out = out.lower() @@ -273,18 +331,23 @@ def test_linux_tty_check(image, volatility, python): assert out.count(b"\n") >= 5 assert rc == 0 + # MAC + def test_mac_pslist(image, volatility, python): rc, out, err = runvol_plugin("mac.pslist.PsList", image, volatility, python) out = out.lower() - assert ((out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1)) + assert (out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1) assert out.count(b"\n") > 10 assert rc == 0 + def test_mac_check_syscall(image, volatility, python): - rc, out, err = runvol_plugin("mac.check_syscall.Check_syscall", image, volatility, python) + rc, out, err = runvol_plugin( + "mac.check_syscall.Check_syscall", image, volatility, python + ) out = out.lower() assert out.find(b"chmod") != -1 @@ -293,22 +356,29 @@ def test_mac_check_syscall(image, volatility, python): assert out.count(b"\n") > 100 assert rc == 0 + def test_mac_check_sysctl(image, volatility, python): - rc, out, err = runvol_plugin("mac.check_sysctl.Check_sysctl", image, volatility, python) + rc, out, err = runvol_plugin( + "mac.check_sysctl.Check_sysctl", image, volatility, python + ) out = out.lower() assert out.find(b"__kernel__") != -1 assert out.count(b"\n") > 250 assert rc == 0 + def test_mac_check_trap_table(image, volatility, python): - rc, out, err = runvol_plugin("mac.check_trap_table.Check_trap_table", image, volatility, python) + rc, out, err = runvol_plugin( + "mac.check_trap_table.Check_trap_table", image, volatility, python + ) out = out.lower() assert out.count(b"kern_invalid") >= 10 assert out.count(b"\n") > 50 assert rc == 0 + def test_mac_ifconfig(image, volatility, python): rc, out, err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) out = out.lower() @@ -318,6 +388,7 @@ def test_mac_ifconfig(image, volatility, python): assert out.count(b"\n") > 9 assert rc == 0 + def test_mac_lsmod(image, volatility, python): rc, out, err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) out = out.lower() @@ -326,6 +397,7 @@ def test_mac_lsmod(image, volatility, python): assert out.count(b"\n") > 10 assert rc == 0 + def test_mac_lsof(image, volatility, python): rc, out, err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) out = out.lower() @@ -333,6 +405,7 @@ def test_mac_lsof(image, volatility, python): assert out.count(b"\n") > 50 assert rc == 0 + def test_mac_malfind(image, volatility, python): rc, out, err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) out = out.lower() @@ -340,6 +413,7 @@ def test_mac_malfind(image, volatility, python): assert out.count(b"\n") > 20 assert rc == 0 + def test_mac_mount(image, volatility, python): rc, out, err = runvol_plugin("mac.mount.Mount", image, volatility, python) out = out.lower() @@ -348,6 +422,7 @@ def test_mac_mount(image, volatility, python): assert out.count(b"\n") > 7 assert rc == 0 + def test_mac_netstat(image, volatility, python): rc, out, err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) @@ -357,6 +432,7 @@ def test_mac_netstat(image, volatility, python): assert out.count(b"\n") > 10 assert rc == 0 + def test_mac_proc_maps(image, volatility, python): rc, out, err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) out = out.lower() @@ -365,6 +441,7 @@ def test_mac_proc_maps(image, volatility, python): assert out.count(b"\n") > 100 assert rc == 0 + def test_mac_psaux(image, volatility, python): rc, out, err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) out = out.lower() @@ -373,13 +450,17 @@ def test_mac_psaux(image, volatility, python): assert out.count(b"\n") > 50 assert rc == 0 + def test_mac_socket_filters(image, volatility, python): - rc, out, err = runvol_plugin("mac.socket_filters.Socket_filters", image, volatility, python) + rc, out, err = runvol_plugin( + "mac.socket_filters.Socket_filters", image, volatility, python + ) out = out.lower() assert out.count(b"\n") > 9 assert rc == 0 + def test_mac_timers(image, volatility, python): rc, out, err = runvol_plugin("mac.timers.Timers", image, volatility, python) out = out.lower() @@ -387,6 +468,7 @@ def test_mac_timers(image, volatility, python): assert out.count(b"\n") > 6 assert rc == 0 + def test_mac_trustedbsd(image, volatility, python): rc, out, err = runvol_plugin("mac.trustedbsd.Trustedbsd", image, volatility, python) out = out.lower() diff --git a/vol.py b/vol.py index 080413545..ff420cad5 100755 --- a/vol.py +++ b/vol.py @@ -6,5 +6,5 @@ import volatility3.cli -if __name__ == '__main__': +if __name__ == "__main__": volatility3.cli.main() diff --git a/volatility3/__init__.py b/volatility3/__init__.py index b6da6e01e..28df7da5a 100644 --- a/volatility3/__init__.py +++ b/volatility3/__init__.py @@ -32,14 +32,16 @@ class WarningFindSpec(abc.MetaPathFinder): used.""" @staticmethod - def find_spec(fullname: str, path: Optional[List[str]], target: None = None, **kwargs) -> None: + def find_spec( + fullname: str, path: Optional[List[str]], target: None = None, **kwargs + ) -> None: """Mock find_spec method that just checks the name, this must go first.""" if fullname.startswith("volatility3.framework.plugins."): warning = "Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins" # Pyinstaller uses walk_packages/_collect_submodules to import, but needs to read the modules to figure out dependencies # As such, we only print the warning when directly imported rather than from within walk_packages/_collect_submodules - if inspect.stack()[-2].function in ['walk_packages', '_collect_submodules']: + if inspect.stack()[-2].function in ["walk_packages", "_collect_submodules"]: raise Warning(warning) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 8851e2b18..fb124a3c9 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -26,7 +26,15 @@ import volatility3.plugins import volatility3.symbols from volatility3 import framework from volatility3.cli import text_renderer, volargparse -from volatility3.framework import automagic, configuration, constants, contexts, exceptions, interfaces, plugins +from volatility3.framework import ( + automagic, + configuration, + constants, + contexts, + exceptions, + interfaces, + plugins, +) from volatility3.framework.automagic import stacker from volatility3.framework.configuration import requirements @@ -36,7 +44,7 @@ rootlog = logging.getLogger() vollog = logging.getLogger(__name__) console = logging.StreamHandler() console.setLevel(logging.WARNING) -formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') +formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") # Trim the console down by default console.setFormatter(formatter) @@ -59,7 +67,7 @@ class PrintedProgress(object): message = f"\rProgress: {round(progress, 2): 7.2f}\t\t{description or ''}" message_len = len(message) self._max_message_len = max([self._max_message_len, message_len]) - sys.stderr.write(message + (' ' * (self._max_message_len - message_len)) + '\r') + sys.stderr.write(message + (" " * (self._max_message_len - message_len)) + "\r") class MuteProgress(PrintedProgress): @@ -72,7 +80,7 @@ class MuteProgress(PrintedProgress): class CommandLine: """Constructs a command-line interface object for users to run plugins.""" - CLI_NAME = 'volatility' + CLI_NAME = "volatility" def __init__(self): self.setup_logging() @@ -90,93 +98,142 @@ class CommandLine: volatility3.framework.require_interface_version(2, 0, 0) - renderers = dict([(x.name.lower(), x) for x in framework.class_subclasses(text_renderer.CLIRenderer)]) + renderers = dict( + [ + (x.name.lower(), x) + for x in framework.class_subclasses(text_renderer.CLIRenderer) + ] + ) - parser = volargparse.HelpfulArgParser(add_help = False, - prog = self.CLI_NAME, - description = "An open-source memory forensics framework") + parser = volargparse.HelpfulArgParser( + add_help=False, + prog=self.CLI_NAME, + description="An open-source memory forensics framework", + ) parser.add_argument( "-h", "--help", - action = "help", - default = argparse.SUPPRESS, - help = "Show this help message and exit, for specific plugin options use '{} --help'".format( - parser.prog)) - parser.add_argument("-c", - "--config", - help = "Load the configuration from a json file", - default = None, - type = str) - parser.add_argument("--parallelism", - help = "Enables parallelism (defaults to off if no argument given)", - nargs = '?', - choices = ['processes', 'threads', 'off'], - const = 'processes', - default = None, - type = str) - parser.add_argument("-e", - "--extend", - help = "Extend the configuration with a new (or changed) setting", - default = None, - action = 'append') - parser.add_argument("-p", - "--plugin-dirs", - help = "Semi-colon separated list of paths to find plugins", - default = "", - type = str) - parser.add_argument("-s", - "--symbol-dirs", - help = "Semi-colon separated list of paths to find symbols", - default = "", - type = str) - parser.add_argument("-v", "--verbosity", help = "Increase output verbosity", default = 0, action = "count") - parser.add_argument("-l", - "--log", - help = "Log output to a file as well as the console", - default = None, - type = str) - parser.add_argument("-o", - "--output-dir", - help = "Directory in which to output any generated files", - default = os.getcwd(), - type = str) - parser.add_argument("-q", "--quiet", help = "Remove progress feedback", default = False, action = 'store_true') - parser.add_argument("-r", - "--renderer", - metavar = 'RENDERER', - help = f"Determines how to render the output ({', '.join(list(renderers))})", - default = "quick", - choices = list(renderers)) - parser.add_argument("-f", - "--file", - metavar = 'FILE', - default = None, - type = str, - help = "Shorthand for --single-location=file:// if single-location is not defined") - parser.add_argument("--write-config", - help = "Write configuration JSON file out to config.json", - default = False, - action = 'store_true') - parser.add_argument("--save-config", - help = "Save configuration JSON file to a file", - default = None, - type = str) - parser.add_argument("--clear-cache", - help = "Clears out all short-term cached items", - default = False, - action = 'store_true') - parser.add_argument("--cache-path", - help = f"Change the default path ({constants.CACHE_PATH}) used to store the cache", - default = constants.CACHE_PATH, - type = str) - parser.add_argument("--offline", - help = "Do not search online for additional JSON files", - default = False, - action = 'store_true') + action="help", + default=argparse.SUPPRESS, + help="Show this help message and exit, for specific plugin options use '{} --help'".format( + parser.prog + ), + ) + parser.add_argument( + "-c", + "--config", + help="Load the configuration from a json file", + default=None, + type=str, + ) + parser.add_argument( + "--parallelism", + help="Enables parallelism (defaults to off if no argument given)", + nargs="?", + choices=["processes", "threads", "off"], + const="processes", + default=None, + type=str, + ) + parser.add_argument( + "-e", + "--extend", + help="Extend the configuration with a new (or changed) setting", + default=None, + action="append", + ) + parser.add_argument( + "-p", + "--plugin-dirs", + help="Semi-colon separated list of paths to find plugins", + default="", + type=str, + ) + parser.add_argument( + "-s", + "--symbol-dirs", + help="Semi-colon separated list of paths to find symbols", + default="", + type=str, + ) + parser.add_argument( + "-v", + "--verbosity", + help="Increase output verbosity", + default=0, + action="count", + ) + parser.add_argument( + "-l", + "--log", + help="Log output to a file as well as the console", + default=None, + type=str, + ) + parser.add_argument( + "-o", + "--output-dir", + help="Directory in which to output any generated files", + default=os.getcwd(), + type=str, + ) + parser.add_argument( + "-q", + "--quiet", + help="Remove progress feedback", + default=False, + action="store_true", + ) + parser.add_argument( + "-r", + "--renderer", + metavar="RENDERER", + help=f"Determines how to render the output ({', '.join(list(renderers))})", + default="quick", + choices=list(renderers), + ) + parser.add_argument( + "-f", + "--file", + metavar="FILE", + default=None, + type=str, + help="Shorthand for --single-location=file:// if single-location is not defined", + ) + parser.add_argument( + "--write-config", + help="Write configuration JSON file out to config.json", + default=False, + action="store_true", + ) + parser.add_argument( + "--save-config", + help="Save configuration JSON file to a file", + default=None, + type=str, + ) + parser.add_argument( + "--clear-cache", + help="Clears out all short-term cached items", + default=False, + action="store_true", + ) + parser.add_argument( + "--cache-path", + help=f"Change the default path ({constants.CACHE_PATH}) used to store the cache", + default=constants.CACHE_PATH, + type=str, + ) + parser.add_argument( + "--offline", + help="Do not search online for additional JSON files", + default=False, + action="store_true", + ) # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. - known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h'] + known_args = [arg for arg in sys.argv if arg != "--help" and arg != "-h"] partial_args, _ = parser.parse_known_args(known_args) banner_output = sys.stdout @@ -185,12 +242,14 @@ class CommandLine: banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") if partial_args.plugin_dirs: - volatility3.plugins.__path__ = [os.path.abspath(p) - for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH + volatility3.plugins.__path__ = [ + os.path.abspath(p) for p in partial_args.plugin_dirs.split(";") + ] + constants.PLUGINS_PATH if partial_args.symbol_dirs: - volatility3.symbols.__path__ = [os.path.abspath(p) - for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS + volatility3.symbols.__path__ = [ + os.path.abspath(p) for p in partial_args.symbol_dirs.split(";") + ] + constants.SYMBOL_BASEPATHS if partial_args.cache_path: constants.CACHE_PATH = partial_args.cache_path @@ -198,8 +257,10 @@ class CommandLine: if partial_args.log: file_logger = logging.FileHandler(partial_args.log) file_logger.setLevel(1) - file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', - fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + file_formatter = logging.Formatter( + datefmt="%y-%m-%d %H:%M:%S", + fmt="%(asctime)s %(name)-12s %(levelname)-8s %(message)s", + ) file_logger.setFormatter(file_formatter) rootlog.addHandler(file_logger) vollog.info("Logging started") @@ -214,9 +275,9 @@ class CommandLine: vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") # Set the PARALLELISM - if partial_args.parallelism == 'processes': + if partial_args.parallelism == "processes": constants.PARALLELISM = constants.Parallelism.Multiprocessing - elif partial_args.parallelism == 'threads': + elif partial_args.parallelism == "threads": constants.PARALLELISM = constants.Parallelism.Threading else: constants.PARALLELISM = constants.Parallelism.Off @@ -229,11 +290,14 @@ class CommandLine: # Do the initialization ctx = contexts.Context() # Construct a blank context - failures = framework.import_files(volatility3.plugins, - True) # Will not log as console's default level is WARNING + failures = framework.import_files( + volatility3.plugins, True + ) # Will not log as console's default level is WARNING if failures: - parser.epilog = "The following plugins could not be loaded (use -vv to see why): " + \ - ", ".join(sorted(failures)) + parser.epilog = ( + "The following plugins could not be loaded (use -vv to see why): " + + ", ".join(sorted(failures)) + ) vollog.info(parser.epilog) automagics = automagic.available(ctx) @@ -248,13 +312,18 @@ class CommandLine: if isinstance(amagic, interfaces.configuration.ConfigurableInterface): self.populate_requirements_argparse(parser, amagic.__class__) - subparser = parser.add_subparsers(title = "Plugins", - dest = "plugin", - description = "For plugin specific options, run '{} --help'".format( - self.CLI_NAME), - action = volargparse.HelpfulSubparserAction) + subparser = parser.add_subparsers( + title="Plugins", + dest="plugin", + description="For plugin specific options, run '{} --help'".format( + self.CLI_NAME + ), + action=volargparse.HelpfulSubparserAction, + ) for plugin in sorted(plugin_list): - plugin_parser = subparser.add_parser(plugin, help = plugin_list[plugin].__doc__) + plugin_parser = subparser.add_parser( + plugin, help=plugin_list[plugin].__doc__ + ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) ### @@ -267,12 +336,16 @@ class CommandLine: if args.plugin is None: parser.error("Please select a plugin to run") - vollog.log(constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}") + vollog.log( + constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}" + ) plugin = plugin_list[args.plugin] chosen_configurables_list[args.plugin] = plugin base_config_path = "plugins" - plugin_config_path = interfaces.configuration.path_join(base_config_path, plugin.__name__) + plugin_config_path = interfaces.configuration.path_join( + base_config_path, plugin.__name__ + ) # Special case the -f argument because people use is so frequently # It has to go here so it can be overridden by single-location if it's defined @@ -281,7 +354,7 @@ class CommandLine: if args.file: try: single_location = self.location_from_file(args.file) - ctx.config['automagic.LayerStacker.single_location'] = single_location + ctx.config["automagic.LayerStacker.single_location"] = single_location except ValueError as excp: parser.error(str(excp)) @@ -289,26 +362,37 @@ class CommandLine: if args.config: with open(args.config, "r") as f: json_val = json.load(f) - ctx.config.splice(plugin_config_path, interfaces.configuration.HierarchicalDict(json_val)) + ctx.config.splice( + plugin_config_path, + interfaces.configuration.HierarchicalDict(json_val), + ) # It should be up to the UI to determine which automagics to run, so this is before BACK TO THE FRAMEWORK automagics = automagic.choose_automagic(automagics, plugin) for amagic in automagics: chosen_configurables_list[amagic.__class__.__name__] = amagic - if ctx.config.get('automagic.LayerStacker.stackers', None) is None: - ctx.config['automagic.LayerStacker.stackers'] = stacker.choose_os_stackers(plugin) + if ctx.config.get("automagic.LayerStacker.stackers", None) is None: + ctx.config["automagic.LayerStacker.stackers"] = stacker.choose_os_stackers( + plugin + ) self.output_dir = args.output_dir if not os.path.exists(self.output_dir): - parser.error(f"The output directory specified does not exist: {self.output_dir}") + parser.error( + f"The output directory specified does not exist: {self.output_dir}" + ) self.populate_config(ctx, chosen_configurables_list, args, plugin_config_path) if args.extend: for extension in args.extend: - if '=' not in extension: - raise ValueError("Invalid extension (extensions must be of the format \"conf.path.value='value'\")") - address, value = extension[:extension.find('=')], json.loads(extension[extension.find('=') + 1:]) + if "=" not in extension: + raise ValueError( + "Invalid extension (extensions must be of the format \"conf.path.value='value'\")" + ) + address, value = extension[: extension.find("=")], json.loads( + extension[extension.find("=") + 1 :] + ) ctx.config[address] = value ### @@ -320,22 +404,40 @@ class CommandLine: if args.quiet: progress_callback = MuteProgress() - constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, - self.file_handler_class_factory()) + constructed = plugins.construct_plugin( + ctx, + automagics, + plugin, + base_config_path, + progress_callback, + self.file_handler_class_factory(), + ) if args.write_config: - vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') - args.save_config = 'config.json' + vollog.warning( + "Use of --write-config has been deprecated, replaced by --save-config " + ) + args.save_config = "config.json" if args.save_config: vollog.debug("Writing out configuration data to {args.save_config}") if os.path.exists(os.path.abspath(args.save_config)): - parser.error(f"Cannot write configuration: file {args.save_config} already exists") + parser.error( + f"Cannot write configuration: file {args.save_config} already exists" + ) with open(args.save_config, "w") as f: - json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + json.dump( + dict(constructed.build_configuration()), + f, + sort_keys=True, + indent=2, + ) f.write("\n") except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) - parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") + parser.exit( + 1, + f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n", + ) try: # Construct and run the plugin @@ -355,10 +457,12 @@ class CommandLine: The URL for the location of the file """ # We want to work in URLs, but we need to accept absolute and relative files (including on windows) - single_location = parse.urlparse(filename, '') - if single_location.scheme == '' or len(single_location.scheme) == 1: - single_location = parse.urlparse(parse.urljoin('file:', request.pathname2url(os.path.abspath(filename)))) - if single_location.scheme == 'file': + single_location = parse.urlparse(filename, "") + if single_location.scheme == "" or len(single_location.scheme) == 1: + single_location = parse.urlparse( + parse.urljoin("file:", request.pathname2url(os.path.abspath(filename))) + ) + if single_location.scheme == "file": if not os.path.exists(request.url2pathname(single_location.path)): filename = request.url2pathname(single_location.path) if not filename: @@ -374,7 +478,7 @@ class CommandLine: sys.stderr.flush() # Log the full exception at a high level for easy access - fulltrace = traceback.TracebackException.from_exception(excp).format(chain = True) + fulltrace = traceback.TracebackException.from_exception(excp).format(chain=True) vollog.debug("".join(fulltrace)) if isinstance(excp, exceptions.InvalidAddressException): @@ -383,22 +487,24 @@ class CommandLine: detail = f"Swap error {hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})" caused_by = [ "No suitable swap file having been provided (locate and provide the correct swap file)", - "An intentionally invalid page (operating system protection)" + "An intentionally invalid page (operating system protection)", ] elif isinstance(excp, exceptions.PagedInvalidAddressException): detail = f"Page error {hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})" caused_by = [ "Memory smear during acquisition (try re-acquiring if possible)", "An intentionally invalid page lookup (operating system protection)", - "A bug in the plugin/volatility3 (re-run with -vvv and file a bug)" + "A bug in the plugin/volatility3 (re-run with -vvv and file a bug)", ] else: - detail = f"{hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})" + detail = ( + f"{hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})" + ) caused_by = [ "The base memory file being incomplete (try re-acquiring if possible)", "Memory smear during acquisition (try re-acquiring if possible)", "An intentionally invalid page lookup (operating system protection)", - "A bug in the plugin/volatility3 (re-run with -vvv and file a bug)" + "A bug in the plugin/volatility3 (re-run with -vvv and file a bug)", ] elif isinstance(excp, exceptions.SymbolError): general = "Volatility experienced a symbol-related issue:" @@ -412,22 +518,28 @@ class CommandLine: general = "Volatility experienced an issue related to a symbol table:" detail = f"{excp}" caused_by = [ - "An invalid symbol table", "A plugin requesting a bad symbol", - "A plugin requesting a symbol from the wrong table" + "An invalid symbol table", + "A plugin requesting a bad symbol", + "A plugin requesting a symbol from the wrong table", ] elif isinstance(excp, exceptions.LayerException): general = f"Volatility experienced a layer-related issue: {excp.layer_name}" detail = f"{excp}" - caused_by = ["A faulty layer implementation (re-run with -vvv and file a bug)"] + caused_by = [ + "A faulty layer implementation (re-run with -vvv and file a bug)" + ] elif isinstance(excp, exceptions.MissingModuleException): general = f"Volatility could not import a necessary module: {excp.module}" detail = f"{excp}" - caused_by = ["A required python module is not installed (install the module and re-run)"] + caused_by = [ + "A required python module is not installed (install the module and re-run)" + ] else: general = "Volatility encountered an unexpected situation." detail = "" caused_by = [ - "Please re-run using with -vvv and file a bug with the output", f"at {constants.BUG_URL}" + "Please re-run using with -vvv and file a bug with the output", + f"at {constants.BUG_URL}", ] # Code that actually renders the exception @@ -447,27 +559,43 @@ class CommandLine: symbols_failed = False for config_path in excp.unsatisfied: translation_failed = translation_failed or isinstance( - excp.unsatisfied[config_path], configuration.requirements.TranslationLayerRequirement) - symbols_failed = symbols_failed or isinstance(excp.unsatisfied[config_path], - configuration.requirements.SymbolTableRequirement) + excp.unsatisfied[config_path], + configuration.requirements.TranslationLayerRequirement, + ) + symbols_failed = symbols_failed or isinstance( + excp.unsatisfied[config_path], + configuration.requirements.SymbolTableRequirement, + ) - print(f"Unsatisfied requirement {config_path}: {excp.unsatisfied[config_path].description}") + print( + f"Unsatisfied requirement {config_path}: {excp.unsatisfied[config_path].description}" + ) if translation_failed: - print("\nA translation layer requirement was not fulfilled. Please verify that:\n" - "\tA file was provided to create this layer (by -f, --single-location or by config)\n" - "\tThe file exists and is readable\n" - "\tThe file is a valid memory image and was acquired cleanly") + print( + "\nA translation layer requirement was not fulfilled. Please verify that:\n" + "\tA file was provided to create this layer (by -f, --single-location or by config)\n" + "\tThe file exists and is readable\n" + "\tThe file is a valid memory image and was acquired cleanly" + ) if symbols_failed: - print("\nA symbol table requirement was not fulfilled. Please verify that:\n" - "\tThe associated translation layer requirement was fulfilled\n" - "\tYou have the correct symbol file for the requirement\n" - "\tThe symbol file is under the correct directory or zip file\n" - "\tThe symbol file is named appropriately or contains the correct banner\n") + print( + "\nA symbol table requirement was not fulfilled. Please verify that:\n" + "\tThe associated translation layer requirement was fulfilled\n" + "\tYou have the correct symbol file for the requirement\n" + "\tThe symbol file is under the correct directory or zip file\n" + "\tThe symbol file is named appropriately or contains the correct banner\n" + ) - def populate_config(self, context: interfaces.context.ContextInterface, - configurables_list: Dict[str, Type[interfaces.configuration.ConfigurableInterface]], - args: argparse.Namespace, plugin_config_path: str) -> None: + def populate_config( + self, + context: interfaces.context.ContextInterface, + configurables_list: Dict[ + str, Type[interfaces.configuration.ConfigurableInterface] + ], + args: argparse.Namespace, + plugin_config_path: str, + ) -> None: """Populate the context config based on the returned args. We have already determined these elements must be descended from ConfigurableInterface @@ -489,34 +617,42 @@ class CommandLine: if not scheme or len(scheme) <= 1: if not os.path.exists(value): raise FileNotFoundError( - f"Non-existent file {value} passed to URIRequirement") + f"Non-existent file {value} passed to URIRequirement" + ) value = f"file://{request.pathname2url(os.path.abspath(value))}" if isinstance(requirement, requirements.ListRequirement): if not isinstance(value, list): - raise TypeError("Configuration for ListRequirement was not a list: {}".format( - requirement.name)) + raise TypeError( + "Configuration for ListRequirement was not a list: {}".format( + requirement.name + ) + ) value = [requirement.element_type(x) for x in value] if not inspect.isclass(configurables_list[configurable]): config_path = configurables_list[configurable].config_path else: # We must be the plugin, so name it appropriately: config_path = plugin_config_path - extended_path = interfaces.configuration.path_join(config_path, requirement.name) + extended_path = interfaces.configuration.path_join( + config_path, requirement.name + ) context.config[extended_path] = value - def file_handler_class_factory(self, direct = True): + def file_handler_class_factory(self, direct=True): output_dir = self.output_dir class CLIFileHandler(interfaces.plugins.FileHandlerInterface): - def _get_final_filename(self): """Gets the final filename""" if output_dir is None: raise TypeError("Output directory is not a string") - os.makedirs(output_dir, exist_ok = True) + os.makedirs(output_dir, exist_ok=True) - pref_name_array = self.preferred_filename.split('.') - filename, extension = os.path.join(output_dir, '.'.join(pref_name_array[:-1])), pref_name_array[-1] + pref_name_array = self.preferred_filename.split(".") + filename, extension = ( + os.path.join(output_dir, ".".join(pref_name_array[:-1])), + pref_name_array[-1], + ) output_filename = f"{filename}.{extension}" counter = 1 @@ -526,7 +662,6 @@ class CommandLine: return output_filename class CLIMemFileHandler(io.BytesIO, CLIFileHandler): - def __init__(self, filename: str): io.BytesIO.__init__(self) CLIFileHandler.__init__(self, filename) @@ -543,18 +678,26 @@ class CommandLine: with open(output_filename, "wb") as current_file: current_file.write(self.read()) self._committed = True - vollog.log(logging.INFO, f"Saved stored plugin file: {output_filename}") + vollog.log( + logging.INFO, f"Saved stored plugin file: {output_filename}" + ) super().close() class CLIDirectFileHandler(CLIFileHandler): - def __init__(self, filename: str): - fd, self._name = tempfile.mkstemp(suffix = '.vol3', prefix = 'tmp_', dir = output_dir) - self._file = io.open(fd, mode = 'w+b') + fd, self._name = tempfile.mkstemp( + suffix=".vol3", prefix="tmp_", dir=output_dir + ) + self._file = io.open(fd, mode="w+b") CLIFileHandler.__init__(self, filename) for item in dir(self._file): - if not item.startswith('_') and item not in ('closed', 'close', 'mode', 'name'): + if not item.startswith("_") and item not in ( + "closed", + "close", + "mode", + "name", + ): setattr(self, item, getattr(self._file, item)) def __getattr__(self, item): @@ -587,8 +730,11 @@ class CommandLine: else: return CLIMemFileHandler - def populate_requirements_argparse(self, parser: Union[argparse.ArgumentParser, argparse._ArgumentGroup], - configurable: Type[interfaces.configuration.ConfigurableInterface]): + def populate_requirements_argparse( + self, + parser: Union[argparse.ArgumentParser, argparse._ArgumentGroup], + configurable: Type[interfaces.configuration.ConfigurableInterface], + ): """Adds the plugin's simple requirements to the provided parser. Args: @@ -596,15 +742,22 @@ class CommandLine: configurable: The plugin object to pull the requirements from """ if not issubclass(configurable, interfaces.configuration.ConfigurableInterface): - raise TypeError(f"Expected ConfigurableInterface type, not: {type(configurable)}") + raise TypeError( + f"Expected ConfigurableInterface type, not: {type(configurable)}" + ) # Construct an argparse group for requirement in configurable.get_requirements(): additional: Dict[str, Any] = {} - if not isinstance(requirement, interfaces.configuration.RequirementInterface): - raise TypeError("Plugin contains requirements that are not RequirementInterfaces: {}".format( - configurable.__name__)) + if not isinstance( + requirement, interfaces.configuration.RequirementInterface + ): + raise TypeError( + "Plugin contains requirements that are not RequirementInterfaces: {}".format( + configurable.__name__ + ) + ) if isinstance(requirement, interfaces.configuration.SimpleTypeRequirement): additional["type"] = requirement.instance_type if isinstance(requirement, requirements.IntRequirement): @@ -613,21 +766,29 @@ class CommandLine: additional["action"] = "store_true" if "type" in additional: del additional["type"] - elif isinstance(requirement, volatility3.framework.configuration.requirements.ListRequirement): + elif isinstance( + requirement, + volatility3.framework.configuration.requirements.ListRequirement, + ): additional["type"] = requirement.element_type - nargs = '*' if requirement.optional else '+' + nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs - elif isinstance(requirement, volatility3.framework.configuration.requirements.ChoiceRequirement): + elif isinstance( + requirement, + volatility3.framework.configuration.requirements.ChoiceRequirement, + ): additional["type"] = str additional["choices"] = requirement.choices else: continue - parser.add_argument("--" + requirement.name.replace('_', '-'), - help = requirement.description, - default = requirement.default, - dest = requirement.name, - required = not requirement.optional, - **additional) + parser.add_argument( + "--" + requirement.name.replace("_", "-"), + help=requirement.description, + default=requirement.default, + dest=requirement.name, + required=not requirement.optional, + **additional, + ) def main(): diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 623153fae..5df378d08 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -44,9 +44,9 @@ def hex_bytes_as_text(value: bytes) -> str: ascii.append(chr(byte) if 0x20 < byte <= 0x7E else ".") if (count % 8) == 7: output += "\n" - output += " ".join(hex[count - 7:count + 1]) + output += " ".join(hex[count - 7 : count + 1]) output += "\t" - output += "".join(ascii[count - 7:count + 1]) + output += "".join(ascii[count - 7 : count + 1]) count += 1 return output @@ -58,10 +58,16 @@ def multitypedata_as_text(value: format_hints.MultiTypeData) -> str: """ if value.show_hex: return hex_bytes_as_text(value) - string_representation = str(value, encoding = value.encoding, errors = 'replace') - if value.split_nulls and ((len(value) / 2 - 1) <= len(string_representation) <= (len(value) / 2)): + string_representation = str(value, encoding=value.encoding, errors="replace") + if value.split_nulls and ( + (len(value) / 2 - 1) <= len(string_representation) <= (len(value) / 2) + ): return "\n".join(string_representation.split("\x00")) - if len(string_representation) - 1 <= len(string_representation.split("\x00")[0]) <= len(string_representation): + if ( + len(string_representation) - 1 + <= len(string_representation.split("\x00")[0]) + <= len(string_representation) + ): return string_representation.split("\x00")[0] return hex_bytes_as_text(value) @@ -87,9 +93,11 @@ def quoted_optional(func: Callable) -> Callable: return "" if isinstance(x, format_hints.MultiTypeData) and x.converted_int: return f"{result}" - if isinstance(x, int) and not isinstance(x, (format_hints.Hex, format_hints.Bin)): + if isinstance(x, int) and not isinstance( + x, (format_hints.Hex, format_hints.Bin) + ): return f"{result}" - return f"\"{result}\"" + return f'"{result}"' return wrapped @@ -106,14 +114,16 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str: if CAPSTONE_PRESENT: disasm_types = { - 'intel': capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32), - 'intel64': capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64), - 'arm': capstone.Cs(capstone.CS_ARCH_ARM, capstone.CS_MODE_ARM), - 'arm64': capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM) + "intel": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32), + "intel64": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64), + "arm": capstone.Cs(capstone.CS_ARCH_ARM, capstone.CS_MODE_ARM), + "arm64": capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM), } output = "" if disasm.architecture is not None: - for i in disasm_types[disasm.architecture].disasm(disasm.data, disasm.offset): + for i in disasm_types[disasm.architecture].disasm( + disasm.data, disasm.offset + ): output += f"\n0x{i.address:x}:\t{i.mnemonic}\t{i.op_str}" return output return QuickTextRenderer._type_renderers[bytes](disasm.data) @@ -121,6 +131,7 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str: class CLIRenderer(interfaces.renderers.Renderer): """Class to add specific requirements for CLI renderers.""" + name = "unnamed" structured_output = False @@ -134,7 +145,7 @@ class QuickTextRenderer(CLIRenderer): interfaces.renderers.Disassembly: optional(display_disassembly), bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), - 'default': optional(lambda x: f"{x}") + "default": optional(lambda x: f"{x}"), } name = "quick" @@ -163,11 +174,16 @@ class QuickTextRenderer(CLIRenderer): def visitor(node: interfaces.renderers.TreeNode, accumulator): accumulator.write("\n") # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case - accumulator.write("*" * max(0, node.path_depth - 1) + ("" if (node.path_depth <= 1) else " ")) + accumulator.write( + "*" * max(0, node.path_depth - 1) + + ("" if (node.path_depth <= 1) else " ") + ) line = [] for column_index in range(len(grid.columns)): column = grid.columns[column_index] - renderer = self._type_renderers.get(column.type, self._type_renderers['default']) + renderer = self._type_renderers.get( + column.type, self._type_renderers["default"] + ) line.append(renderer(node.values[column_index])) accumulator.write("{}".format("\t".join(line))) accumulator.flush() @@ -176,13 +192,14 @@ class QuickTextRenderer(CLIRenderer): if not grid.populated: grid.populate(visitor, outfd) else: - grid.visit(node = None, function = visitor, initial_accumulator = outfd) + grid.visit(node=None, function=visitor, initial_accumulator=outfd) outfd.write("\n") class NoneRenderer(CLIRenderer): """Outputs no results""" + name = "none" def get_render_options(self): @@ -202,7 +219,7 @@ class CSVRenderer(CLIRenderer): interfaces.renderers.Disassembly: optional(display_disassembly), bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), - 'default': optional(lambda x: f"{x}") + "default": optional(lambda x: f"{x}"), } name = "csv" @@ -219,28 +236,30 @@ class CSVRenderer(CLIRenderer): """ outfd = sys.stdout - header_list = ['TreeDepth'] + header_list = ["TreeDepth"] for column in grid.columns: # Ignore the type because namedtuples don't realize they have accessible attributes header_list.append(f"{column.name}") - writer = csv.DictWriter(outfd, header_list, lineterminator='\n') + writer = csv.DictWriter(outfd, header_list, lineterminator="\n") writer.writeheader() def visitor(node: interfaces.renderers.TreeNode, accumulator): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case - row = {'TreeDepth': str(max(0, node.path_depth - 1))} + row = {"TreeDepth": str(max(0, node.path_depth - 1))} for column_index in range(len(grid.columns)): column = grid.columns[column_index] - renderer = self._type_renderers.get(column.type, self._type_renderers['default']) - row[f'{column.name}'] = renderer(node.values[column_index]) + renderer = self._type_renderers.get( + column.type, self._type_renderers["default"] + ) + row[f"{column.name}"] = renderer(node.values[column_index]) accumulator.writerow(row) return accumulator if not grid.populated: grid.populate(visitor, writer) else: - grid.visit(node = None, function = visitor, initial_accumulator = writer) + grid.visit(node=None, function=visitor, initial_accumulator=writer) outfd.write("\n") @@ -270,23 +289,34 @@ class PrettyTextRenderer(CLIRenderer): display_alignment = ">" column_separator = " | " - tree_indent_column = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20)) - max_column_widths = dict([(column.name, len(column.name)) for column in grid.columns]) + tree_indent_column = "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(20) + ) + max_column_widths = dict( + [(column.name, len(column.name)) for column in grid.columns] + ) def visitor( - node: interfaces.renderers.TreeNode, - accumulator: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]] + node: interfaces.renderers.TreeNode, + accumulator: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]], ) -> List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]]: # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case - max_column_widths[tree_indent_column] = max(max_column_widths.get(tree_indent_column, 0), node.path_depth) + max_column_widths[tree_indent_column] = max( + max_column_widths.get(tree_indent_column, 0), node.path_depth + ) line = {} for column_index in range(len(grid.columns)): column = grid.columns[column_index] - renderer = self._type_renderers.get(column.type, self._type_renderers['default']) + renderer = self._type_renderers.get( + column.type, self._type_renderers["default"] + ) data = renderer(node.values[column_index]) - field_width = max([len(self.tab_stop(x)) for x in f"{data}".split("\n")]) - max_column_widths[column.name] = max(max_column_widths.get(column.name, len(column.name)), - field_width) + field_width = max( + [len(self.tab_stop(x)) for x in f"{data}".split("\n")] + ) + max_column_widths[column.name] = max( + max_column_widths.get(column.name, len(column.name)), field_width + ) line[column] = data.split("\n") accumulator.append((node.path_depth, line)) return accumulator @@ -295,14 +325,22 @@ class PrettyTextRenderer(CLIRenderer): if not grid.populated: grid.populate(visitor, final_output) else: - grid.visit(node = None, function = visitor, initial_accumulator = final_output) + grid.visit(node=None, function=visitor, initial_accumulator=final_output) # Always align the tree to the left - format_string_list = ["{0:<" + str(max_column_widths.get(tree_indent_column, 0)) + "s}"] + format_string_list = [ + "{0:<" + str(max_column_widths.get(tree_indent_column, 0)) + "s}" + ] for column_index in range(len(grid.columns)): column = grid.columns[column_index] - format_string_list.append("{" + str(column_index + 1) + ":" + display_alignment + - str(max_column_widths[column.name]) + "s}") + format_string_list.append( + "{" + + str(column_index + 1) + + ":" + + display_alignment + + str(max_column_widths[column.name]) + + "s}" + ) format_string = column_separator.join(format_string_list) + "\n" @@ -314,14 +352,30 @@ class PrettyTextRenderer(CLIRenderer): line[column] = line[column] + ([""] * (nums_line - len(line[column]))) for index in range(nums_line): if index == 0: - outfd.write(format_string.format("*" * depth, *[self.tab_stop(line[column][index]) for column in grid.columns])) + outfd.write( + format_string.format( + "*" * depth, + *[ + self.tab_stop(line[column][index]) + for column in grid.columns + ], + ) + ) else: - outfd.write(format_string.format(" " * depth, *[self.tab_stop(line[column][index]) for column in grid.columns])) + outfd.write( + format_string.format( + " " * depth, + *[ + self.tab_stop(line[column][index]) + for column in grid.columns + ], + ) + ) def tab_stop(self, line: str) -> str: tab_width = 8 - while line.find('\t') >= 0: - i = line.find('\t') + while line.find("\t") >= 0: + i = line.find("\t") pad = " " * (tab_width - (i % tab_width)) line = line.replace("\t", pad, 1) return line @@ -333,11 +387,13 @@ class JsonRenderer(CLIRenderer): interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), - datetime.datetime: lambda x: x.isoformat() if not isinstance(x, interfaces.renderers.BaseAbsentValue) else None, - 'default': lambda x: x + datetime.datetime: lambda x: x.isoformat() + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else None, + "default": lambda x: x, } - name = 'JSON' + name = "JSON" structured_output = True def get_render_options(self) -> List[interfaces.renderers.RenderOption]: @@ -345,30 +401,35 @@ class JsonRenderer(CLIRenderer): def output_result(self, outfd, result): """Outputs the JSON data to a file in a particular format""" - outfd.write("{}\n".format(json.dumps(result, indent = 2, sort_keys = True))) + outfd.write("{}\n".format(json.dumps(result, indent=2, sort_keys=True))) def render(self, grid: interfaces.renderers.TreeGrid): outfd = sys.stdout outfd.write("\n") - final_output: Tuple[Dict[str, List[interfaces.renderers.TreeNode]], List[interfaces.renderers.TreeNode]] = ( - {}, []) + final_output: Tuple[ + Dict[str, List[interfaces.renderers.TreeNode]], + List[interfaces.renderers.TreeNode], + ] = ({}, []) def visitor( - node: interfaces.renderers.TreeNode, accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]] + node: interfaces.renderers.TreeNode, + accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]], ) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]: # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case acc_map, final_tree = accumulator - node_dict: Dict[str, Any] = {'__children': []} + node_dict: Dict[str, Any] = {"__children": []} for column_index in range(len(grid.columns)): column = grid.columns[column_index] - renderer = self._type_renderers.get(column.type, self._type_renderers['default']) + renderer = self._type_renderers.get( + column.type, self._type_renderers["default"] + ) data = renderer(list(node.values)[column_index]) if isinstance(data, interfaces.renderers.BaseAbsentValue): data = None node_dict[column.name] = data if node.parent: - acc_map[node.parent.path]['__children'].append(node_dict) + acc_map[node.parent.path]["__children"].append(node_dict) else: final_tree.append(node_dict) acc_map[node.path] = node_dict @@ -378,16 +439,16 @@ class JsonRenderer(CLIRenderer): if not grid.populated: grid.populate(visitor, final_output) else: - grid.visit(node = None, function = visitor, initial_accumulator = final_output) + grid.visit(node=None, function=visitor, initial_accumulator=final_output) self.output_result(outfd, final_output[1]) class JsonLinesRenderer(JsonRenderer): - name = 'JSONL' + name = "JSONL" def output_result(self, outfd, result): """Outputs the JSON results as JSON lines""" for line in result: - outfd.write(json.dumps(line, sort_keys = True)) + outfd.write(json.dumps(line, sort_keys=True)) outfd.write("\n") diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index 8ba807fee..dd89a64fd 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -24,13 +24,15 @@ class HelpfulSubparserAction(argparse._SubParsersAction): # We don't want the action self-check to kick in, so we remove the choices list, the check happens in __call__ self.choices = None - def __call__(self, - parser: argparse.ArgumentParser, - namespace: argparse.Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[str] = None) -> None: + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: Union[str, Sequence[Any], None], + option_string: Optional[str] = None, + ) -> None: - parser_name = '' + parser_name = "" arg_strings = [] # type: List[str] if values is not None: for value in values: @@ -43,7 +45,9 @@ class HelpfulSubparserAction(argparse._SubParsersAction): if self.dest != argparse.SUPPRESS: setattr(namespace, self.dest, parser_name) - matched_parsers = [name for name in self._name_parser_map if parser_name in name] + matched_parsers = [ + name for name in self._name_parser_map if parser_name in name + ] if len(matched_parsers) < 1: msg = f"invalid choice {parser_name} (choose from {', '.join(self._name_parser_map)})" @@ -52,7 +56,7 @@ class HelpfulSubparserAction(argparse._SubParsersAction): msg = f"plugin {parser_name} matches multiple plugins ({', '.join(matched_parsers)})" raise argparse.ArgumentError(self, msg) parser = self._name_parser_map[matched_parsers[0]] - setattr(namespace, 'plugin', matched_parsers[0]) + setattr(namespace, "plugin", matched_parsers[0]) # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top @@ -71,7 +75,6 @@ class HelpfulSubparserAction(argparse._SubParsersAction): class HelpfulArgParser(argparse.ArgumentParser): - def _match_argument(self, action, arg_strings_pattern) -> int: # match the pattern for this action to the arg strings nargs_pattern = self._get_nargs_pattern(action) @@ -80,13 +83,18 @@ class HelpfulArgParser(argparse.ArgumentParser): # raise an exception if we weren't able to find a match if match is None: nargs_errors = { - None: gettext.gettext('expected one argument'), - argparse.OPTIONAL: gettext.gettext('expected at most one argument'), - argparse.ONE_OR_MORE: gettext.gettext('expected at least one argument'), + None: gettext.gettext("expected one argument"), + argparse.OPTIONAL: gettext.gettext("expected at most one argument"), + argparse.ONE_OR_MORE: gettext.gettext("expected at least one argument"), } msg = nargs_errors.get(action.nargs) if msg is None: - msg = gettext.ngettext('expected %s argument', 'expected %s arguments', action.nargs) % action.nargs + msg = ( + gettext.ngettext( + "expected %s argument", "expected %s arguments", action.nargs + ) + % action.nargs + ) if action.choices: msg = f"{msg} (from: {', '.join(action.choices)})" raise argparse.ArgumentError(action, msg) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index f32a587e0..3b982394f 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -12,7 +12,14 @@ import volatility3.plugins import volatility3.symbols from volatility3 import cli, framework from volatility3.cli.volshell import generic, linux, mac, windows -from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins +from volatility3.framework import ( + automagic, + constants, + contexts, + exceptions, + interfaces, + plugins, +) # Make sure we log everything vollog = logging.getLogger() @@ -20,7 +27,7 @@ vollog.setLevel(0) # Trim the console down by default console = logging.StreamHandler() console.setLevel(logging.WARNING) -formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') +formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) vollog.addHandler(console) @@ -40,84 +47,139 @@ class VolShell(cli.CommandLine): def run(self): """Executes the command line module, taking the system arguments, determining the plugin to run and then running it.""" - sys.stdout.write(f"Volshell (Volatility 3 Framework) {constants.PACKAGE_VERSION}\n") + sys.stdout.write( + f"Volshell (Volatility 3 Framework) {constants.PACKAGE_VERSION}\n" + ) framework.require_interface_version(2, 0, 0) - parser = argparse.ArgumentParser(prog = self.CLI_NAME, - description = "A tool for interactivate forensic analysis of memory images") - parser.add_argument("-c", - "--config", - help = "Load the configuration from a json file", - default = None, - type = str) - parser.add_argument("-e", - "--extend", - help = "Extend the configuration with a new (or changed) setting", - default = None, - action = 'append') - parser.add_argument("-p", - "--plugin-dirs", - help = "Semi-colon separated list of paths to find plugins", - default = "", - type = str) - parser.add_argument("-s", - "--symbol-dirs", - help = "Semi-colon separated list of paths to find symbols", - default = "", - type = str) - parser.add_argument("-v", "--verbosity", help = "Increase output verbosity", default = 0, action = "count") - parser.add_argument("-o", - "--output-dir", - help = "Directory in which to output any generated files", - default = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')), - type = str) - parser.add_argument("-q", "--quiet", help = "Remove progress feedback", default = False, action = 'store_true') - parser.add_argument("--log", help = "Log output to a file as well as the console", default = None, type = str) - parser.add_argument("-f", - "--file", - metavar = 'FILE', - default = None, - type = str, - help = "Shorthand for --single-location=file:// if single-location is not defined") - parser.add_argument("--write-config", - help = "Write configuration JSON file out to config.json", - default = False, - action = 'store_true') - parser.add_argument("--save-config", - help = "Save configuration JSON file to a file", - default = None, - type = str) - parser.add_argument("--clear-cache", - help = "Clears out all short-term cached items", - default = False, - action = 'store_true') - parser.add_argument("--cache-path", - help = f"Change the default path ({constants.CACHE_PATH}) used to store the cache", - default = constants.CACHE_PATH, - type = str) + parser = argparse.ArgumentParser( + prog=self.CLI_NAME, + description="A tool for interactivate forensic analysis of memory images", + ) + parser.add_argument( + "-c", + "--config", + help="Load the configuration from a json file", + default=None, + type=str, + ) + parser.add_argument( + "-e", + "--extend", + help="Extend the configuration with a new (or changed) setting", + default=None, + action="append", + ) + parser.add_argument( + "-p", + "--plugin-dirs", + help="Semi-colon separated list of paths to find plugins", + default="", + type=str, + ) + parser.add_argument( + "-s", + "--symbol-dirs", + help="Semi-colon separated list of paths to find symbols", + default="", + type=str, + ) + parser.add_argument( + "-v", + "--verbosity", + help="Increase output verbosity", + default=0, + action="count", + ) + parser.add_argument( + "-o", + "--output-dir", + help="Directory in which to output any generated files", + default=os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..") + ), + type=str, + ) + parser.add_argument( + "-q", + "--quiet", + help="Remove progress feedback", + default=False, + action="store_true", + ) + parser.add_argument( + "--log", + help="Log output to a file as well as the console", + default=None, + type=str, + ) + parser.add_argument( + "-f", + "--file", + metavar="FILE", + default=None, + type=str, + help="Shorthand for --single-location=file:// if single-location is not defined", + ) + parser.add_argument( + "--write-config", + help="Write configuration JSON file out to config.json", + default=False, + action="store_true", + ) + parser.add_argument( + "--save-config", + help="Save configuration JSON file to a file", + default=None, + type=str, + ) + parser.add_argument( + "--clear-cache", + help="Clears out all short-term cached items", + default=False, + action="store_true", + ) + parser.add_argument( + "--cache-path", + help=f"Change the default path ({constants.CACHE_PATH}) used to store the cache", + default=constants.CACHE_PATH, + type=str, + ) # Volshell specific flags - os_specific = parser.add_mutually_exclusive_group(required = False) - os_specific.add_argument("-w", - "--windows", - default = False, - action = "store_true", - help = "Run a Windows volshell") - os_specific.add_argument("-l", "--linux", default = False, action = "store_true", help = "Run a Linux volshell") - os_specific.add_argument("-m", "--mac", default = False, action = "store_true", help = "Run a Mac volshell") + os_specific = parser.add_mutually_exclusive_group(required=False) + os_specific.add_argument( + "-w", + "--windows", + default=False, + action="store_true", + help="Run a Windows volshell", + ) + os_specific.add_argument( + "-l", + "--linux", + default=False, + action="store_true", + help="Run a Linux volshell", + ) + os_specific.add_argument( + "-m", "--mac", default=False, action="store_true", help="Run a Mac volshell" + ) # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. - known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h'] + known_args = [arg for arg in sys.argv if arg != "--help" and arg != "-h"] partial_args, _ = parser.parse_known_args(known_args) if partial_args.plugin_dirs: - volatility3.plugins.__path__ = [os.path.abspath(p) - for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH + volatility3.plugins.__path__ = [ + os.path.abspath(p) for p in partial_args.plugin_dirs.split(";") + ] + constants.PLUGINS_PATH if partial_args.symbol_dirs: - volatility3.symbols.__path__ = [os.path.abspath(p) - for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS + volatility3.symbols.__path__ = [ + os.path.abspath(p) for p in partial_args.symbol_dirs.split(";") + ] + constants.SYMBOL_BASEPATHS if partial_args.cache_path: constants.CACHE_PATH = partial_args.cache_path @@ -128,8 +190,10 @@ class VolShell(cli.CommandLine): if partial_args.log: file_logger = logging.FileHandler(partial_args.log) file_logger.setLevel(0) - file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', - fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + file_formatter = logging.Formatter( + datefmt="%y-%m-%d %H:%M:%S", + fmt="%(asctime)s %(name)-12s %(levelname)-8s %(message)s", + ) file_logger.setFormatter(file_formatter) vollog.addHandler(file_logger) vollog.info("Logging started") @@ -144,11 +208,14 @@ class VolShell(cli.CommandLine): # Do the initialization ctx = contexts.Context() # Construct a blank context - failures = framework.import_files(volatility3.plugins, - True) # Will not log as console's default level is WARNING + failures = framework.import_files( + volatility3.plugins, True + ) # Will not log as console's default level is WARNING if failures: - parser.epilog = "The following plugins could not be loaded (use -vv to see why): " + \ - ", ".join(sorted(failures)) + parser.epilog = ( + "The following plugins could not be loaded (use -vv to see why): " + + ", ".join(sorted(failures)) + ) vollog.info(parser.epilog) automagics = automagic.available(ctx) @@ -166,11 +233,17 @@ class VolShell(cli.CommandLine): configurables_list[amagic.__class__.__name__] = amagic # We don't list plugin arguments, because they can be provided within python - volshell_plugin_list = {'generic': generic.Volshell, 'windows': windows.Volshell} + volshell_plugin_list = { + "generic": generic.Volshell, + "windows": windows.Volshell, + } for plugin in volshell_plugin_list: - subparser = parser.add_argument_group(title = plugin.capitalize(), - description = "Configuration options based on {} options".format( - plugin.capitalize())) + subparser = parser.add_argument_group( + title=plugin.capitalize(), + description="Configuration options based on {} options".format( + plugin.capitalize() + ), + ) self.populate_requirements_argparse(subparser, volshell_plugin_list[plugin]) configurables_list[plugin] = volshell_plugin_list[plugin] @@ -182,7 +255,9 @@ class VolShell(cli.CommandLine): # Run the argparser args = parser.parse_args() - vollog.log(constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}") + vollog.log( + constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}" + ) plugin = generic.Volshell if args.windows: @@ -193,7 +268,9 @@ class VolShell(cli.CommandLine): plugin = mac.Volshell base_config_path = "plugins" - plugin_config_path = interfaces.configuration.path_join(base_config_path, plugin.__name__) + plugin_config_path = interfaces.configuration.path_join( + base_config_path, plugin.__name__ + ) # Special case the -f argument because people use is so frequently # It has to go here so it can be overridden by single-location if it's defined @@ -202,7 +279,7 @@ class VolShell(cli.CommandLine): if args.file: try: single_location = self.location_from_file(args.file) - ctx.config['automagic.LayerStacker.single_location'] = single_location + ctx.config["automagic.LayerStacker.single_location"] = single_location except ValueError as excp: parser.error(str(excp)) @@ -210,15 +287,22 @@ class VolShell(cli.CommandLine): if args.config: with open(args.config, "r") as f: json_val = json.load(f) - ctx.config.splice(plugin_config_path, interfaces.configuration.HierarchicalDict(json_val)) + ctx.config.splice( + plugin_config_path, + interfaces.configuration.HierarchicalDict(json_val), + ) self.populate_config(ctx, configurables_list, args, plugin_config_path) if args.extend: for extension in args.extend: - if '=' not in extension: - raise ValueError("Invalid extension (extensions must be of the format \"conf.path.value='value'\")") - address, value = extension[:extension.find('=')], json.loads(extension[extension.find('=') + 1:]) + if "=" not in extension: + raise ValueError( + "Invalid extension (extensions must be of the format \"conf.path.value='value'\")" + ) + address, value = extension[: extension.find("=")], json.loads( + extension[extension.find("=") + 1 :] + ) ctx.config[address] = value # It should be up to the UI to determine which automagics to run, so this is before BACK TO THE FRAMEWORK @@ -233,22 +317,40 @@ class VolShell(cli.CommandLine): if args.quiet: progress_callback = cli.MuteProgress() - constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, - self.file_handler_class_factory()) + constructed = plugins.construct_plugin( + ctx, + automagics, + plugin, + base_config_path, + progress_callback, + self.file_handler_class_factory(), + ) if args.write_config: - vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') - args.save_config = 'config.json' + vollog.warning( + "Use of --write-config has been deprecated, replaced by --save-config " + ) + args.save_config = "config.json" if args.save_config: vollog.debug("Writing out configuration data to {args.save_config}") if os.path.exists(os.path.abspath(args.save_config)): - parser.error(f"Cannot write configuration: file {args.save_config} already exists") + parser.error( + f"Cannot write configuration: file {args.save_config} already exists" + ) with open(args.save_config, "w") as f: - json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + json.dump( + dict(constructed.build_configuration()), + f, + sort_keys=True, + indent=2, + ) f.write("\n") except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) - parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") + parser.exit( + 1, + f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n", + ) try: # Construct and run the plugin diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 274b6ca17..df369c53e 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -26,6 +26,7 @@ except ImportError: class Volshell(interfaces.plugins.PluginInterface): """Shell environment to directly interact with a memory image.""" + _required_framework_version = (2, 0, 0) def __init__(self, *args, **kwargs): @@ -36,23 +37,29 @@ class Volshell(interfaces.plugins.PluginInterface): self.__console = None def random_string(self, length: int = 32) -> str: - return ''.join(random.sample(string.ascii_uppercase + string.digits, length)) + return "".join(random.sample(string.ascii_uppercase + string.digits, length)) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: reqs: List[interfaces.configuration.RequirementInterface] = [] if cls == Volshell: reqs = [ - requirements.URIRequirement(name = 'script', - description = 'File to load and execute at start', - default = None, - optional = True) + requirements.URIRequirement( + name="script", + description="File to load and execute at start", + default=None, + optional=True, + ) ] return reqs + [ - requirements.TranslationLayerRequirement(name = 'primary', description = 'Memory layer for the kernel'), + requirements.TranslationLayerRequirement( + name="primary", description="Memory layer for the kernel" + ), ] - def run(self, additional_locals: Dict[str, Any] = None) -> interfaces.renderers.TreeGrid: + def run( + self, additional_locals: Dict[str, Any] = None + ) -> interfaces.renderers.TreeGrid: """Runs the interactive volshell plugin. Returns: @@ -66,14 +73,15 @@ class Volshell(interfaces.plugins.PluginInterface): pass else: import rlcompleter - completer = rlcompleter.Completer(namespace = self._construct_locals_dict()) + + completer = rlcompleter.Completer(namespace=self._construct_locals_dict()) readline.set_completer(completer.complete) readline.parse_and_bind("tab: complete") print("Readline imported successfully") # TODO: provide help, consider generic functions (pslist?) and/or providing windows/linux functions - mode = self.__module__.split('.')[-1] + mode = self.__module__.split(".")[-1] mode = mode[0].upper() + mode[1:] banner = f""" @@ -86,13 +94,13 @@ class Volshell(interfaces.plugins.PluginInterface): """ sys.ps1 = f"({self.current_layer}) >>> " - self.__console = code.InteractiveConsole(locals = self._construct_locals_dict()) + self.__console = code.InteractiveConsole(locals=self._construct_locals_dict()) # Since we have to do work to add the option only once for all different modes of volshell, we can't # rely on the default having been set - if self.config.get('script', None) is not None: - self.run_script(location = self.config['script']) + if self.config.get("script", None) is not None: + self.run_script(location=self.config["script"]) - self.__console.interact(banner = banner) + self.__console.interact(banner=banner) return renderers.TreeGrid([("Terminating", str)], None) @@ -119,47 +127,70 @@ class Volshell(interfaces.plugins.PluginInterface): def construct_locals(self) -> List[Tuple[List[str], Any]]: """Returns a dictionary listing the functions to be added to the environment.""" - return [(['dt', 'display_type'], self.display_type), (['db', 'display_bytes'], self.display_bytes), - (['dw', 'display_words'], self.display_words), (['dd', - 'display_doublewords'], self.display_doublewords), - (['dq', 'display_quadwords'], self.display_quadwords), (['dis', 'disassemble'], self.disassemble), - (['cl', 'change_layer'], self.change_layer), - (['cs', 'change_symboltable'], self.change_symbol_table), - (['ck', 'change_kernel'], self.change_kernel), - (['context'], self.context), (['self'], self), - (['dpo', 'display_plugin_output'], self.display_plugin_output), - (['gt', 'generate_treegrid'], self.generate_treegrid), (['rt', - 'render_treegrid'], self.render_treegrid), - (['ds', 'display_symbols'], self.display_symbols), (['hh', 'help'], self.help), - (['cc', 'create_configurable'], self.create_configurable), (['lf', 'load_file'], self.load_file), - (['rs', 'run_script'], self.run_script)] + return [ + (["dt", "display_type"], self.display_type), + (["db", "display_bytes"], self.display_bytes), + (["dw", "display_words"], self.display_words), + (["dd", "display_doublewords"], self.display_doublewords), + (["dq", "display_quadwords"], self.display_quadwords), + (["dis", "disassemble"], self.disassemble), + (["cl", "change_layer"], self.change_layer), + (["cs", "change_symboltable"], self.change_symbol_table), + (["ck", "change_kernel"], self.change_kernel), + (["context"], self.context), + (["self"], self), + (["dpo", "display_plugin_output"], self.display_plugin_output), + (["gt", "generate_treegrid"], self.generate_treegrid), + (["rt", "render_treegrid"], self.render_treegrid), + (["ds", "display_symbols"], self.display_symbols), + (["hh", "help"], self.help), + (["cc", "create_configurable"], self.create_configurable), + (["lf", "load_file"], self.load_file), + (["rs", "run_script"], self.run_script), + ] def _construct_locals_dict(self) -> Dict[str, Any]: - """Returns a dictionary of the locals """ + """Returns a dictionary of the locals""" result = {} for aliases, value in self.construct_locals(): for alias in aliases: result[alias] = value return result - def _read_data(self, offset, count = 128, layer_name = None): + def _read_data(self, offset, count=128, layer_name=None): """Reads the bytes necessary for the display_* methods""" return self.context.layers[layer_name or self.current_layer].read(offset, count) - def _display_data(self, offset: int, remaining_data: bytes, format_string: str = "B", ascii: bool = True): + def _display_data( + self, + offset: int, + remaining_data: bytes, + format_string: str = "B", + ascii: bool = True, + ): """Display a series of bytes""" chunk_size = struct.calcsize(format_string) data_length = len(remaining_data) - remaining_data = remaining_data[:data_length - (data_length % chunk_size)] + remaining_data = remaining_data[: data_length - (data_length % chunk_size)] while remaining_data: current_line, remaining_data = remaining_data[:16], remaining_data[16:] - data_blocks = [current_line[chunk_size * i:chunk_size * (i + 1)] for i in range(16 // chunk_size)] - data_blocks = [x for x in data_blocks if x != b''] - valid_data = [("{:0" + str(2 * chunk_size) + "x}").format(struct.unpack(format_string, x)[0]) - for x in data_blocks] - padding_data = [" " * 2 * chunk_size for _ in range((16 - len(current_line)) // chunk_size)] + data_blocks = [ + current_line[chunk_size * i : chunk_size * (i + 1)] + for i in range(16 // chunk_size) + ] + data_blocks = [x for x in data_blocks if x != b""] + valid_data = [ + ("{:0" + str(2 * chunk_size) + "x}").format( + struct.unpack(format_string, x)[0] + ) + for x in data_blocks + ] + padding_data = [ + " " * 2 * chunk_size + for _ in range((16 - len(current_line)) // chunk_size) + ] hex_data = " ".join(valid_data + padding_data) ascii_data = "" @@ -175,12 +206,14 @@ class Volshell(interfaces.plugins.PluginInterface): @staticmethod def _ascii_bytes(bytes): """Converts bytes into an ascii string""" - return "".join([chr(x) if 32 < x < 127 else '.' for x in binascii.unhexlify(bytes)]) + return "".join( + [chr(x) if 32 < x < 127 else "." for x in binascii.unhexlify(bytes)] + ) @property def current_layer(self): if self.__current_layer is None: - self.__current_layer = self.config['primary'] + self.__current_layer = self.config["primary"] return self.__current_layer @property @@ -192,7 +225,7 @@ class Volshell(interfaces.plugins.PluginInterface): @property def current_kernel_name(self): if self.__current_kernel_name is None: - self.__current_kernel_name = self.config.get('kernel', None) + self.__current_kernel_name = self.config.get("kernel", None) return self.__current_kernel_name @property @@ -217,7 +250,9 @@ class Volshell(interfaces.plugins.PluginInterface): if not symbol_table_name: print("No symbol table provided, not changing current symbol table") if symbol_table_name not in self.context.symbol_space: - print(f"Symbol table {symbol_table_name} not present in context symbol_space") + print( + f"Symbol table {symbol_table_name} not present in context symbol_space" + ) else: self.__current_symbol_table = symbol_table_name print(f"Current Symbol Table: {self.current_symbol_table}") @@ -231,51 +266,64 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_kernel_name = kernel_name print(f"Current kernel : {self.current_kernel_name}") - def display_bytes(self, offset, count = 128, layer_name = None): + def display_bytes(self, offset, count=128, layer_name=None): """Displays byte values and ASCII characters""" - remaining_data = self._read_data(offset, count = count, layer_name = layer_name) + remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data) - def display_quadwords(self, offset, count = 128, layer_name = None): + def display_quadwords(self, offset, count=128, layer_name=None): """Displays quad-word values (8 bytes) and corresponding ASCII characters""" - remaining_data = self._read_data(offset, count = count, layer_name = layer_name) - self._display_data(offset, remaining_data, format_string = "Q") + remaining_data = self._read_data(offset, count=count, layer_name=layer_name) + self._display_data(offset, remaining_data, format_string="Q") - def display_doublewords(self, offset, count = 128, layer_name = None): + def display_doublewords(self, offset, count=128, layer_name=None): """Displays double-word values (4 bytes) and corresponding ASCII characters""" - remaining_data = self._read_data(offset, count = count, layer_name = layer_name) - self._display_data(offset, remaining_data, format_string = "I") + remaining_data = self._read_data(offset, count=count, layer_name=layer_name) + self._display_data(offset, remaining_data, format_string="I") - def display_words(self, offset, count = 128, layer_name = None): + def display_words(self, offset, count=128, layer_name=None): """Displays word values (2 bytes) and corresponding ASCII characters""" - remaining_data = self._read_data(offset, count = count, layer_name = layer_name) - self._display_data(offset, remaining_data, format_string = "H") + remaining_data = self._read_data(offset, count=count, layer_name=layer_name) + self._display_data(offset, remaining_data, format_string="H") - def disassemble(self, offset, count = 128, layer_name = None, architecture = None): + def disassemble(self, offset, count=128, layer_name=None, architecture=None): """Disassembles a number of instructions from the code at offset""" - remaining_data = self._read_data(offset, count = count, layer_name = layer_name) + remaining_data = self._read_data(offset, count=count, layer_name=layer_name) if not has_capstone: - print("Capstone not available - please install it to use the disassemble command") + print( + "Capstone not available - please install it to use the disassemble command" + ) else: - if isinstance(self.context.layers[layer_name or self.current_layer], intel.Intel32e): - architecture = 'intel64' - elif isinstance(self.context.layers[layer_name or self.current_layer], intel.Intel): - architecture = 'intel' + if isinstance( + self.context.layers[layer_name or self.current_layer], intel.Intel32e + ): + architecture = "intel64" + elif isinstance( + self.context.layers[layer_name or self.current_layer], intel.Intel + ): + architecture = "intel" disasm_types = { - 'intel': capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32), - 'intel64': capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64), - 'arm': capstone.Cs(capstone.CS_ARCH_ARM, capstone.CS_MODE_ARM), - 'arm64': capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM) + "intel": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32), + "intel64": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64), + "arm": capstone.Cs(capstone.CS_ARCH_ARM, capstone.CS_MODE_ARM), + "arm64": capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM), } if architecture is not None: for i in disasm_types[architecture].disasm(remaining_data, offset): print(f"0x{i.address:x}:\t{i.mnemonic}\t{i.op_str}") - def display_type(self, - object: Union[str, interfaces.objects.ObjectInterface, interfaces.objects.Template], - offset: int = None): + def display_type( + self, + object: Union[ + str, interfaces.objects.ObjectInterface, interfaces.objects.Template + ], + offset: int = None, + ): """Display Type describes the members of a particular object in alphabetical order""" - if not isinstance(object, (str, interfaces.objects.ObjectInterface, interfaces.objects.Template)): + if not isinstance( + object, + (str, interfaces.objects.ObjectInterface, interfaces.objects.Template), + ): print("Cannot display information about non-type object") return @@ -287,20 +335,29 @@ class Volshell(interfaces.plugins.PluginInterface): volobject = self.context.symbol_space.get_type(object) else: # Str and offset - volobject = self.context.object(object, layer_name = self.current_layer, offset = offset) + volobject = self.context.object( + object, layer_name=self.current_layer, offset=offset + ) if offset is not None: - volobject = self.context.object(volobject.vol.type_name, layer_name = self.current_layer, offset = offset) + volobject = self.context.object( + volobject.vol.type_name, layer_name=self.current_layer, offset=offset + ) - if hasattr(volobject.vol, 'size'): + if hasattr(volobject.vol, "size"): print(f"{volobject.vol.type_name} ({volobject.vol.size} bytes)") - elif hasattr(volobject.vol, 'data_format'): + elif hasattr(volobject.vol, "data_format"): data_format = volobject.vol.data_format - print("{} ({} bytes, {} endian, {})".format(volobject.vol.type_name, data_format.length, - data_format.byteorder, - 'signed' if data_format.signed else 'unsigned')) + print( + "{} ({} bytes, {} endian, {})".format( + volobject.vol.type_name, + data_format.length, + data_format.byteorder, + "signed" if data_format.signed else "unsigned", + ) + ) - if hasattr(volobject.vol, 'members'): + if hasattr(volobject.vol, "members"): longest_member = longest_offset = longest_typename = 0 for member in volobject.vol.members: relative_offset, member_type = volobject.vol.members[member] @@ -308,20 +365,37 @@ class Volshell(interfaces.plugins.PluginInterface): longest_offset = max(len(hex(relative_offset)), longest_offset) longest_typename = max(len(member_type.vol.type_name), longest_typename) - for member in sorted(volobject.vol.members, key = lambda x: (volobject.vol.members[x][0], x)): + for member in sorted( + volobject.vol.members, key=lambda x: (volobject.vol.members[x][0], x) + ): relative_offset, member_type = volobject.vol.members[member] len_offset = len(hex(relative_offset)) len_member = len(member) len_typename = len(member_type.vol.type_name) if isinstance(volobject, interfaces.objects.ObjectInterface): # We're an instance, so also display the data - print(" " * (longest_offset - len_offset), hex(relative_offset), ": ", member, - " " * (longest_member - len_member), " ", - member_type.vol.type_name, " " * (longest_typename - len_typename), " ", - self._display_value(getattr(volobject, member))) + print( + " " * (longest_offset - len_offset), + hex(relative_offset), + ": ", + member, + " " * (longest_member - len_member), + " ", + member_type.vol.type_name, + " " * (longest_typename - len_typename), + " ", + self._display_value(getattr(volobject, member)), + ) else: - print(" " * (longest_offset - len_offset), hex(relative_offset), ": ", member, - " " * (longest_member - len_member), " ", member_type.vol.type_name) + print( + " " * (longest_offset - len_offset), + hex(relative_offset), + ": ", + member, + " " * (longest_member - len_member), + " ", + member_type.vol.type_name, + ) @classmethod def _display_value(cls, value: Any) -> str: @@ -332,8 +406,9 @@ class Volshell(interfaces.plugins.PluginInterface): else: return hex(value.vol.offset) - def generate_treegrid(self, plugin: Type[interfaces.plugins.PluginInterface], - **kwargs) -> Optional[interfaces.renderers.TreeGrid]: + def generate_treegrid( + self, plugin: Type[interfaces.plugins.PluginInterface], **kwargs + ) -> Optional[interfaces.renderers.TreeGrid]: """Generates a TreeGrid based on a specific plugin passing in kwarg configuration values""" path_join = interfaces.configuration.path_join @@ -346,21 +421,29 @@ class Volshell(interfaces.plugins.PluginInterface): self.config[path_join(plugin_config_suffix, plugin.__name__, name)] = value try: - constructed = plugins.construct_plugin(self.context, [], plugin, plugin_path, None, NullFileHandler) + constructed = plugins.construct_plugin( + self.context, [], plugin, plugin_path, None, NullFileHandler + ) return constructed.run() except exceptions.UnsatisfiedException as excp: - print(f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") + print( + f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n" + ) return None - def render_treegrid(self, - treegrid: interfaces.renderers.TreeGrid, - renderer: Optional[interfaces.renderers.Renderer] = None) -> None: + def render_treegrid( + self, + treegrid: interfaces.renderers.TreeGrid, + renderer: Optional[interfaces.renderers.Renderer] = None, + ) -> None: """Renders a treegrid as produced by generate_treegrid""" if renderer is None: renderer = text_renderer.QuickTextRenderer() renderer.render(treegrid) - def display_plugin_output(self, plugin: Type[interfaces.plugins.PluginInterface], **kwargs) -> None: + def display_plugin_output( + self, plugin: Type[interfaces.plugins.PluginInterface], **kwargs + ) -> None: """Displays the output for a particular plugin (with keyword arguments)""" treegrid = self.generate_treegrid(plugin, **kwargs) if treegrid is not None: @@ -382,7 +465,12 @@ class Volshell(interfaces.plugins.PluginInterface): for symbol_name in sorted(table.symbols): symbol = table.get_symbol(symbol_name) len_offset = len(hex(symbol.address)) - print(" " * (longest_offset - len_offset), hex(symbol.address), " ", symbol.name) + print( + " " * (longest_offset - len_offset), + hex(symbol.address), + " ", + symbol.name, + ) def run_script(self, location: str): """Runs a python script within the context of volshell""" @@ -390,32 +478,45 @@ class Volshell(interfaces.plugins.PluginInterface): location = "file:" + request.pathname2url(location) print(f"Running code from {location}\n") accessor = resources.ResourceAccessor() - with accessor.open(url = location) as fp: - self.__console.runsource(io.TextIOWrapper(fp.read(), encoding = 'utf-8'), symbol = 'exec') + with accessor.open(url=location) as fp: + self.__console.runsource( + io.TextIOWrapper(fp.read(), encoding="utf-8"), symbol="exec" + ) print("\nCode complete") def load_file(self, location: str): """Loads a file into a Filelayer and returns the name of the layer""" layer_name = self.context.layers.free_layer_name() location = volshell.VolShell.location_from_file(location) - current_config_path = 'volshell.layers.' + layer_name - self.context.config[interfaces.configuration.path_join(current_config_path, "location")] = location + current_config_path = "volshell.layers." + layer_name + self.context.config[ + interfaces.configuration.path_join(current_config_path, "location") + ] = location layer = physical.FileLayer(self.context, current_config_path, layer_name) self.context.add_layer(layer) return layer_name - def create_configurable(self, clazz: Type[interfaces.configuration.ConfigurableInterface], **kwargs): + def create_configurable( + self, clazz: Type[interfaces.configuration.ConfigurableInterface], **kwargs + ): """Creates a configurable object, converting arguments to configuration""" config_name = self.random_string() - config_path = 'volshell.configurable.' + config_name + config_path = "volshell.configurable." + config_name constructor_args = {} constructor_keywords = [] if issubclass(clazz, interfaces.layers.DataLayerInterface): - constructor_keywords = [('name', self.context.layers.free_layer_name(config_name)), ('metadata', None)] + constructor_keywords = [ + ("name", self.context.layers.free_layer_name(config_name)), + ("metadata", None), + ] if issubclass(clazz, interfaces.symbols.SymbolTableInterface): - constructor_keywords = [('name', self.context.symbol_space.free_table_name(config_name)), - ('native_types', None), ('table_mapping', None), ('class_types', None)] + constructor_keywords = [ + ("name", self.context.symbol_space.free_table_name(config_name)), + ("native_types", None), + ("table_mapping", None), + ("class_types", None), + ] for argname, default in constructor_keywords: constructor_args[argname] = kwargs.get(argname, default) @@ -424,10 +525,16 @@ class Volshell(interfaces.plugins.PluginInterface): for keyword in kwargs: val = kwargs[keyword] - if not isinstance(val, interfaces.configuration.BasicTypes) and not isinstance(val, list): - if not isinstance(val, list) or all([isinstance(x, interfaces.configuration.BasicTypes) for x in val]): - raise TypeError("Configurable values must be simple types (int, bool, str, bytes)") - self.context.config[config_path + '.' + keyword] = val + if not isinstance( + val, interfaces.configuration.BasicTypes + ) and not isinstance(val, list): + if not isinstance(val, list) or all( + [isinstance(x, interfaces.configuration.BasicTypes) for x in val] + ): + raise TypeError( + "Configurable values must be simple types (int, bool, str, bytes)" + ) + self.context.config[config_path + "." + keyword] = val constructed = clazz(self.context, config_path, **constructor_args) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 0f2a90c7e..8c23bbec3 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -15,13 +15,19 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return ([ - requirements.ModuleRequirement(name = "kernel", description = "Linux kernel module"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) - ]) + return [ + requirements.ModuleRequirement( + name="kernel", description="Linux kernel module" + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.IntRequirement( + name="pid", description="Process ID", optional=True + ), + ] - def change_task(self, pid = None): + def change_task(self, pid=None): """Change the current process and layer, based on a process ID""" tasks = self.list_tasks() for task in tasks: @@ -42,17 +48,21 @@ class Volshell(generic.Volshell): def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ - (['ct', 'change_task', 'cp'], self.change_task), - (['lt', 'list_tasks', 'ps'], self.list_tasks), - (['symbols'], self.context.symbol_space[self.current_symbol_table]), + (["ct", "change_task", "cp"], self.change_task), + (["lt", "list_tasks", "ps"], self.list_tasks), + (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] - if self.config.get('pid', None) is not None: - self.change_task(self.config['pid']) + if self.config.get("pid", None) is not None: + self.change_task(self.config["pid"]) return result - def display_type(self, - object: Union[str, interfaces.objects.ObjectInterface, interfaces.objects.Template], - offset: int = None): + def display_type( + self, + object: Union[ + str, interfaces.objects.ObjectInterface, interfaces.objects.Template + ], + offset: int = None, + ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 6744f3394..b709511b1 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -15,13 +15,19 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return ([ - requirements.ModuleRequirement(name = "kernel", description = "Darwin kernel module"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), - requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) - ]) + return [ + requirements.ModuleRequirement( + name="kernel", description="Darwin kernel module" + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), + requirements.IntRequirement( + name="pid", description="Process ID", optional=True + ), + ] - def change_task(self, pid = None): + def change_task(self, pid=None): """Change the current process and layer, based on a process ID""" tasks = self.list_tasks() for task in tasks: @@ -34,25 +40,31 @@ class Volshell(generic.Volshell): return print(f"No task with task ID {pid} found") - def list_tasks(self, method = None): + def list_tasks(self, method=None): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.get_list_tasks(method)(self.context, self.current_kernel_name)) + return list( + pslist.PsList.get_list_tasks(method)(self.context, self.current_kernel_name) + ) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ - (['ct', 'change_task', 'cp'], self.change_task), - (['lt', 'list_tasks', 'ps'], self.list_tasks), - (['symbols'], self.context.symbol_space[self.current_symbol_table]), + (["ct", "change_task", "cp"], self.change_task), + (["lt", "list_tasks", "ps"], self.list_tasks), + (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] - if self.config.get('pid', None) is not None: - self.change_task(self.config['pid']) + if self.config.get("pid", None) is not None: + self.change_task(self.config["pid"]) return result - def display_type(self, - object: Union[str, interfaces.objects.ObjectInterface, interfaces.objects.Template], - offset: int = None): + def display_type( + self, + object: Union[ + str, interfaces.objects.ObjectInterface, interfaces.objects.Template + ], + offset: int = None, + ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 2cc5d3e1d..652b2e66b 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -15,13 +15,17 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return ([ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel'), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) - ]) + return [ + requirements.ModuleRequirement(name="kernel", description="Windows kernel"), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.IntRequirement( + name="pid", description="Process ID", optional=True + ), + ] - def change_process(self, pid = None): + def change_process(self, pid=None): """Change the current process and layer, based on a process ID""" processes = self.list_processes() for process in processes: @@ -34,22 +38,30 @@ class Volshell(generic.Volshell): def list_processes(self): """Returns a list of EPROCESS objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_processes(self.context, self.current_layer, self.current_symbol_table)) + return list( + pslist.PsList.list_processes( + self.context, self.current_layer, self.current_symbol_table + ) + ) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ - (['cp', 'change_process'], self.change_process), - (['lp', 'list_processes', 'ps'], self.list_processes), - (['symbols'], self.context.symbol_space[self.current_symbol_table]), + (["cp", "change_process"], self.change_process), + (["lp", "list_processes", "ps"], self.list_processes), + (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] - if self.config.get('pid', None) is not None: - self.change_process(self.config['pid']) + if self.config.get("pid", None) is not None: + self.change_process(self.config["pid"]) return result - def display_type(self, - object: Union[str, interfaces.objects.ObjectInterface, interfaces.objects.Template], - offset: int = None): + def display_type( + self, + object: Union[ + str, interfaces.objects.ObjectInterface, interfaces.objects.Template + ], + offset: int = None, + ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 176eb2242..949aecce9 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -8,10 +8,19 @@ import sys import zipfile required_python_version = (3, 6, 0) -if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or - (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): +if ( + sys.version_info.major != required_python_version[0] + or sys.version_info.minor < required_python_version[1] + or ( + sys.version_info.minor == required_python_version[1] + and sys.version_info.micro < required_python_version[2] + ) +): raise RuntimeError( - "Volatility framework requires python version {}.{}.{} or greater".format(*required_python_version)) + "Volatility framework requires python version {}.{}.{} or greater".format( + *required_python_version + ) + ) import importlib import inspect @@ -45,24 +54,29 @@ def require_interface_version(*args) -> None: """Checks the required version of a plugin.""" if len(args): if args[0] != interface_version()[0]: - raise RuntimeError("Framework interface version {} is incompatible with required version {}".format( - interface_version()[0], args[0])) + raise RuntimeError( + "Framework interface version {} is incompatible with required version {}".format( + interface_version()[0], args[0] + ) + ) if len(args) > 1: if args[1] > interface_version()[1]: raise RuntimeError( "Framework interface version {} is an older revision than the required version {}".format( - ".".join([str(x) for x in interface_version()[0:2]]), ".".join([str(x) for x in args[0:2]]))) + ".".join([str(x) for x in interface_version()[0:2]]), + ".".join([str(x) for x in args[0:2]]), + ) + ) class NonInheritable(object): - def __init__(self, value: Any, cls: Type) -> None: self.default_value = value self.cls = cls def __get__(self, obj: Any, get_type: Type = None) -> Any: if type == self.cls: - if hasattr(self.default_value, '__get__'): + if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) return self.default_value raise AttributeError @@ -73,7 +87,7 @@ def hide_from_subclasses(cls: Type) -> Type: return cls -T = TypeVar('T') +T = TypeVar("T") def class_subclasses(cls: Type[T]) -> Generator[Type[T], None, None]: @@ -82,7 +96,7 @@ def class_subclasses(cls: Type[T]) -> Generator[Type[T], None, None]: raise TypeError(f"class_subclasses parameter not a valid class: {cls}") for clazz in cls.__subclasses__(): # The typing system is not clever enough to realize that clazz has a hidden attr after the hasattr check - if not hasattr(clazz, 'hidden') or not clazz.hidden: # type: ignore + if not hasattr(clazz, "hidden") or not clazz.hidden: # type: ignore yield clazz for return_value in class_subclasses(clazz): yield return_value @@ -93,10 +107,12 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]: failures = [] if not isinstance(base_module.__path__, list): raise TypeError("[base_module].__path__ must be a list of paths") - vollog.log(constants.LOGLEVEL_VVVV, - f"Importing from the following paths: {', '.join(base_module.__path__)}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Importing from the following paths: {', '.join(base_module.__path__)}", + ) for path in base_module.__path__: - for root, _, files in os.walk(path, followlinks = True): + for root, _, files in os.walk(path, followlinks=True): # TODO: Figure out how to import pycache files if root.endswith("__pycache__"): continue @@ -104,35 +120,51 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]: if zipfile.is_zipfile(os.path.join(root, filename)): # Use the root to add this to the module path, and sub-traverse the files new_module = base_module - premodules = root[len(path) + len(os.path.sep):].replace(os.path.sep, '.') - for component in premodules.split('.'): + premodules = root[len(path) + len(os.path.sep) :].replace( + os.path.sep, "." + ) + for component in premodules.split("."): if component: try: new_module = getattr(new_module, component) except AttributeError: - failures += [new_module + '.' + component] - new_module.__path__ = [os.path.join(root, filename)] + new_module.__path__ + failures += [new_module + "." + component] + new_module.__path__ = [ + os.path.join(root, filename) + ] + new_module.__path__ for ziproot, zipfiles in _zipwalk(os.path.join(root, filename)): for zfile in zipfiles: if _filter_files(zfile): - submodule = zfile[:zfile.rfind('.')].replace(os.path.sep, '.') - failures += import_file(new_module.__name__ + '.' + submodule, - os.path.join(path, ziproot, zfile)) + submodule = zfile[: zfile.rfind(".")].replace( + os.path.sep, "." + ) + failures += import_file( + new_module.__name__ + "." + submodule, + os.path.join(path, ziproot, zfile), + ) else: if _filter_files(filename): - modpath = os.path.join(root[len(path) + len(os.path.sep):], filename[:filename.rfind(".")]) + modpath = os.path.join( + root[len(path) + len(os.path.sep) :], + filename[: filename.rfind(".")], + ) submodule = modpath.replace(os.path.sep, ".") - failures += import_file(base_module.__name__ + '.' + submodule, - os.path.join(root, filename), - ignore_errors) + failures += import_file( + base_module.__name__ + "." + submodule, + os.path.join(root, filename), + ignore_errors, + ) return failures def _filter_files(filename: str): """Ensures that a filename traversed is an importable python file""" - return (filename.endswith(".py") or filename.endswith(".pyc") or filename.endswith( - ".pyo")) and not filename.startswith("__") + return ( + filename.endswith(".py") + or filename.endswith(".pyc") + or filename.endswith(".pyo") + ) and not filename.startswith("__") def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str]: @@ -152,7 +184,9 @@ def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str importlib.import_module(module) except ImportError as e: vollog.debug(str(e)) - vollog.debug("Failed to import module {} based on file: {}".format(module, path)) + vollog.debug( + "Failed to import module {} based on file: {}".format(module, path) + ) failures.append(module) if not ignore_errors: raise @@ -167,7 +201,9 @@ def _zipwalk(path: str): if not file.is_dir(): dirlist = zip_results.get(os.path.dirname(file.filename), []) dirlist.append(os.path.basename(file.filename)) - zip_results[os.path.join(path, os.path.dirname(file.filename))] = dirlist + zip_results[ + os.path.join(path, os.path.dirname(file.filename)) + ] = dirlist for value in zip_results: yield value, zip_results[value] @@ -177,14 +213,14 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: for plugin in class_subclasses(interfaces.plugins.PluginInterface): plugin_name = plugin.__module__ + "." + plugin.__name__ if plugin_name.startswith("volatility3.plugins."): - plugin_name = plugin_name[len("volatility3.plugins."):] + plugin_name = plugin_name[len("volatility3.plugins.") :] plugin_list[plugin_name] = plugin return plugin_list -def clear_cache(complete = False): - glob_pattern = '*.cache' +def clear_cache(complete=False): + glob_pattern = "*.cache" if not complete: - glob_pattern = 'data_' + glob_pattern + glob_pattern = "data_" + glob_pattern for cache_filename in glob.glob(os.path.join(constants.CACHE_PATH, glob_pattern)): os.unlink(cache_filename) diff --git a/volatility3/framework/automagic/__init__.py b/volatility3/framework/automagic/__init__.py index 7567f206d..42c728b59 100644 --- a/volatility3/framework/automagic/__init__.py +++ b/volatility3/framework/automagic/__init__.py @@ -22,7 +22,9 @@ from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) -def available(context: interfaces.context.ContextInterface) -> List[interfaces.automagic.AutomagicInterface]: +def available( + context: interfaces.context.ContextInterface, +) -> List[interfaces.automagic.AutomagicInterface]: """Returns an ordered list of all subclasses of :class:`~volatility3.framework.interfaces.automagic.AutomagicInterface`. @@ -34,21 +36,26 @@ def available(context: interfaces.context.ContextInterface) -> List[interfaces.a """ import_files(sys.modules[__name__]) config_path = constants.AUTOMAGIC_CONFIG_PATH - return sorted([ - clazz(context, interfaces.configuration.path_join(config_path, clazz.__name__)) - for clazz in class_subclasses(interfaces.automagic.AutomagicInterface) - ], - key = lambda x: x.priority) + return sorted( + [ + clazz( + context, interfaces.configuration.path_join(config_path, clazz.__name__) + ) + for clazz in class_subclasses(interfaces.automagic.AutomagicInterface) + ], + key=lambda x: x.priority, + ) def choose_automagic( - automagics: List[Type[interfaces.automagic.AutomagicInterface]], - plugin: Type[interfaces.plugins.PluginInterface]) -> List[Type[interfaces.automagic.AutomagicInterface]]: + automagics: List[Type[interfaces.automagic.AutomagicInterface]], + plugin: Type[interfaces.plugins.PluginInterface], +) -> List[Type[interfaces.automagic.AutomagicInterface]]: """Chooses which automagics to run, maintaining the order they were handed in.""" plugin_category = "None" - plugin_categories = plugin.__module__.split('.') + plugin_categories = plugin.__module__.split(".") lowest_index = len(plugin_categories) for os in constants.OS_CATEGORIES: try: @@ -73,12 +80,16 @@ def choose_automagic( return output -def run(automagics: List[interfaces.automagic.AutomagicInterface], - context: interfaces.context.ContextInterface, - configurable: Union[interfaces.configuration.ConfigurableInterface, - Type[interfaces.configuration.ConfigurableInterface]], - config_path: str, - progress_callback: constants.ProgressCallback = None) -> List[traceback.TracebackException]: +def run( + automagics: List[interfaces.automagic.AutomagicInterface], + context: interfaces.context.ContextInterface, + configurable: Union[ + interfaces.configuration.ConfigurableInterface, + Type[interfaces.configuration.ConfigurableInterface], + ], + config_path: str, + progress_callback: constants.ProgressCallback = None, +) -> List[traceback.TracebackException]: """Runs through the list of `automagics` in order, allowing them to make changes to the context. @@ -99,10 +110,13 @@ def run(automagics: List[interfaces.automagic.AutomagicInterface], """ for automagic in automagics: if not isinstance(automagic, interfaces.automagic.AutomagicInterface): - raise TypeError("Automagics must only contain AutomagicInterface subclasses") + raise TypeError( + "Automagics must only contain AutomagicInterface subclasses" + ) - if (not isinstance(configurable, interfaces.configuration.ConfigurableInterface) - and not issubclass(configurable, interfaces.configuration.ConfigurableInterface)): + if not isinstance( + configurable, interfaces.configuration.ConfigurableInterface + ) and not issubclass(configurable, interfaces.configuration.ConfigurableInterface): raise TypeError("Automagic operates on configurables only") # TODO: Fix need for top level config element just because we're using a MultiRequirement to group the @@ -112,7 +126,7 @@ def run(automagics: List[interfaces.automagic.AutomagicInterface], configurable_class = configurable.__class__ else: configurable_class = configurable - requirement = requirements.MultiRequirement(name = configurable_class.__name__) + requirement = requirements.MultiRequirement(name=configurable_class.__name__) for req in configurable.get_requirements(): requirement.add_requirement(req) diff --git a/volatility3/framework/automagic/construct_layers.py b/volatility3/framework/automagic/construct_layers.py index 40a17419f..ceed2fe50 100644 --- a/volatility3/framework/automagic/construct_layers.py +++ b/volatility3/framework/automagic/construct_layers.py @@ -25,39 +25,60 @@ class ConstructionMagic(interfaces.automagic.AutomagicInterface): :warning: This `automagic` should run first to allow existing configurations to have been constructed for use by later automagic """ + priority = 0 - def __call__(self, - context: interfaces.context.ContextInterface, - config_path: str, - requirement: interfaces.configuration.RequirementInterface, - progress_callback = None, - optional = False) -> List[str]: + def __call__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.RequirementInterface, + progress_callback=None, + optional=False, + ) -> List[str]: # Make sure we import the layers, so they can reconstructed - framework.import_files(sys.modules['volatility3.framework.layers']) + framework.import_files(sys.modules["volatility3.framework.layers"]) result: List[str] = [] if requirement.unsatisfied(context, config_path): # Having called validate at the top level tells us both that we need to dig deeper # but also ensures that TranslationLayerRequirements have got the correct subrequirements if their class is populated - subreq_config_path = interfaces.configuration.path_join(config_path, requirement.name) + subreq_config_path = interfaces.configuration.path_join( + config_path, requirement.name + ) for subreq in requirement.requirements.values(): try: - self(context, subreq_config_path, subreq, optional = optional or subreq.optional) + self( + context, + subreq_config_path, + subreq, + optional=optional or subreq.optional, + ) except Exception as e: # We don't really care if this fails, it tends to mean the configuration isn't complete for that item - vollog.log(constants.LOGLEVEL_VVVV, f"Construction Exception occurred: {e}") + vollog.log( + constants.LOGLEVEL_VVVV, f"Construction Exception occurred: {e}" + ) invalid = subreq.unsatisfied(context, subreq_config_path) # We want to traverse optional paths, so don't check until we've tried to validate # We also don't want to emit a debug message when a parent is optional, hence the optional parameter if invalid and not (optional or subreq.optional): - vollog.log(constants.LOGLEVEL_V, f"Failed on requirement: {subreq_config_path}") - result.append(interfaces.configuration.path_join(subreq_config_path, subreq.name)) + vollog.log( + constants.LOGLEVEL_V, + f"Failed on requirement: {subreq_config_path}", + ) + result.append( + interfaces.configuration.path_join( + subreq_config_path, subreq.name + ) + ) if result: return result - elif isinstance(requirement, interfaces.configuration.ConstructableRequirementInterface): + elif isinstance( + requirement, interfaces.configuration.ConstructableRequirementInterface + ): # We know all the subrequirements are filled, so let's populate requirement.construct(context, config_path) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 9bb2dae9b..2eebcc2dc 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -17,19 +17,24 @@ vollog = logging.getLogger(__name__) class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): stack_order = 35 - exclusion_list = ['mac', 'windows'] + exclusion_list = ["mac", "windows"] @classmethod - def stack(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify linux within this layer.""" # Version check the SQlite cache required = (1, 0, 0) - if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + if not requirements.VersionRequirement.matches_required( + required, symbol_cache.SqliteCache.version + ): vollog.info( - f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}" + ) return None # Bail out by default unless we can stack properly @@ -41,56 +46,70 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) - linux_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( - operating_system = 'linux') + identifiers_path = os.path.join( + constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + ) + linux_banners = symbol_cache.SqliteCache( + identifiers_path + ).get_identifier_dictionary(operating_system="linux") # If we have no banners, don't bother scanning if not linux_banners: - vollog.info("No Linux banners found - if this is a linux plugin, please check your symbol files location") + vollog.info( + "No Linux banners found - if this is a linux plugin, please check your symbol files location" + ) return None mss = scanners.MultiStringScanner([x for x in linux_banners if x is not None]) - for _, banner in layer.scan(context = context, scanner = mss, progress_callback = progress_callback): + for _, banner in layer.scan( + context=context, scanner=mss, progress_callback=progress_callback + ): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") isf_path = linux_banners.get(banner, None) if isf_path: - table_name = context.symbol_space.free_table_name('LintelStacker') - table = linux.LinuxKernelIntermedSymbols(context, - 'temporary.' + table_name, - name = table_name, - isf_url = isf_path) + table_name = context.symbol_space.free_table_name("LintelStacker") + table = linux.LinuxKernelIntermedSymbols( + context, + "temporary." + table_name, + name=table_name, + isf_url=isf_path, + ) context.symbol_space.append(table) - kaslr_shift, aslr_shift = cls.find_aslr(context, - table_name, - layer_name, - progress_callback = progress_callback) + kaslr_shift, aslr_shift = cls.find_aslr( + context, table_name, layer_name, progress_callback=progress_callback + ) layer_class: Type = intel.Intel - if 'init_top_pgt' in table.symbols: + if "init_top_pgt" in table.symbols: layer_class = intel.Intel32e - dtb_symbol_name = 'init_top_pgt' - elif 'init_level4_pgt' in table.symbols: + dtb_symbol_name = "init_top_pgt" + elif "init_level4_pgt" in table.symbols: layer_class = intel.Intel32e - dtb_symbol_name = 'init_level4_pgt' + dtb_symbol_name = "init_level4_pgt" else: - dtb_symbol_name = 'swapper_pg_dir' + dtb_symbol_name = "swapper_pg_dir" - dtb = cls.virtual_to_physical_address(table.get_symbol(dtb_symbol_name).address + kaslr_shift) + dtb = cls.virtual_to_physical_address( + table.get_symbol(dtb_symbol_name).address + kaslr_shift + ) # Build the new layer new_layer_name = context.layers.free_layer_name("IntelLayer") config_path = join("IntelHelper", new_layer_name) context.config[join(config_path, "memory_layer")] = layer_name context.config[join(config_path, "page_map_offset")] = dtb - context.config[join(config_path, LinuxSymbolFinder.banner_config_key)] = str(banner, 'latin-1') + context.config[ + join(config_path, LinuxSymbolFinder.banner_config_key) + ] = str(banner, "latin-1") - layer = layer_class(context, - config_path = config_path, - name = new_layer_name, - metadata = {'os': 'Linux'}) - layer.config['kernel_virtual_offset'] = aslr_shift + layer = layer_class( + context, + config_path=config_path, + name=new_layer_name, + metadata={"os": "Linux"}, + ) + layer.config["kernel_virtual_offset"] = aslr_shift if layer and dtb: vollog.debug(f"DTB was found at: 0x{dtb:0x}") @@ -99,43 +118,63 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): return None @classmethod - def find_aslr(cls, - context: interfaces.context.ContextInterface, - symbol_table: str, - layer_name: str, - progress_callback: constants.ProgressCallback = None) \ - -> Tuple[int, int]: + def find_aslr( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Tuple[int, int]: """Determines the offset of the actual DTB in physical space and its symbol offset.""" - init_task_symbol = symbol_table + constants.BANG + 'init_task' - init_task_json_address = context.symbol_space.get_symbol(init_task_symbol).address + init_task_symbol = symbol_table + constants.BANG + "init_task" + init_task_json_address = context.symbol_space.get_symbol( + init_task_symbol + ).address swapper_signature = rb"swapper(\/0|\x00\x00)\x00\x00\x00\x00\x00\x00" module = context.module(symbol_table, layer_name, 0) - address_mask = context.symbol_space[symbol_table].config.get('symbol_mask', None) + address_mask = context.symbol_space[symbol_table].config.get( + "symbol_mask", None + ) - task_symbol = module.get_type('task_struct') - comm_child_offset = task_symbol.relative_child_offset('comm') + task_symbol = module.get_type("task_struct") + comm_child_offset = task_symbol.relative_child_offset("comm") - for offset in context.layers[layer_name].scan(scanner = scanners.RegExScanner(swapper_signature), - context = context, - progress_callback = progress_callback): + for offset in context.layers[layer_name].scan( + scanner=scanners.RegExScanner(swapper_signature), + context=context, + progress_callback=progress_callback, + ): init_task_address = offset - comm_child_offset - init_task = module.object(object_type = 'task_struct', offset = init_task_address, absolute = True) + init_task = module.object( + object_type="task_struct", offset=init_task_address, absolute=True + ) if init_task.pid != 0: continue - elif init_task.has_member('state') and init_task.state.cast('unsigned int') != 0: + elif ( + init_task.has_member("state") + and init_task.state.cast("unsigned int") != 0 + ): continue # This we get for free - aslr_shift = init_task.files.cast('long unsigned int') - module.get_symbol('init_files').address - kaslr_shift = init_task_address - cls.virtual_to_physical_address(init_task_json_address) + aslr_shift = ( + init_task.files.cast("long unsigned int") + - module.get_symbol("init_files").address + ) + kaslr_shift = init_task_address - cls.virtual_to_physical_address( + init_task_json_address + ) if address_mask: aslr_shift = aslr_shift & address_mask - if aslr_shift & 0xfff != 0 or kaslr_shift & 0xfff != 0: + if aslr_shift & 0xFFF != 0 or kaslr_shift & 0xFFF != 0: continue - vollog.debug("Linux ASLR shift values determined: physical {:0x} virtual {:0x}".format( - kaslr_shift, aslr_shift)) + vollog.debug( + "Linux ASLR shift values determined: physical {:0x} virtual {:0x}".format( + kaslr_shift, aslr_shift + ) + ) return kaslr_shift, aslr_shift # We don't throw an exception, because we may legitimately not have an ASLR shift, but we report it @@ -146,16 +185,16 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): def virtual_to_physical_address(cls, addr: int) -> int: """Converts a virtual linux address to a physical one (does not account of ASLR)""" - if addr > 0xffffffff80000000: - return addr - 0xffffffff80000000 - return addr - 0xc0000000 + if addr > 0xFFFFFFFF80000000: + return addr - 0xFFFFFFFF80000000 + return addr - 0xC0000000 class LinuxSymbolFinder(symbol_finder.SymbolFinder): """Linux symbol loader based on uname signature strings.""" banner_config_key = "kernel_banner" - operating_system = 'linux' + operating_system = "linux" symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] - exclusion_list = ['mac', 'windows'] + exclusion_list = ["mac", "windows"] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 9bb3ad5f0..3ca0b4ea2 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -18,19 +18,24 @@ vollog = logging.getLogger(__name__) class MacIntelStacker(interfaces.automagic.StackerLayerInterface): stack_order = 35 - exclusion_list = ['windows', 'linux'] + exclusion_list = ["windows", "linux"] @classmethod - def stack(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify mac within this layer.""" # Version check the SQlite cache required = (1, 0, 0) - if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + if not requirements.VersionRequirement.matches_required( + required, symbol_cache.SqliteCache.version + ): vollog.info( - f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}" + ) return None # Bail out by default unless we can stack properly @@ -43,56 +48,76 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) - mac_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( - operating_system = 'mac') + identifiers_path = os.path.join( + constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + ) + mac_banners = symbol_cache.SqliteCache( + identifiers_path + ).get_identifier_dictionary(operating_system="mac") # If we have no banners, don't bother scanning if not mac_banners: - vollog.info("No Mac banners found - if this is a mac plugin, please check your symbol files location") + vollog.info( + "No Mac banners found - if this is a mac plugin, please check your symbol files location" + ) return None mss = scanners.MultiStringScanner([x for x in mac_banners if x]) - for banner_offset, banner in layer.scan(context = context, scanner = mss, - progress_callback = progress_callback): + for banner_offset, banner in layer.scan( + context=context, scanner=mss, progress_callback=progress_callback + ): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") isf_path = mac_banners.get(banner, None) if isf_path: - table_name = context.symbol_space.free_table_name('MacintelStacker') - table = mac.MacKernelIntermedSymbols(context = context, - config_path = join('temporary', table_name), - name = table_name, - isf_url = isf_path) + table_name = context.symbol_space.free_table_name("MacintelStacker") + table = mac.MacKernelIntermedSymbols( + context=context, + config_path=join("temporary", table_name), + name=table_name, + isf_url=isf_path, + ) context.symbol_space.append(table) - kaslr_shift = cls.find_aslr(context = context, - symbol_table = table_name, - layer_name = layer_name, - compare_banner = banner, - compare_banner_offset = banner_offset, - progress_callback = progress_callback) + kaslr_shift = cls.find_aslr( + context=context, + symbol_table=table_name, + layer_name=layer_name, + compare_banner=banner, + compare_banner_offset=banner_offset, + progress_callback=progress_callback, + ) if kaslr_shift == 0: - vollog.log(constants.LOGLEVEL_VVV, f"Invalid kalsr_shift found at offset: {banner_offset}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Invalid kalsr_shift found at offset: {banner_offset}", + ) continue - bootpml4_addr = cls.virtual_to_physical_address(table.get_symbol("BootPML4").address + kaslr_shift) + bootpml4_addr = cls.virtual_to_physical_address( + table.get_symbol("BootPML4").address + kaslr_shift + ) new_layer_name = context.layers.free_layer_name("MacDTBTempLayer") config_path = join("automagic", "MacIntelHelper", new_layer_name) context.config[join(config_path, "memory_layer")] = layer_name context.config[join(config_path, "page_map_offset")] = bootpml4_addr - layer = layers.intel.Intel32e(context, - config_path = config_path, - name = new_layer_name, - metadata = {'os': 'Mac'}) + layer = layers.intel.Intel32e( + context, + config_path=config_path, + name=new_layer_name, + metadata={"os": "Mac"}, + ) idlepml4_ptr = table.get_symbol("IdlePML4").address + kaslr_shift try: idlepml4_str = layer.read(idlepml4_ptr, 4) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVVV, f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}", + ) continue idlepml4_addr = struct.unpack(" int: + def find_aslr( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + layer_name: str, + compare_banner: str = "", + compare_banner_offset: int = 0, + progress_callback: constants.ProgressCallback = None, + ) -> int: """Determines the offset of the actual DTB in physical space and its symbol offset.""" - version_symbol = symbol_table + constants.BANG + 'version' + version_symbol = symbol_table + constants.BANG + "version" version_json_address = context.symbol_space.get_symbol(version_symbol).address - version_major_symbol = symbol_table + constants.BANG + 'version_major' - version_major_json_address = context.symbol_space.get_symbol(version_major_symbol).address - version_major_phys_offset = cls.virtual_to_physical_address(version_major_json_address) + version_major_symbol = symbol_table + constants.BANG + "version_major" + version_major_json_address = context.symbol_space.get_symbol( + version_major_symbol + ).address + version_major_phys_offset = cls.virtual_to_physical_address( + version_major_json_address + ) - version_minor_symbol = symbol_table + constants.BANG + 'version_minor' - version_minor_json_address = context.symbol_space.get_symbol(version_minor_symbol).address - version_minor_phys_offset = cls.virtual_to_physical_address(version_minor_json_address) + version_minor_symbol = symbol_table + constants.BANG + "version_minor" + version_minor_json_address = context.symbol_space.get_symbol( + version_minor_symbol + ).address + version_minor_phys_offset = cls.virtual_to_physical_address( + version_minor_json_address + ) if not compare_banner_offset or not compare_banner: - offset_generator = cls._scan_generator(context, layer_name, progress_callback) + offset_generator = cls._scan_generator( + context, layer_name, progress_callback + ) else: offset_generator = [(compare_banner_offset, compare_banner)] @@ -155,24 +199,30 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): for offset, banner in offset_generator: banner_major, banner_minor = [int(x) for x in banner[22:].split(b".")[0:2]] - tmp_aslr_shift = offset - cls.virtual_to_physical_address(version_json_address) + tmp_aslr_shift = offset - cls.virtual_to_physical_address( + version_json_address + ) - major_string = context.layers[layer_name].read(version_major_phys_offset + tmp_aslr_shift, 4) + major_string = context.layers[layer_name].read( + version_major_phys_offset + tmp_aslr_shift, 4 + ) major = struct.unpack(" int: """Converts a virtual mac address to a physical one (does not account of ASLR)""" - if addr > 0xffffff8000000000: - addr = addr - 0xffffff8000000000 + if addr > 0xFFFFFF8000000000: + addr = addr - 0xFFFFFF8000000000 else: - addr = addr - 0xff8000000000 + addr = addr - 0xFF8000000000 return addr @classmethod def _scan_generator(cls, context, layer_name, progress_callback): - darwin_signature = rb"Darwin Kernel Version \d{1,3}\.\d{1,3}\.\d{1,3}: [^\x00]+\x00" + darwin_signature = ( + rb"Darwin Kernel Version \d{1,3}\.\d{1,3}\.\d{1,3}: [^\x00]+\x00" + ) - for offset in context.layers[layer_name].scan(scanner = scanners.RegExScanner(darwin_signature), - context = context, - progress_callback = progress_callback): + for offset in context.layers[layer_name].scan( + scanner=scanners.RegExScanner(darwin_signature), + context=context, + progress_callback=progress_callback, + ): banner = context.layers[layer_name].read(offset, 128) @@ -210,8 +264,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): class MacSymbolFinder(symbol_finder.SymbolFinder): """Mac symbol loader based on uname signature strings.""" - banner_config_key = 'kernel_banner' - operating_system = 'mac' + banner_config_key = "kernel_banner" + operating_system = "mac" find_aslr = MacIntelStacker.find_aslr symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols" - exclusion_list = ['windows', 'linux'] + exclusion_list = ["windows", "linux"] diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index 6810a58e2..2bdaf3f62 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -10,36 +10,55 @@ class KernelModule(interfaces.automagic.AutomagicInterface): priority = 100 - def __call__(self, - context: interfaces.context.ContextInterface, - config_path: str, - requirement: interfaces.configuration.RequirementInterface, - progress_callback: constants.ProgressCallback = None) -> None: - new_config_path = interfaces.configuration.path_join(config_path, requirement.name) + def __call__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.RequirementInterface, + progress_callback: constants.ProgressCallback = None, + ) -> None: + new_config_path = interfaces.configuration.path_join( + config_path, requirement.name + ) if not isinstance(requirement, configuration.requirements.ModuleRequirement): # Check subrequirements for req in requirement.requirements: - self(context, new_config_path, requirement.requirements[req], progress_callback) + self( + context, + new_config_path, + requirement.requirements[req], + progress_callback, + ) return if not requirement.unsatisfied(context, config_path): return # The requirement is unfulfilled and is a ModuleRequirement - context.config[interfaces.configuration.path_join( - new_config_path, 'class')] = 'volatility3.framework.contexts.Module' + context.config[ + interfaces.configuration.path_join(new_config_path, "class") + ] = "volatility3.framework.contexts.Module" for req in requirement.requirements: - if requirement.requirements[req].unsatisfied(context, new_config_path) and req != 'offset': + if ( + requirement.requirements[req].unsatisfied(context, new_config_path) + and req != "offset" + ): return # We now just have the offset requirement, but the layer requirement has been fulfilled. # Unfortunately we don't know the layer name requirement's exact name for req in requirement.requirements: - if isinstance(requirement.requirements[req], configuration.requirements.TranslationLayerRequirement): - layer_kvo_config_path = interfaces.configuration.path_join(new_config_path, req, - 'kernel_virtual_offset') - offset_config_path = interfaces.configuration.path_join(new_config_path, 'offset') + if isinstance( + requirement.requirements[req], + configuration.requirements.TranslationLayerRequirement, + ): + layer_kvo_config_path = interfaces.configuration.path_join( + new_config_path, req, "kernel_virtual_offset" + ) + offset_config_path = interfaces.configuration.path_join( + new_config_path, "offset" + ) offset = context.config[layer_kvo_config_path] context.config[offset_config_path] = offset diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 36288ef90..06b2111b4 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -22,7 +22,9 @@ from volatility3.framework.symbols.windows.pdbutil import PDBUtility if __name__ == "__main__": import sys - sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) + sys.path.append( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + ) vollog = logging.getLogger(__name__) @@ -43,12 +45,17 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): searches for a particular structure that lists the kernel module's virtual address, its size (not checked) and the module's name. This value is then used if one was not found using the previous method. """ + priority = 30 max_pdb_size = 0x400000 - exclusion_list = ['linux', 'mac'] + exclusion_list = ["linux", "mac"] - def find_virtual_layers_from_req(self, context: interfaces.context.ContextInterface, config_path: str, - requirement: interfaces.configuration.RequirementInterface) -> List[str]: + def find_virtual_layers_from_req( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.RequirementInterface, + ) -> List[str]: """Traverses the requirement tree, rooted at `requirement` looking for virtual layers that might contain a windows PDB. @@ -62,27 +69,36 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): Returns: A list of (layer_name, scan_results) """ - sub_config_path = interfaces.configuration.path_join(config_path, requirement.name) + sub_config_path = interfaces.configuration.path_join( + config_path, requirement.name + ) results: List[str] = [] if isinstance(requirement, requirements.TranslationLayerRequirement): # Check for symbols in this layer # FIXME: optionally allow a full (slow) scan # FIXME: Determine the physical layer no matter the virtual layer virtual_layer_name = context.config.get(sub_config_path, None) - layer_name = context.config.get(interfaces.configuration.path_join(sub_config_path, "memory_layer"), None) + layer_name = context.config.get( + interfaces.configuration.path_join(sub_config_path, "memory_layer"), + None, + ) if layer_name and virtual_layer_name: memlayer = context.layers[virtual_layer_name] if isinstance(memlayer, intel.Intel): results = [virtual_layer_name] else: for subreq in requirement.requirements.values(): - results += self.find_virtual_layers_from_req(context, sub_config_path, subreq) + results += self.find_virtual_layers_from_req( + context, sub_config_path, subreq + ) return results - def recurse_symbol_fulfiller(self, - context: interfaces.context.ContextInterface, - valid_kernel: ValidKernelType, - progress_callback: constants.ProgressCallback = None) -> None: + def recurse_symbol_fulfiller( + self, + context: interfaces.context.ContextInterface, + valid_kernel: ValidKernelType, + progress_callback: constants.ProgressCallback = None, + ) -> None: """Fulfills the SymbolTableRequirements in `self._symbol_requirements` found by the `recurse_symbol_requirements`. @@ -99,22 +115,28 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): if valid_kernel: # TODO: Check that the symbols for this kernel will fulfill the requirement virtual_layer, _kvo, kernel = valid_kernel - if not isinstance(kernel['pdb_name'], str) or not isinstance(kernel['GUID'], str): + if not isinstance(kernel["pdb_name"], str) or not isinstance( + kernel["GUID"], str + ): raise TypeError("PDB name or GUID not a string value") PDBUtility.load_windows_symbol_table( - context = context, - guid = kernel['GUID'], - age = kernel['age'], - pdb_name = kernel['pdb_name'], - symbol_table_class = "volatility3.framework.symbols.windows.WindowsKernelIntermedSymbols", - config_path = sub_config_path, - progress_callback = progress_callback) + context=context, + guid=kernel["GUID"], + age=kernel["age"], + pdb_name=kernel["pdb_name"], + symbol_table_class="volatility3.framework.symbols.windows.WindowsKernelIntermedSymbols", + config_path=sub_config_path, + progress_callback=progress_callback, + ) else: vollog.debug("No suitable kernel pdb signature found") - def set_kernel_virtual_offset(self, context: interfaces.context.ContextInterface, - valid_kernel: ValidKernelType) -> None: + def set_kernel_virtual_offset( + self, + context: interfaces.context.ContextInterface, + valid_kernel: ValidKernelType, + ) -> None: """Traverses the requirement tree, looking for kernel_virtual_offset values that may need setting and sets it based on the previously identified `valid_kernel`. @@ -127,72 +149,98 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): # Set the virtual offset under the TranslationLayer it applies to virtual_layer, kvo, kernel = valid_kernel if kvo is not None: - kvo_path = interfaces.configuration.path_join(context.layers[virtual_layer].config_path, - 'kernel_virtual_offset') + kvo_path = interfaces.configuration.path_join( + context.layers[virtual_layer].config_path, "kernel_virtual_offset" + ) context.config[kvo_path] = kvo vollog.debug(f"Setting kernel_virtual_offset to {hex(kvo)}") def get_physical_layer_name(self, context, vlayer): - return context.config.get(interfaces.configuration.path_join(vlayer.config_path, 'memory_layer'), None) + return context.config.get( + interfaces.configuration.path_join(vlayer.config_path, "memory_layer"), None + ) - def method_slow_scan(self, - context: interfaces.context.ContextInterface, - vlayer: layers.intel.Intel, - progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - - def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ - ValidKernelType]: + def method_slow_scan( + self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[ValidKernelType]: + def test_virtual_kernel( + physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any] + ) -> Optional[ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) - if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): + if kernel["mz_offset"] is None or not isinstance(kernel["mz_offset"], int): # Rule out kernels that couldn't find a suitable MZ header return None - return (virtual_layer_name, kernel['mz_offset'], kernel) + return (virtual_layer_name, kernel["mz_offset"], kernel) vollog.debug("Kernel base determination - optimized scan virtual layer") - valid_kernel = self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, True, False, progress_callback) + valid_kernel = self._method_layer_pdb_scan( + context, vlayer, test_virtual_kernel, True, False, progress_callback + ) if valid_kernel is not None: return valid_kernel vollog.debug("Kernel base determination - slow scan virtual layer") - return self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, False, False, progress_callback) + return self._method_layer_pdb_scan( + context, vlayer, test_virtual_kernel, False, False, progress_callback + ) - def method_fixed_mapping(self, - context: interfaces.context.ContextInterface, - vlayer: layers.intel.Intel, - progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - - def test_physical_kernel(physical_layer_name: str, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ - ValidKernelType]: + def method_fixed_mapping( + self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[ValidKernelType]: + def test_physical_kernel( + physical_layer_name: str, virtual_layer_name: str, kernel: Dict[str, Any] + ) -> Optional[ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) - if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): + if kernel["mz_offset"] is None or not isinstance(kernel["mz_offset"], int): # Rule out kernels that couldn't find a suitable MZ header return None if vlayer.bits_per_register == 64: - kvo = kernel['mz_offset'] + (31 << int(math.ceil(math.log2(vlayer.maximum_address + 1)) - 5)) + kvo = kernel["mz_offset"] + ( + 31 << int(math.ceil(math.log2(vlayer.maximum_address + 1)) - 5) + ) else: - kvo = kernel['mz_offset'] + (1 << (vlayer.bits_per_register - 1)) + kvo = kernel["mz_offset"] + (1 << (vlayer.bits_per_register - 1)) try: kvp = vlayer.mapping(kvo, 0) - if (any([(p == kernel['mz_offset'] and layer_name == physical_layer_name) - for (_, _, p, _, layer_name) in kvp])): + if any( + [ + (p == kernel["mz_offset"] and layer_name == physical_layer_name) + for (_, _, p, _, layer_name) in kvp + ] + ): return (virtual_layer_name, kvo, kernel) else: - vollog.debug("Potential kernel_virtual_offset did not map to expected location: {}".format( - hex(kvo))) + vollog.debug( + "Potential kernel_virtual_offset did not map to expected location: {}".format( + hex(kvo) + ) + ) except exceptions.InvalidAddressException: - vollog.debug(f"Potential kernel_virtual_offset caused a page fault: {hex(kvo)}") + vollog.debug( + f"Potential kernel_virtual_offset caused a page fault: {hex(kvo)}" + ) return None vollog.debug("Kernel base determination - testing fixed base address") - return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, False, True, progress_callback) + return self._method_layer_pdb_scan( + context, vlayer, test_physical_kernel, False, True, progress_callback + ) - def _method_layer_pdb_scan(self, - context: interfaces.context.ContextInterface, - vlayer: layers.intel.Intel, - test_kernel: Callable, - optimized: bool = False, - physical: bool = True, - progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: + def _method_layer_pdb_scan( + self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + test_kernel: Callable, + optimized: bool = False, + physical: bool = True, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[ValidKernelType]: # TODO: Verify this is a windows image valid_kernel = None virtual_layer_name = vlayer.name @@ -203,102 +251,145 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): layer_to_scan = virtual_layer_name start_scan_address = 0 - if optimized and not physical and context.layers[layer_to_scan].metadata.architecture in ["Intel64"]: + if ( + optimized + and not physical + and context.layers[layer_to_scan].metadata.architecture in ["Intel64"] + ): # TODO: change this value accordingly when 5-Level paging is supported. - start_scan_address = (0x1f0 << 39) + start_scan_address = 0x1F0 << 39 - kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] - kernels = PDBUtility.pdbname_scan(ctx = context, - layer_name = layer_to_scan, - start = start_scan_address, - page_size = vlayer.page_size, - pdb_names = kernel_pdb_names, - progress_callback = progress_callback) + kernel_pdb_names = [ + bytes(name + ".pdb", "utf-8") + for name in constants.windows.KERNEL_MODULE_NAMES + ] + kernels = PDBUtility.pdbname_scan( + ctx=context, + layer_name=layer_to_scan, + start=start_scan_address, + page_size=vlayer.page_size, + pdb_names=kernel_pdb_names, + progress_callback=progress_callback, + ) for kernel in kernels: valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) if valid_kernel is not None: break return valid_kernel - def _method_offset(self, - context: interfaces.context.ContextInterface, - vlayer: layers.intel.Intel, - pattern: bytes, - result_offset: int, - progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: + def _method_offset( + self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + pattern: bytes, + result_offset: int, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[ValidKernelType]: """Method for finding a suitable kernel offset based on a module table.""" - vollog.debug("Kernel base determination - searching layer module list structure") + vollog.debug( + "Kernel base determination - searching layer module list structure" + ) valid_kernel: Optional[ValidKernelType] = None # If we're here, chances are high we're in a Win10 x64 image with kernel base randomization physical_layer_name = self.get_physical_layer_name(context, vlayer) physical_layer = context.layers[physical_layer_name] # TODO: On older windows, this might be \WINDOWS\system32\nt rather than \SystemRoot\system32\nt - results = physical_layer.scan(context, scanners.BytesScanner(pattern), progress_callback = progress_callback) + results = physical_layer.scan( + context, scanners.BytesScanner(pattern), progress_callback=progress_callback + ) seen: Set[int] = set() # Because this will launch a scan of the virtual layer, we want to be careful for result in results: # TODO: Identify the specific structure we're finding and document this a bit better - pointer = context.object("pdbscan!unsigned long long", - offset = (result + result_offset), - layer_name = physical_layer_name) + pointer = context.object( + "pdbscan!unsigned long long", + offset=(result + result_offset), + layer_name=physical_layer_name, + ) address = pointer & vlayer.address_mask if address in seen: continue seen.add(address) - valid_kernel = self.check_kernel_offset(context, vlayer, address, progress_callback) + valid_kernel = self.check_kernel_offset( + context, vlayer, address, progress_callback + ) if valid_kernel: break return valid_kernel - def method_module_offset(self, - context: interfaces.context.ContextInterface, - vlayer: layers.intel.Intel, - progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - return self._method_offset(context, vlayer, b"\\SystemRoot\\system32\\nt", - -16 - int(vlayer.bits_per_register / 8), progress_callback) + def method_module_offset( + self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[ValidKernelType]: + return self._method_offset( + context, + vlayer, + b"\\SystemRoot\\system32\\nt", + -16 - int(vlayer.bits_per_register / 8), + progress_callback, + ) - def method_kdbg_offset(self, - context: interfaces.context.ContextInterface, - vlayer: layers.intel.Intel, - progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: + def method_kdbg_offset( + self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[ValidKernelType]: return self._method_offset(context, vlayer, b"KDBG", 8, progress_callback) - def check_kernel_offset(self, - context: interfaces.context.ContextInterface, - vlayer: layers.intel.Intel, - address: int, - progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: + def check_kernel_offset( + self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + address: int, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[ValidKernelType]: """Scans a virtual address.""" # Scan a few megs of the virtual space at the location to see if they're potential kernels valid_kernel: Optional[ValidKernelType] = None - kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] + kernel_pdb_names = [ + bytes(name + ".pdb", "utf-8") + for name in constants.windows.KERNEL_MODULE_NAMES + ] virtual_layer_name = vlayer.name with contextlib.suppress(exceptions.InvalidAddressException): - if vlayer.read(address, 0x2) == b'MZ': + if vlayer.read(address, 0x2) == b"MZ": res = list( - PDBUtility.pdbname_scan(ctx = context, - layer_name = vlayer.name, - page_size = vlayer.page_size, - pdb_names = kernel_pdb_names, - progress_callback = progress_callback, - start = address, - end = address + self.max_pdb_size)) + PDBUtility.pdbname_scan( + ctx=context, + layer_name=vlayer.name, + page_size=vlayer.page_size, + pdb_names=kernel_pdb_names, + progress_callback=progress_callback, + start=address, + end=address + self.max_pdb_size, + ) + ) if res: valid_kernel = (virtual_layer_name, address, res[0]) return valid_kernel # List of methods to be run, in order, to determine the valid kernels - methods = [method_kdbg_offset, method_module_offset, method_fixed_mapping, method_slow_scan] + methods = [ + method_kdbg_offset, + method_module_offset, + method_fixed_mapping, + method_slow_scan, + ] - def determine_valid_kernel(self, - context: interfaces.context.ContextInterface, - potential_layers: List[str], - progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: + def determine_valid_kernel( + self, + context: interfaces.context.ContextInterface, + potential_layers: List[str], + progress_callback: constants.ProgressCallback = None, + ) -> Optional[ValidKernelType]: """Runs through the identified potential kernels and verifies their suitability. @@ -327,27 +418,36 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vollog.info("No suitable kernels found during pdbscan") return valid_kernel - def __call__(self, - context: interfaces.context.ContextInterface, - config_path: str, - requirement: interfaces.configuration.RequirementInterface, - progress_callback: constants.ProgressCallback = None) -> None: + def __call__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.RequirementInterface, + progress_callback: constants.ProgressCallback = None, + ) -> None: if requirement.unsatisfied(context, config_path): if "pdbscan" not in context.symbol_space: - context.symbol_space.append(native.NativeTable("pdbscan", native.std_ctypes)) + context.symbol_space.append( + native.NativeTable("pdbscan", native.std_ctypes) + ) # TODO: check if this is a windows symbol requirement, otherwise ignore it - self._symbol_requirements = self.find_requirements(context, config_path, requirement, - requirements.SymbolTableRequirement) - potential_layers = self.find_virtual_layers_from_req(context = context, - config_path = config_path, - requirement = requirement) + self._symbol_requirements = self.find_requirements( + context, config_path, requirement, requirements.SymbolTableRequirement + ) + potential_layers = self.find_virtual_layers_from_req( + context=context, config_path=config_path, requirement=requirement + ) for sub_config_path, symbol_req in self._symbol_requirements: parent_path = interfaces.configuration.parent_path(sub_config_path) if symbol_req.unsatisfied(context, parent_path): - valid_kernel = self.determine_valid_kernel(context, potential_layers, progress_callback) + valid_kernel = self.determine_valid_kernel( + context, potential_layers, progress_callback + ) if valid_kernel: self.set_kernel_virtual_offset(context, valid_kernel) - self.recurse_symbol_fulfiller(context, valid_kernel, progress_callback) + self.recurse_symbol_fulfiller( + context, valid_kernel, progress_callback + ) if progress_callback is not None: progress_callback(100, "PDB scanning finished") diff --git a/volatility3/framework/automagic/stacker.py b/volatility3/framework/automagic/stacker.py index 928e3d068..e611b5f06 100644 --- a/volatility3/framework/automagic/stacker.py +++ b/volatility3/framework/automagic/stacker.py @@ -35,6 +35,7 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): Upon completion it will re-call the :class:`~volatility3.framework.automagic.construct_layers.ConstructionMagic`, so that any stacked layers are actually constructed and added to the context. """ + # Most important automagic, must happen first! priority = 10 @@ -42,14 +43,16 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): super().__init__(*args, **kwargs) self._cached = None - def __call__(self, - context: interfaces.context.ContextInterface, - config_path: str, - requirement: interfaces.configuration.RequirementInterface, - progress_callback: constants.ProgressCallback = None) -> Optional[List[str]]: + def __call__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.RequirementInterface, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[List[str]]: """Runs the automagic over the configurable.""" - framework.import_files(sys.modules['volatility3.framework.layers']) + framework.import_files(sys.modules["volatility3.framework.layers"]) # Quick exit if we're not needed if not requirement.unsatisfied(context, config_path): @@ -58,10 +61,14 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): # Bow out quickly if the UI hasn't provided a single_location unsatisfied = self.unsatisfied(self.context, self.config_path) if unsatisfied: - vollog.info(f"Unable to run LayerStacker, unsatisfied requirement: {unsatisfied}") + vollog.info( + f"Unable to run LayerStacker, unsatisfied requirement: {unsatisfied}" + ) return list(unsatisfied) - if not self.config or not self.config.get('single_location', None): - raise ValueError("Unable to run LayerStacker, single_location parameter not provided") + if not self.config or not self.config.get("single_location", None): + raise ValueError( + "Unable to run LayerStacker, single_location parameter not provided" + ) # Search for suitable requirements self.stack(context, config_path, requirement, progress_callback) @@ -70,9 +77,13 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): progress_callback(100, "Stacking attempts finished") return None - def stack(self, context: interfaces.context.ContextInterface, config_path: str, - requirement: interfaces.configuration.RequirementInterface, - progress_callback: constants.ProgressCallback) -> None: + def stack( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.RequirementInterface, + progress_callback: constants.ProgressCallback, + ) -> None: """Stacks the various layers and attaches these to a specific requirement. @@ -85,7 +96,9 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): # If we're cached, find Now we need to find where to apply the stack configuration if self._cached: top_layer_name, subconfig = self._cached - result = self.find_suitable_requirements(context, config_path, requirement, [top_layer_name]) + result = self.find_suitable_requirements( + context, config_path, requirement, [top_layer_name] + ) if result: appropriate_config_path, layer_name = result context.config.merge(appropriate_config_path, subconfig) @@ -94,43 +107,65 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): self._cached = None new_context = context.clone() - location = self.config.get('single_location', None) + location = self.config.get("single_location", None) # Setup the local copy of the resource current_layer_name = context.layers.free_layer_name("FileLayer") - current_config_path = interfaces.configuration.path_join(config_path, "stack", current_layer_name) + current_config_path = interfaces.configuration.path_join( + config_path, "stack", current_layer_name + ) # This must be specific to get us started, setup the config and run - new_context.config[interfaces.configuration.path_join(current_config_path, "location")] = location - physical_layer = physical.FileLayer(new_context, current_config_path, current_layer_name) + new_context.config[ + interfaces.configuration.path_join(current_config_path, "location") + ] = location + physical_layer = physical.FileLayer( + new_context, current_config_path, current_layer_name + ) new_context.add_layer(physical_layer) - stacked_layers = self.stack_layer(new_context, current_layer_name, self.create_stackers_list(), - progress_callback) + stacked_layers = self.stack_layer( + new_context, + current_layer_name, + self.create_stackers_list(), + progress_callback, + ) if stacked_layers is not None: # Applies the stacked_layers to each requirement in the requirements list - result = self.find_suitable_requirements(new_context, config_path, requirement, stacked_layers) + result = self.find_suitable_requirements( + new_context, config_path, requirement, stacked_layers + ) if result: path, layer = result # splice in the new configuration into the original context - context.config.merge(path, new_context.layers[layer].build_configuration()) + context.config.merge( + path, new_context.layers[layer].build_configuration() + ) # Call the construction magic now we may have new things to construct constructor = construct_layers.ConstructionMagic( - context, interfaces.configuration.path_join(self.config_path, "ConstructionMagic")) + context, + interfaces.configuration.path_join( + self.config_path, "ConstructionMagic" + ), + ) constructor(context, config_path, requirement) # Stash the changed config items - self._cached = context.config.get(path, None), context.config.branch(path) + self._cached = context.config.get(path, None), context.config.branch( + path + ) vollog.debug(f"Stacked layers: {stacked_layers}") @classmethod - def stack_layer(cls, - context: interfaces.context.ContextInterface, - initial_layer: str, - stack_set: List[Type[interfaces.automagic.StackerLayerInterface]] = None, - progress_callback: constants.ProgressCallback = None): + def stack_layer( + cls, + context: interfaces.context.ContextInterface, + initial_layer: str, + stack_set: List[Type[interfaces.automagic.StackerLayerInterface]] = None, + progress_callback: constants.ProgressCallback = None, + ): """Stacks as many possible layers on top of the initial layer as can be done. WARNING: This modifies the context provided and may pollute it with unnecessary layers @@ -154,11 +189,15 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): stacked = True stacked_layers = [initial_layer] if stack_set is None: - stack_set = list(framework.class_subclasses(interfaces.automagic.StackerLayerInterface)) + stack_set = list( + framework.class_subclasses(interfaces.automagic.StackerLayerInterface) + ) for stacker_item in stack_set: if not issubclass(stacker_item, interfaces.automagic.StackerLayerInterface): - raise TypeError(f"Stacker {stacker_item.__name__} is not a descendent of StackerLayerInterface") + raise TypeError( + f"Stacker {stacker_item.__name__} is not a descendent of StackerLayerInterface" + ) while stacked: stacked = False @@ -167,17 +206,27 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): for stacker_cls in stack_set: stacker = stacker_cls() try: - vollog.log(constants.LOGLEVEL_VV, f"Attempting to stack using {stacker_cls.__name__}") + vollog.log( + constants.LOGLEVEL_VV, + f"Attempting to stack using {stacker_cls.__name__}", + ) new_layer = stacker.stack(context, initial_layer, progress_callback) if new_layer: context.layers.add_layer(new_layer) - vollog.log(constants.LOGLEVEL_VV, - f"Stacked {new_layer.name} using {stacker_cls.__name__}") + vollog.log( + constants.LOGLEVEL_VV, + f"Stacked {new_layer.name} using {stacker_cls.__name__}", + ) break except Exception as excp: # Stacking exceptions are likely only of interest to developers, so the lowest level of logging - fulltrace = traceback.TracebackException.from_exception(excp).format(chain = True) - vollog.log(constants.LOGLEVEL_VVV, f"Exception during stacking: {str(excp)}") + fulltrace = traceback.TracebackException.from_exception( + excp + ).format(chain=True) + vollog.log( + constants.LOGLEVEL_VVV, + f"Exception during stacking: {str(excp)}", + ) vollog.log(constants.LOGLEVEL_VVVV, "\n".join(fulltrace)) else: stacked = False @@ -188,11 +237,15 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): stack_set.remove(stacker_cls) return stacked_layers - def create_stackers_list(self) -> List[Type[interfaces.automagic.StackerLayerInterface]]: + def create_stackers_list( + self, + ) -> List[Type[interfaces.automagic.StackerLayerInterface]]: """Creates the list of stackers to use based on the config option""" - stack_set = sorted(framework.class_subclasses(interfaces.automagic.StackerLayerInterface), - key = lambda x: x.stack_order) - stacker_list = self.config.get('stackers', []) + stack_set = sorted( + framework.class_subclasses(interfaces.automagic.StackerLayerInterface), + key=lambda x: x.stack_order, + ) + stacker_list = self.config.get("stackers", []) if len(stacker_list): result = [] for stacker in stack_set: @@ -202,9 +255,13 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): return stack_set @classmethod - def find_suitable_requirements(cls, context: interfaces.context.ContextInterface, config_path: str, - requirement: interfaces.configuration.RequirementInterface, - stacked_layers: List[str]) -> Optional[Tuple[str, str]]: + def find_suitable_requirements( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.RequirementInterface, + stacked_layers: List[str], + ) -> Optional[Tuple[str, str]]: """Looks for translation layer requirements and attempts to apply the stacked layers to it. If it succeeds it returns the configuration path and layer name where the stacked nodes were spliced into the tree. @@ -213,7 +270,9 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): A tuple of a configuration path and layer name for the top of the stacked layers or None if suitable requirements are not found """ - child_config_path = interfaces.configuration.path_join(config_path, requirement.name) + child_config_path = interfaces.configuration.path_join( + config_path, requirement.name + ) if isinstance(requirement, requirements.TranslationLayerRequirement): if requirement.unsatisfied(context, config_path): original_setting = context.config.get(child_config_path, None) @@ -229,7 +288,9 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): else: return child_config_path, context.config.get(child_config_path, None) for req_name, req in requirement.requirements.items(): - result = cls.find_suitable_requirements(context, child_config_path, req, stacked_layers) + result = cls.find_suitable_requirements( + context, child_config_path, req, stacked_layers + ) if result: return result return None @@ -238,23 +299,29 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # This is not optional for the stacker to run, so optional must be marked as False return [ - requirements.URIRequirement(name = "single_location", - description = "Specifies a base location on which to stack", - optional = True), - requirements.ListRequirement(name = "stackers", description = "List of stackers", optional = True) + requirements.URIRequirement( + name="single_location", + description="Specifies a base location on which to stack", + optional=True, + ), + requirements.ListRequirement( + name="stackers", description="List of stackers", optional=True + ), ] def choose_os_stackers(plugin: Type[interfaces.plugins.PluginInterface]) -> List[str]: """Identifies the stackers that should be run, based on the plugin (and thus os) provided""" - plugin_first_level = plugin.__module__.split('.')[2] + plugin_first_level = plugin.__module__.split(".")[2] # Ensure all stackers are loaded - framework.import_files(sys.modules['volatility3.framework.layers']) + framework.import_files(sys.modules["volatility3.framework.layers"]) result = [] - for stacker in sorted(framework.class_subclasses(interfaces.automagic.StackerLayerInterface), - key = lambda x: x.stack_order): + for stacker in sorted( + framework.class_subclasses(interfaces.automagic.StackerLayerInterface), + key=lambda x: x.stack_order, + ): if plugin_first_level in stacker.exclusion_list: continue result.append(stacker.__name__) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 164021340..44a76506c 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -26,6 +26,7 @@ BannersType = Dict[bytes, List[str]] ### Identifiers + class IdentifierProcessor: operating_system = None @@ -40,47 +41,53 @@ class IdentifierProcessor: Returns: identifier is valid or None if not found """ - raise NotImplementedError("This base class has no get_identifier method defined") + raise NotImplementedError( + "This base class has no get_identifier method defined" + ) class WindowsIdentifier(IdentifierProcessor): - operating_system = 'windows' - separator = '|' + operating_system = "windows" + separator = "|" @classmethod def get_identifier(cls, json) -> Optional[bytes]: """Returns the identifier for the file if one can be found""" - windows_metadata = json.get('metadata', {}).get('windows', {}).get('pdb', {}) + windows_metadata = json.get("metadata", {}).get("windows", {}).get("pdb", {}) if windows_metadata: - guid = windows_metadata.get('GUID', None) - age = windows_metadata.get('age', None) - database = windows_metadata.get('database', None) + guid = windows_metadata.get("GUID", None) + age = windows_metadata.get("age", None) + database = windows_metadata.get("database", None) if guid and age and database: return cls.generate(database, guid, age) return None @classmethod def generate(cls, pdb_name: str, guid: str, age: int) -> bytes: - return bytes(cls.separator.join([pdb_name, guid.upper(), str(age)]), 'latin-1') + return bytes(cls.separator.join([pdb_name, guid.upper(), str(age)]), "latin-1") class MacIdentifier(IdentifierProcessor): - operating_system = 'mac' + operating_system = "mac" @classmethod def get_identifier(cls, json) -> Optional[bytes]: - mac_banner = json.get('symbols', {}).get('version', {}).get('constant_data', None) + mac_banner = ( + json.get("symbols", {}).get("version", {}).get("constant_data", None) + ) if mac_banner: return base64.b64decode(mac_banner) return None class LinuxIdentifier(IdentifierProcessor): - operating_system = 'linux' + operating_system = "linux" @classmethod def get_identifier(cls, json) -> Optional[bytes]: - linux_banner = json.get('symbols', {}).get('linux_banner', {}).get('constant_data', None) + linux_banner = ( + json.get("symbols", {}).get("linux_banner", {}).get("constant_data", None) + ) if linux_banner: return base64.b64decode(linux_banner) return None @@ -88,6 +95,7 @@ class LinuxIdentifier(IdentifierProcessor): ### CacheManagers + class CacheManagerInterface(interfaces.configuration.VersionableInterface): def __init__(self, filename: str): super().__init__() @@ -100,7 +108,9 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Adds an identifier to the store""" pass - def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + def find_location( + self, identifier: bytes, operating_system: Optional[str] + ) -> Optional[str]: """Returns the location of the symbol file given the identifier Args: @@ -123,8 +133,9 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """ pass - def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ - Dict[bytes, str]: + def get_identifier_dictionary( + self, operating_system: Optional[str] = None, local_only: bool = False + ) -> Dict[bytes, str]: """Returns a dictionary of identifiers and locations Args: @@ -144,7 +155,9 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns all identifiers for a particular operating system""" pass - def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + def get_location_statistics( + self, location: str + ) -> Optional[Tuple[int, int, int, int]]: """Returns ISF statistics based on the location Returns: @@ -158,7 +171,6 @@ class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) - def __init__(self, filename: str): super().__init__(filename) self.cache_period = constants.SQLITE_CACHE_PERIOD @@ -173,27 +185,39 @@ class SqliteCache(CacheManagerInterface): database.row_factory = sqlite3.Row database.cursor().execute( - f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})') - schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() + f"CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})" + ) + schema_version = ( + database.cursor() + .execute("SELECT schema_version FROM database_info") + .fetchone() + ) if not schema_version: - database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCHEMA_VERSION})') - elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCHEMA_VERSION: + database.cursor().execute( + f"INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCHEMA_VERSION})" + ) + elif schema_version["schema_version"] == constants.CACHE_SQLITE_SCHEMA_VERSION: # All good, so pass and move on pass else: - vollog.info(f"Previous cache schema version found: {schema_version['schema_version']}") + vollog.info( + f"Previous cache schema version found: {schema_version['schema_version']}" + ) # TODO: Implement code if the schema changes # Current this should never happen so we start over again database.close() os.unlink(path) return self._connect_storage(path) database.cursor().execute( - 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, hash TEXT,' - 'stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)') + "CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, hash TEXT," + "stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)" + ) database.commit() return database - def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + def find_location( + self, identifier: bytes, operating_system: Optional[str] + ) -> Optional[str]: """Returns the location of the symbol file given the identifier. If multiple locations exist for an identifier, the last found is returned @@ -204,55 +228,82 @@ class SqliteCache(CacheManagerInterface): Returns: The location of the symbols file that matches the identifier or None """ - statement = 'SELECT location FROM cache WHERE identifier = ?' + statement = "SELECT location FROM cache WHERE identifier = ?" parameters = (identifier,) if operating_system is not None: - statement = 'SELECT location FROM cache WHERE identifier = ? AND operating_system = ?' + statement = "SELECT location FROM cache WHERE identifier = ? AND operating_system = ?" parameters = (identifier, operating_system) results = self._database.cursor().execute(statement, parameters).fetchall() result = None for row in results: - result = row['location'] + result = row["location"] return result def get_local_locations(self) -> Generator[str, None, None]: - result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = 1').fetchall() + result = ( + self._database.cursor() + .execute("SELECT DISTINCT location FROM cache WHERE local = 1") + .fetchall() + ) for row in result: - yield row['location'] + yield row["location"] def is_url_local(self, url: str) -> bool: """Determines whether an url is local or not""" parsed = urllib.parse.urlparse(url) - return parsed.scheme in ['file', 'jar'] + return parsed.scheme in ["file", "jar"] def get_identifier(self, location: str) -> Optional[bytes]: - results = self._database.cursor().execute('SELECT identifier FROM cache WHERE location = ?', - (location,)).fetchall() + results = ( + self._database.cursor() + .execute("SELECT identifier FROM cache WHERE location = ?", (location,)) + .fetchall() + ) for row in results: - return row['identifier'] + return row["identifier"] return None - def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: - results = self._database.cursor().execute( - 'SELECT stats_base_types, stats_types, stats_enums, stats_symbols FROM cache WHERE location = ?', - (location,)).fetchall() + def get_location_statistics( + self, location: str + ) -> Optional[Tuple[int, int, int, int]]: + results = ( + self._database.cursor() + .execute( + "SELECT stats_base_types, stats_types, stats_enums, stats_symbols FROM cache WHERE location = ?", + (location,), + ) + .fetchall() + ) for row in results: - return row['stats_base_types'], row['stats_types'], row['stats_enums'], row['stats_symbols'] + return ( + row["stats_base_types"], + row["stats_types"], + row["stats_enums"], + row["stats_symbols"], + ) return None def get_hash(self, location: str) -> Optional[str]: - results = self._database.cursor().execute('SELECT hash FROM cache WHERE location = ?', - (location,)).fetchall() + results = ( + self._database.cursor() + .execute("SELECT hash FROM cache WHERE location = ?", (location,)) + .fetchall() + ) for row in results: - return row['hash'] + return row["hash"] return None - def update(self, progress_callback = None): + def update(self, progress_callback=None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. """ - on_disk_locations = set([filename for filename in intermed.IntermediateSymbolTable.file_symbol_url('')]) + on_disk_locations = set( + [ + filename + for filename in intermed.IntermediateSymbolTable.file_symbol_url("") + ] + ) cached_locations = set(self.get_local_locations()) new_locations = on_disk_locations.difference(cached_locations) @@ -261,32 +312,41 @@ class SqliteCache(CacheManagerInterface): cache_update = set() files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: - result = self._database.cursor().execute("SELECT location, cached FROM cache WHERE local = 1 " - f"AND cached < date('now', '{self.cache_period}');") + result = self._database.cursor().execute( + "SELECT location, cached FROM cache WHERE local = 1 " + f"AND cached < date('now', '{self.cache_period}');" + ) for row in result: - location = row['location'] - stored_timestamp = datetime.datetime.fromisoformat(row['cached']) - timestamp = stored_timestamp # Default to requiring update + location = row["location"] + stored_timestamp = datetime.datetime.fromisoformat(row["cached"]) + timestamp = stored_timestamp # Default to requiring update # See if the file is a local URL type we can handle: parsed = urllib.parse.urlparse(location) pathname = None - if parsed.scheme == 'file': + if parsed.scheme == "file": pathname = urllib.request.url2pathname(parsed.path) - if parsed.scheme == 'jar': + if parsed.scheme == "jar": inner_url = urllib.parse.urlparse(parsed.path) - if inner_url.scheme == 'file': - pathname = inner_url.path.split('!')[0] + if inner_url.scheme == "file": + pathname = inner_url.path.split("!")[0] if pathname: - timestamp = datetime.datetime.fromtimestamp(os.stat(pathname).st_mtime) + timestamp = datetime.datetime.fromtimestamp( + os.stat(pathname).st_mtime + ) else: - vollog.log(constants.LOGLEVEL_VVVV, - "File location in database classed as local but not file/jar URL") + vollog.log( + constants.LOGLEVEL_VVVV, + "File location in database classed as local but not file/jar URL", + ) # If we're supposed to include it, and our last check is older than (or equal to) the file timestamp - if row['location'] in files_to_timestamp and stored_timestamp < timestamp: - cache_update.add(row['location']) + if ( + row["location"] in files_to_timestamp + and stored_timestamp < timestamp + ): + cache_update.add(row["location"]) idextractors = list(framework.class_subclasses(IdentifierProcessor)) @@ -298,8 +358,10 @@ class SqliteCache(CacheManagerInterface): try: for counter, location in enumerate(files_to_process): # Open location - progress_callback(counter * 100 / number_files_to_process, - f"Updating caches for {number_files_to_process} files...") + progress_callback( + counter * 100 / number_files_to_process, + f"Updating caches for {number_files_to_process} files...", + ) try: with resources.ResourceAccessor().open(location) as fp: json_obj = json.load(fp) @@ -307,10 +369,10 @@ class SqliteCache(CacheManagerInterface): identifier = None # Get stats - stats_base_types = len(json_obj.get('base_types', {})) - stats_types = len(json_obj.get('types', {})) - stats_enums = len(json_obj.get('enums', {})) - stats_symbols = len(json_obj.get('symbols', {})) + stats_base_types = len(json_obj.get("base_types", {})) + stats_types = len(json_obj.get("types", {})) + stats_enums = len(json_obj.get("enums", {})) + stats_symbols = len(json_obj.get("symbols", {})) operating_system = None for idextractor in idextractors: @@ -334,12 +396,19 @@ class SqliteCache(CacheManagerInterface): stats_types, stats_enums, stats_symbols, - self.is_url_local(location) - )) + self.is_url_local(location), + ), + ) if identifier is not None: - vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") + vollog.log( + constants.LOGLEVEL_VV, + f"Identified {location} as {identifier}", + ) else: - vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"No identifier found for {location}", + ) except Exception as excp: vollog.log(constants.LOGLEVEL_VVVV, excp) finally: @@ -348,20 +417,23 @@ class SqliteCache(CacheManagerInterface): # Remote Entries if not constants.OFFLINE and constants.REMOTE_ISF_URL: - progress_callback(0, 'Reading remote ISF list') + progress_callback(0, "Reading remote ISF list") cursor = self._database.cursor() cursor.execute( - f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', {self.cache_period})") + f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', {self.cache_period})" + ) remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) - progress_callback(50, 'Reading remote ISF list') + progress_callback(50, "Reading remote ISF list") for operating_system in constants.OS_CATEGORIES: - identifiers = remote_identifiers.process({}, operating_system = operating_system) + identifiers = remote_identifiers.process( + {}, operating_system=operating_system + ) for identifier, location in identifiers: cursor.execute( "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - (location, identifier, operating_system, False) + (location, identifier, operating_system, False), ) - progress_callback(100, 'Reading remote ISF list') + progress_callback(100, "Reading remote ISF list") self._database.commit() # Missing entries @@ -369,52 +441,69 @@ class SqliteCache(CacheManagerInterface): if missing_locations: self._database.cursor().execute( f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", - [x for x in missing_locations]) + [x for x in missing_locations], + ) self._database.commit() - def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ - Dict[bytes, str]: + def get_identifier_dictionary( + self, operating_system: Optional[str] = None, local_only: bool = False + ) -> Dict[bytes, str]: output = {} additions = [] - statement = 'SELECT location, identifier FROM cache' + statement = "SELECT location, identifier FROM cache" if local_only: - additions.append('local = 1') + additions.append("local = 1") if operating_system: additions.append(f"operating_system = '{operating_system}'") if additions: statement += f" WHERE {' AND '.join(additions)}" results = self._database.cursor().execute(statement) for row in results: - if row['identifier'] in output and row['identifier'] and row['location']: + if row["identifier"] in output and row["identifier"] and row["location"]: vollog.debug( - f"Duplicate entry for identifier {row['identifier']}: {row['location']} and {output[row['identifier']]}") - output[row['identifier']] = row['location'] + f"Duplicate entry for identifier {row['identifier']}: {row['location']} and {output[row['identifier']]}" + ) + output[row["identifier"]] = row["location"] return output def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: if operating_system: - results = self._database.cursor().execute('SELECT identifier FROM cache WHERE operating_system = ?', - (operating_system,)).fetchall() + results = ( + self._database.cursor() + .execute( + "SELECT identifier FROM cache WHERE operating_system = ?", + (operating_system,), + ) + .fetchall() + ) else: - results = self._database.cursor().execute('SELECT identifier FROM cache').fetchall() + results = ( + self._database.cursor() + .execute("SELECT identifier FROM cache") + .fetchall() + ) output = [] for row in results: - output.append(row['identifier']) + output.append(row["identifier"]) return output ### Automagic + class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): """Runs through all symbol tables and caches their identifiers""" + priority = 0 def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + identifiers_path = os.path.join( + constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + ) self._cache = SqliteCache(identifiers_path) - def __call__(self, context, config_path, configurable, progress_callback = None): + def __call__(self, context, config_path, configurable, progress_callback=None): """Runs the automagic over the configurable.""" self._cache.update(progress_callback) @@ -422,37 +511,45 @@ class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: """Returns a list of RequirementInterface objects required by this object.""" - return [requirements.VersionRequirement(name = 'SQLiteCache', component = SqliteCache, version = (1, 0, 0))] + return [ + requirements.VersionRequirement( + name="SQLiteCache", component=SqliteCache, version=(1, 0, 0) + ) + ] class RemoteIdentifierFormat: def __init__(self, location: str): self._location = location - with resources.ResourceAccessor().open(url = location) as fp: + with resources.ResourceAccessor().open(url=location) as fp: self._data = json.load(fp) if not self._verify(): raise ValueError("Unsupported version for remote identifier list format") def _verify(self) -> bool: - version = self._data.get('version', 0) + version = self._data.get("version", 0) if version in [1]: - setattr(self, 'process', getattr(self, f'process_v{version}')) + setattr(self, "process", getattr(self, f"process_v{version}")) return True return False - def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]) -> Generator[ - Tuple[bytes, str], None, None]: + def process( + self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str] + ) -> Generator[Tuple[bytes, str], None, None]: raise ValueError("Identifier List version not verified") - def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]) -> Generator[ - Tuple[bytes, str], None, None]: + def process_v1( + self, + identifiers: Optional[Dict[bytes, List[str]]], + operating_system: Optional[str], + ) -> Generator[Tuple[bytes, str], None, None]: if operating_system in self._data: for identifier in self._data[operating_system]: binary_identifier = base64.b64decode(identifier) for value in self._data[operating_system][identifier]: yield binary_identifier, value - if 'additional' in self._data: - for location in self._data['additional']: + if "additional" in self._data: + for location in self._data["additional"]: try: subrbf = RemoteIdentifierFormat(location) yield from subrbf.process(identifiers, operating_system) diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 7a197dffc..0143e74b1 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -16,6 +16,7 @@ vollog = logging.getLogger(__name__) class SymbolFinder(interfaces.automagic.AutomagicInterface): """Symbol loader based on signature strings.""" + priority = 40 banner_config_key: str = "banner" @@ -23,17 +24,23 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): symbol_class: Optional[str] = None find_aslr: Optional[Callable] = None - def __init__(self, context: interfaces.context.ContextInterface, config_path: str) -> None: + def __init__( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> None: super().__init__(context, config_path) - self._requirements: List[Tuple[str, interfaces.configuration.RequirementInterface]] = [] + self._requirements: List[ + Tuple[str, interfaces.configuration.RequirementInterface] + ] = [] self._banners: symbol_cache.BannersType = {} @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.VersionRequirement(name = 'SQLiteCache', - component = symbol_cache.SqliteCache, - version = (1, 0, 0)) + requirements.VersionRequirement( + name="SQLiteCache", + component=symbol_cache.SqliteCache, + version=(1, 0, 0), + ) ] @property @@ -41,16 +48,22 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): """Creates a cached copy of the results, but only it's been requested.""" if not self._banners: - identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + identifiers_path = os.path.join( + constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + ) cache = symbol_cache.SqliteCache(identifiers_path) - self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system) + self._banners = cache.get_identifier_dictionary( + operating_system=self.operating_system + ) return self._banners - def __call__(self, - context: interfaces.context.ContextInterface, - config_path: str, - requirement: interfaces.configuration.RequirementInterface, - progress_callback: constants.ProgressCallback = None) -> None: + def __call__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.RequirementInterface, + progress_callback: constants.ProgressCallback = None, + ) -> None: """Searches for SymbolTableRequirements and attempt to populate them.""" @@ -61,30 +74,47 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): self._requirements = self.find_requirements( context, config_path, - requirement, (requirements.TranslationLayerRequirement, requirements.SymbolTableRequirement), - shortcut = False) + requirement, + ( + requirements.TranslationLayerRequirement, + requirements.SymbolTableRequirement, + ), + shortcut=False, + ) for (sub_path, requirement) in self._requirements: parent_path = interfaces.configuration.parent_path(sub_path) - if (isinstance(requirement, requirements.SymbolTableRequirement) - and requirement.unsatisfied(context, parent_path)): + if isinstance( + requirement, requirements.SymbolTableRequirement + ) and requirement.unsatisfied(context, parent_path): for (tl_sub_path, tl_requirement) in self._requirements: tl_parent_path = interfaces.configuration.parent_path(tl_sub_path) # Find the TranslationLayer sibling to the SymbolTableRequirement - if (isinstance(tl_requirement, requirements.TranslationLayerRequirement) - and tl_parent_path == parent_path): + if ( + isinstance( + tl_requirement, requirements.TranslationLayerRequirement + ) + and tl_parent_path == parent_path + ): if context.config.get(tl_sub_path, None): - self._banner_scan(context, parent_path, requirement, context.config[tl_sub_path], - progress_callback) + self._banner_scan( + context, + parent_path, + requirement, + context.config[tl_sub_path], + progress_callback, + ) break - def _banner_scan(self, - context: interfaces.context.ContextInterface, - config_path: str, - requirement: interfaces.configuration.ConstructableRequirementInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> None: + def _banner_scan( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.ConstructableRequirementInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> None: """Accepts a context, config_path and SymbolTableRequirement, with a constructed layer_name and scans the layer for banners.""" @@ -98,15 +128,18 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): # Check if the Stacker has already found what we're looking for if layer.config.get(self.banner_config_key, None): - banner_list = [(0, bytes(layer.config[self.banner_config_key], - 'raw_unicode_escape'))] # type: Iterable[Any] + banner_list = [ + (0, bytes(layer.config[self.banner_config_key], "raw_unicode_escape")) + ] # type: Iterable[Any] else: # Swap to the physical layer for scanning # Only traverse down a layer if it's an intel layer # TODO: Fix this so it works for layers other than just Intel if isinstance(layer, layers.intel.Intel): - layer = context.layers[layer.config['memory_layer']] - banner_list = layer.scan(context = context, scanner = mss, progress_callback = progress_callback) + layer = context.layers[layer.config["memory_layer"]] + banner_list = layer.scan( + context=context, scanner=mss, progress_callback=progress_callback + ) for _, banner in banner_list: vollog.debug(f"Identified banner: {repr(banner)}") @@ -117,9 +150,15 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join - context.config[path_join(config_path, requirement.name, "class")] = clazz - context.config[path_join(config_path, requirement.name, "isf_url")] = isf_path - context.config[path_join(config_path, requirement.name, "symbol_mask")] = layer.address_mask + context.config[ + path_join(config_path, requirement.name, "class") + ] = clazz + context.config[ + path_join(config_path, requirement.name, "isf_url") + ] = isf_path + context.config[ + path_join(config_path, requirement.name, "symbol_mask") + ] = layer.address_mask # Construct the appropriate symbol table requirement.construct(context, config_path) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index aaef3e820..986eeae22 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -41,8 +41,14 @@ class DtbSelfReferential: """A generic DTB test which looks for a self-referential pointer at *any* index within the page.""" - def __init__(self, layer_type: Type[layers.intel.Intel], ptr_struct: str, mask: int, - valid_range: Iterable[int], reserved_bits: int) -> None: + def __init__( + self, + layer_type: Type[layers.intel.Intel], + ptr_struct: str, + mask: int, + valid_range: Iterable[int], + reserved_bits: int, + ) -> None: self.layer_type = layer_type self.ptr_struct = ptr_struct self.ptr_size = struct.calcsize(ptr_struct) @@ -51,22 +57,26 @@ class DtbSelfReferential: self.valid_range = valid_range self.reserved_bits = reserved_bits - def __call__(self, data: bytes, data_offset: int, page_offset: int) -> Optional[Tuple[int, int]]: - page = data[page_offset:page_offset + self.page_size] + def __call__( + self, data: bytes, data_offset: int, page_offset: int + ) -> Optional[Tuple[int, int]]: + page = data[page_offset : page_offset + self.page_size] if not page: return None ref_pages = set() for ref in range(0, self.page_size, self.ptr_size): - ptr_data = page[ref:ref + self.ptr_size] - ptr, = struct.unpack(self.ptr_struct, ptr_data) + ptr_data = page[ref : ref + self.ptr_size] + (ptr,) = struct.unpack(self.ptr_struct, ptr_data) # For both Intel-32e, bit 7 is reserved (more are reserved in PAE), so if that's ever set, # we can move on if (ptr & self.reserved_bits) and (ptr & 0x01): return None - if ((ptr & self.mask) == (data_offset + page_offset)) and (data_offset + page_offset > 0): + if ((ptr & self.mask) == (data_offset + page_offset)) and ( + data_offset + page_offset > 0 + ): # Pointer must be valid - if (ptr & 0x01): + if ptr & 0x01: ref_pages.add(ref) # The DTB is extremely unlikely to refer back to itself. so the number of reference should always be exactly 1 @@ -78,62 +88,78 @@ class DtbSelfReferential: class DtbSelfRef32bit(DtbSelfReferential): - def __init__(self): - super().__init__(layer_type = layers.intel.WindowsIntel, - ptr_struct = "I", - mask = 0xFFFFF000, - valid_range = [0x300], - reserved_bits = 0x0) + super().__init__( + layer_type=layers.intel.WindowsIntel, + ptr_struct="I", + mask=0xFFFFF000, + valid_range=[0x300], + reserved_bits=0x0, + ) class DtbSelfRef64bit(DtbSelfReferential): - def __init__(self) -> None: - super().__init__(layer_type = layers.intel.WindowsIntel32e, - ptr_struct = "Q", - mask = 0x3FFFFFFFFFF000, - valid_range = range(0x100, 0x1ff), - reserved_bits = 0x80) + super().__init__( + layer_type=layers.intel.WindowsIntel32e, + ptr_struct="Q", + mask=0x3FFFFFFFFFF000, + valid_range=range(0x100, 0x1FF), + reserved_bits=0x80, + ) class DtbSelfRef64bitOldWindows(DtbSelfReferential): - def __init__(self) -> None: - super().__init__(layer_type = layers.intel.WindowsIntel32e, - ptr_struct = "Q", - mask = 0x3FFFFFFFFFF000, - valid_range = [0x1ed], - reserved_bits = 0x80) + super().__init__( + layer_type=layers.intel.WindowsIntel32e, + ptr_struct="Q", + mask=0x3FFFFFFFFFF000, + valid_range=[0x1ED], + reserved_bits=0x80, + ) class DtbSelfRefPae(DtbSelfReferential): - def __init__(self) -> None: - super().__init__(layer_type = layers.intel.WindowsIntelPAE, - ptr_struct = "Q", - valid_range = [0x3], - mask = 0x3FFFFFFFFFF000, - reserved_bits = 0x0) + super().__init__( + layer_type=layers.intel.WindowsIntelPAE, + ptr_struct="Q", + valid_range=[0x3], + mask=0x3FFFFFFFFFF000, + reserved_bits=0x0, + ) @staticmethod def _and_bytes(abytes, bbytes): return bytes([a & b for a, b in zip(abytes[::-1], bbytes[::-1])][::-1]) - def __call__(self, data: bytes, data_offset: int, page_offset: int) -> Optional[Tuple[int, int]]: + def __call__( + self, data: bytes, data_offset: int, page_offset: int + ) -> Optional[Tuple[int, int]]: dtb = super().__call__(data, data_offset, page_offset) if dtb: # Find the top page top_pae_page = dtb[0] - 0x4000 # The top page should map to the next four pages after it # Build what we expect the page table to be - expected_table = b''.join([struct.pack(self.ptr_struct, top_pae_page + (i * 0x1000)) for i in range(1, 5)]) + expected_table = b"".join( + [ + struct.pack(self.ptr_struct, top_pae_page + (i * 0x1000)) + for i in range(1, 5) + ] + ) # Mask off the page bits of top level page map page_table_mask = b"\x00\xf0\xff\xff\xff\xff\xff\xff" * 4 - page_table = data[top_pae_page - data_offset: top_pae_page - data_offset + (4 * self.ptr_size)] + page_table = data[ + top_pae_page + - data_offset : top_pae_page + - data_offset + + (4 * self.ptr_size) + ] # Compare them anded_bytes = self._and_bytes(page_table, page_table_mask) - if (anded_bytes == expected_table): + if anded_bytes == expected_table: return top_pae_page, dtb[1] # Return None since the dtb value *isn't* None return None @@ -143,6 +169,7 @@ class DtbSelfRefPae(DtbSelfReferential): class PageMapScanner(interfaces.layers.ScannerInterface): """Scans through all pages using DTB tests to determine a dtb offset and architecture.""" + overlap = 0x4000 thread_safe = True tests = [DtbSelfRef64bit(), DtbSelfRefPae(), DtbSelfRef32bit()] @@ -153,7 +180,9 @@ class PageMapScanner(interfaces.layers.ScannerInterface): if tests: self.tests = tests - def __call__(self, data: bytes, data_offset: int) -> Generator[Tuple[DtbSelfReferential, int], None, None]: + def __call__( + self, data: bytes, data_offset: int + ) -> Generator[Tuple[DtbSelfReferential, int], None, None]: for page_offset in range(0, len(data), 0x1000): for test in self.tests: result = test(data, data_offset, page_offset) @@ -163,20 +192,29 @@ class PageMapScanner(interfaces.layers.ScannerInterface): class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): stack_order = 40 - exclusion_list = ['mac', 'linux'] + exclusion_list = ["mac", "linux"] # Group these by region so we only run over the data once - test_sets = [("Detecting Self-referential pointer for recent windows", - [DtbSelfRef64bit()], [(0x150000, 0x150000), (0x650000, 0xa0000)]), - ("Older windows fixed location self-referential pointers", - [DtbSelfRefPae(), DtbSelfRef32bit(), DtbSelfRef64bitOldWindows()], [(0x30000, 0x1000000)]) - ] + test_sets = [ + ( + "Detecting Self-referential pointer for recent windows", + [DtbSelfRef64bit()], + [(0x150000, 0x150000), (0x650000, 0xA0000)], + ), + ( + "Older windows fixed location self-referential pointers", + [DtbSelfRefPae(), DtbSelfRef32bit(), DtbSelfRef64bitOldWindows()], + [(0x30000, 0x1000000)], + ), + ] @classmethod - def stack(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to determine and stack an intel layer on a physical layer where possible. @@ -192,29 +230,43 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): base_layer = context.layers[layer_name] if isinstance(base_layer, intel.Intel): return None - if base_layer.metadata.get('os', None) not in ['Windows', 'Unknown']: + if base_layer.metadata.get("os", None) not in ["Windows", "Unknown"]: return None layer = config_path = None # Check the metadata - if (base_layer.metadata.get('os', None) == 'Windows' and base_layer.metadata.get('page_map_offset')): - arch = base_layer.metadata.get('architecture', None) - if arch not in ['Intel32', 'Intel64']: + if base_layer.metadata.get("os", None) == "Windows" and base_layer.metadata.get( + "page_map_offset" + ): + arch = base_layer.metadata.get("architecture", None) + if arch not in ["Intel32", "Intel64"]: return None # Set the layer type layer_type: Type = intel.WindowsIntel - if arch == 'Intel64': + if arch == "Intel64": layer_type = intel.WindowsIntel32e - elif base_layer.metadata.get('pae', False): + elif base_layer.metadata.get("pae", False): layer_type = intel.WindowsIntelPAE # Construct the layer new_layer_name = context.layers.free_layer_name("IntelLayer") - config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name) - context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name - context.config[interfaces.configuration.path_join( - config_path, "page_map_offset")] = base_layer.metadata['page_map_offset'] - layer = layer_type(context, config_path = config_path, name = new_layer_name, metadata = {'os': 'Windows'}) - page_map_offset = context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] + config_path = interfaces.configuration.path_join( + "IntelHelper", new_layer_name + ) + context.config[ + interfaces.configuration.path_join(config_path, "memory_layer") + ] = layer_name + context.config[ + interfaces.configuration.path_join(config_path, "page_map_offset") + ] = base_layer.metadata["page_map_offset"] + layer = layer_type( + context, + config_path=config_path, + name=new_layer_name, + metadata={"os": "Windows"}, + ) + page_map_offset = context.config[ + interfaces.configuration.path_join(config_path, "page_map_offset") + ] vollog.debug(f"DTB was given to us by base layer: {hex(page_map_offset)}") return layer @@ -222,10 +274,12 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): for description, tests, sections in cls.test_sets: vollog.debug(description) # There is a very high chance that the DTB will live in these very narrow segments, assuming we couldn't find them previously - hits = base_layer.scan(context, - PageMapScanner(tests = tests), - sections = sections, - progress_callback = progress_callback) + hits = base_layer.scan( + context, + PageMapScanner(tests=tests), + sections=sections, + progress_callback=progress_callback, + ) # Flatten the generator def sort_by_tests(x): @@ -236,13 +290,19 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): """Determines a pointer from a page_table""" max_ptr = 0 for index in range(0, len(page_table), ptr_size): - pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] + pointer = struct.unpack( + test.ptr_struct, page_table[index : index + ptr_size] + )[0] # Make sure the pointer is valid, ignore large pages which would require more calculation if pointer & 0x1 and not pointer & 0x80: - max_ptr = max(max_ptr, (pointer ^ (pointer & 0xfff)) % test.layer_type.maximum_address) + max_ptr = max( + max_ptr, + (pointer ^ (pointer & 0xFFF)) + % test.layer_type.maximum_address, + ) return max_ptr - hits = sorted(list(hits), key = sort_by_tests) + hits = sorted(list(hits), key=sort_by_tests) for test, page_map_offset in hits: # Turn the page tables into integers and find the largest one @@ -251,26 +311,45 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): max_pointer = get_max_pointer(page_table, test, ptr_size) if max_pointer <= base_layer.maximum_address: - vollog.debug(f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}") + vollog.debug( + f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}" + ) new_layer_name = context.layers.free_layer_name("IntelLayer") - config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name) - context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name + config_path = interfaces.configuration.path_join( + "IntelHelper", new_layer_name + ) context.config[ - interfaces.configuration.path_join(config_path, "page_map_offset")] = page_map_offset - layer = test.layer_type(context, - config_path = config_path, - name = new_layer_name, - metadata = {'os': 'Windows'}) + interfaces.configuration.path_join(config_path, "memory_layer") + ] = layer_name + context.config[ + interfaces.configuration.path_join( + config_path, "page_map_offset" + ) + ] = page_map_offset + layer = test.layer_type( + context, + config_path=config_path, + name=new_layer_name, + metadata={"os": "Windows"}, + ) break else: vollog.debug( - f"Max pointer for hit with test {test.__class__.__name__} not met: {hex(max_pointer)} > {hex(base_layer.maximum_address)}") + f"Max pointer for hit with test {test.__class__.__name__} not met: {hex(max_pointer)} > {hex(base_layer.maximum_address)}" + ) if layer is not None and config_path: break if layer is not None and config_path: - vollog.debug("DTB was found at: 0x{:0x}".format(context.config[interfaces.configuration.path_join( - config_path, "page_map_offset")])) + vollog.debug( + "DTB was found at: 0x{:0x}".format( + context.config[ + interfaces.configuration.path_join( + config_path, "page_map_offset" + ) + ] + ) + ) return layer @@ -278,31 +357,37 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): """Class to read swap_layers filenames from single-swap-layers, create the layers and populate the single-layers swap_layers.""" - exclusion_list = ['linux', 'mac'] + exclusion_list = ["linux", "mac"] - def __call__(self, - context: interfaces.context.ContextInterface, - config_path: str, - requirement: interfaces.configuration.RequirementInterface, - progress_callback: constants.ProgressCallback = None) -> None: + def __call__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.RequirementInterface, + progress_callback: constants.ProgressCallback = None, + ) -> None: """Finds translation layers that can have swap layers added.""" path_join = interfaces.configuration.path_join - self._translation_requirement = self.find_requirements(context, - config_path, - requirement, - requirements.TranslationLayerRequirement, - shortcut = False) + self._translation_requirement = self.find_requirements( + context, + config_path, + requirement, + requirements.TranslationLayerRequirement, + shortcut=False, + ) for trans_sub_config, trans_req in self._translation_requirement: if not isinstance(trans_req, requirements.TranslationLayerRequirement): # We need this so the type-checker knows we're a TranslationLayerRequirement continue - swap_sub_config, swap_req = self.find_swap_requirement(trans_sub_config, trans_req) + swap_sub_config, swap_req = self.find_swap_requirement( + trans_sub_config, trans_req + ) counter = 0 swap_config = interfaces.configuration.parent_path(swap_sub_config) if swap_req and swap_req.unsatisfied(context, swap_config): # See if any of them need constructing - for swap_location in self.config.get('single_swap_locations', []): + for swap_location in self.config.get("single_swap_locations", []): # Setup config locations/paths current_layer_name = swap_req.name + str(counter) current_layer_path = path_join(swap_sub_config, current_layer_name) @@ -314,32 +399,41 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): if swap_location: context.config[current_layer_path] = current_layer_name context.config[layer_loc_path] = swap_location - context.config[layer_class_path] = 'volatility3.framework.layers.physical.FileLayer' + context.config[ + layer_class_path + ] = "volatility3.framework.layers.physical.FileLayer" # Add the requirement - new_req = requirements.TranslationLayerRequirement(name = current_layer_name, - description = "Swap Layer", - optional = False) + new_req = requirements.TranslationLayerRequirement( + name=current_layer_name, + description="Swap Layer", + optional=False, + ) swap_req.add_requirement(new_req) - context.config[path_join(swap_sub_config, 'number_of_elements')] = counter + context.config[ + path_join(swap_sub_config, "number_of_elements") + ] = counter context.config[swap_sub_config] = True swap_req.construct(context, swap_config) @staticmethod - def find_swap_requirement(config: str, - requirement: requirements.TranslationLayerRequirement) \ - -> Tuple[str, Optional[requirements.LayerListRequirement]]: + def find_swap_requirement( + config: str, requirement: requirements.TranslationLayerRequirement + ) -> Tuple[str, Optional[requirements.LayerListRequirement]]: """Takes a Translation layer and returns its swap_layer requirement.""" swap_req = None for req_name in requirement.requirements: req = requirement.requirements[req_name] - if isinstance(req, requirements.LayerListRequirement) and req.name == 'swap_layers': + if ( + isinstance(req, requirements.LayerListRequirement) + and req.name == "swap_layers" + ): swap_req = req continue - swap_config = interfaces.configuration.path_join(config, 'swap_layers') + swap_config = interfaces.configuration.path_join(config, "swap_layers") return swap_config, swap_req @classmethod @@ -347,10 +441,11 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): """Returns the requirements of this plugin.""" return [ requirements.ListRequirement( - name = "single_swap_locations", - element_type = str, - min_elements = 0, - max_elements = 16, - description = "Specifies a list of swap layer URIs for use with single-location", - optional = True) + name="single_swap_locations", + element_type=str, + min_elements=0, + max_elements=16, + description="Specifies a list of swap layer URIs for use with single-location", + optional=True, + ) ] diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index cc4f05ae6..6b64b1cb9 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -24,23 +24,27 @@ class MultiRequirement(interfaces.configuration.RequirementInterface): so this is a concrete implementation. """ - def unsatisfied(self, context: interfaces.context.ContextInterface, - config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: + def unsatisfied( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: return self.unsatisfied_children(context, config_path) class BooleanRequirement(interfaces.configuration.SimpleTypeRequirement): """A requirement type that contains a boolean value.""" + # Note, this must be a separate class in order to differentiate between Booleans and other instance requirements class IntRequirement(interfaces.configuration.SimpleTypeRequirement): """A requirement type that contains a single integer.""" + instance_type: ClassVar[Type] = int class StringRequirement(interfaces.configuration.SimpleTypeRequirement): """A requirement type that contains a single unicode string.""" + # TODO: Maybe add string length limits? instance_type: ClassVar[Type] = str @@ -48,11 +52,13 @@ class StringRequirement(interfaces.configuration.SimpleTypeRequirement): class URIRequirement(StringRequirement): """A requirement type that contains a single unicode string that is a valid URI.""" + # TODO: Maybe a a check that to unsatisfied that the path really is a URL? class BytesRequirement(interfaces.configuration.SimpleTypeRequirement): """A requirement type that contains a byte string.""" + instance_type: ClassVar[Type] = bytes @@ -67,12 +73,14 @@ class ListRequirement(interfaces.configuration.RequirementInterface): and does not allow for a dynamic number of values. """ - def __init__(self, - element_type: Type[interfaces.configuration.SimpleTypes] = str, - max_elements: Optional[int] = 0, - min_elements: Optional[int] = None, - *args, - **kwargs) -> None: + def __init__( + self, + element_type: Type[interfaces.configuration.SimpleTypes] = str, + max_elements: Optional[int] = 0, + min_elements: Optional[int] = None, + *args, + **kwargs, + ) -> None: """Constructs the object. Args: @@ -82,24 +90,33 @@ class ListRequirement(interfaces.configuration.RequirementInterface): """ super().__init__(*args, **kwargs) if not issubclass(element_type, interfaces.configuration.BasicTypes): - raise TypeError("ListRequirements can only be populated with simple InstanceRequirements") + raise TypeError( + "ListRequirements can only be populated with simple InstanceRequirements" + ) self.element_type: Type = element_type self.min_elements: int = min_elements or 0 self.max_elements: Optional[int] = max_elements - def unsatisfied(self, context: interfaces.context.ContextInterface, - config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: + def unsatisfied( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: """Check the types on each of the returned values and their number and then call the element type's check for each one.""" config_path = interfaces.configuration.path_join(config_path, self.name) default = None value = self.config_value(context, config_path, default) if not value and self.min_elements > 0: - vollog.log(constants.LOGLEVEL_V, "ListRequirement Unsatisfied - ListRequirement has non-zero min_elements") + vollog.log( + constants.LOGLEVEL_V, + "ListRequirement Unsatisfied - ListRequirement has non-zero min_elements", + ) return {config_path: self} if value is None and not self.optional: # We need to differentiate between no value and an empty list - vollog.log(constants.LOGLEVEL_V, "ListRequirement Unsatisfied - Value was not specified") + vollog.log( + constants.LOGLEVEL_V, + "ListRequirement Unsatisfied - Value was not specified", + ) return {config_path: self} elif value is None: context.config[config_path] = [] @@ -107,13 +124,22 @@ class ListRequirement(interfaces.configuration.RequirementInterface): # TODO: Check this is the correct response for an error raise TypeError(f"Unexpected config value found: {repr(value)}") if not (self.min_elements <= len(value)): - vollog.log(constants.LOGLEVEL_V, "TypeError - Too few values provided to list option.") + vollog.log( + constants.LOGLEVEL_V, + "TypeError - Too few values provided to list option.", + ) return {config_path: self} if self.max_elements and not (len(value) < self.max_elements): - vollog.log(constants.LOGLEVEL_V, "TypeError - Too many values provided to list option.") + vollog.log( + constants.LOGLEVEL_V, + "TypeError - Too many values provided to list option.", + ) return {config_path: self} if not all([isinstance(element, self.element_type) for element in value]): - vollog.log(constants.LOGLEVEL_V, "TypeError - At least one element in the list is not of the correct type.") + vollog.log( + constants.LOGLEVEL_V, + "TypeError - At least one element in the list is not of the correct type.", + ) return {config_path: self} return {} @@ -128,37 +154,48 @@ class ChoiceRequirement(interfaces.configuration.RequirementInterface): choices: A list of possible string options that can be chosen from """ super().__init__(*args, **kwargs) - if not isinstance(choices, list) or any([not isinstance(choice, str) for choice in choices]): + if not isinstance(choices, list) or any( + [not isinstance(choice, str) for choice in choices] + ): raise TypeError("ChoiceRequirement takes a list of strings as choices") self.choices = choices - def unsatisfied(self, context: interfaces.context.ContextInterface, - config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: + def unsatisfied( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: """Validates the provided value to ensure it is one of the available choices.""" config_path = interfaces.configuration.path_join(config_path, self.name) value = self.config_value(context, config_path) if value not in self.choices: - vollog.log(constants.LOGLEVEL_V, "ValueError - Value is not within the set of available choices") + vollog.log( + constants.LOGLEVEL_V, + "ValueError - Value is not within the set of available choices", + ) return {config_path: self} return {} -class ComplexListRequirement(MultiRequirement, - interfaces.configuration.ConfigurableRequirementInterface, - metaclass = abc.ABCMeta): +class ComplexListRequirement( + MultiRequirement, + interfaces.configuration.ConfigurableRequirementInterface, + metaclass=abc.ABCMeta, +): """Allows a variable length list of requirements.""" - def unsatisfied(self, context: interfaces.context.ContextInterface, - config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: + def unsatisfied( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: """Validates the provided value to ensure it is one of the available choices.""" config_path = interfaces.configuration.path_join(config_path, self.name) ret_list = super().unsatisfied(context, config_path) if ret_list: return ret_list - if (self.config_value(context, config_path, None) is None - or self.config_value(context, interfaces.configuration.path_join(config_path, 'number_of_elements'))): + if self.config_value(context, config_path, None) is None or self.config_value( + context, + interfaces.configuration.path_join(config_path, "number_of_elements"), + ): return {config_path: self} return {} @@ -166,13 +203,17 @@ class ComplexListRequirement(MultiRequirement, def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # This is not optional for the stacker to run, so optional must be marked as False return [ - IntRequirement("number_of_elements", - description = "Determines how many layers are in this list", - optional = False) + IntRequirement( + "number_of_elements", + description="Determines how many layers are in this list", + optional=False, + ) ] @abc.abstractmethod - def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None: + def construct( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> None: """Method for constructing within the context any required elements from subrequirements.""" @@ -180,17 +221,22 @@ class ComplexListRequirement(MultiRequirement, def new_requirement(self, index) -> interfaces.configuration.RequirementInterface: """Builds a new requirement based on the specified index.""" - def build_configuration(self, context: interfaces.context.ContextInterface, config_path: str, - _: Any) -> interfaces.configuration.HierarchicalDict: + def build_configuration( + self, context: interfaces.context.ContextInterface, config_path: str, _: Any + ) -> interfaces.configuration.HierarchicalDict: result = interfaces.configuration.HierarchicalDict() - num_elem_config_path = interfaces.configuration.path_join(config_path, self.name, 'number_of_elements') + num_elem_config_path = interfaces.configuration.path_join( + config_path, self.name, "number_of_elements" + ) num_elements = context.config.get(num_elem_config_path, None) if num_elements is not None: result["number_of_elements"] = num_elements for i in range(num_elements): req = self.new_requirement(i) self.add_requirement(req) - value_path = interfaces.configuration.path_join(config_path, self.name, req.name) + value_path = interfaces.configuration.path_join( + config_path, self.name, req.name + ) value = context.config.get(value_path, None) if value is not None: result.splice(req.name, context.layers[value].build_configuration()) @@ -201,11 +247,15 @@ class ComplexListRequirement(MultiRequirement, class LayerListRequirement(ComplexListRequirement): """Allows a variable length list of layers that must exist.""" - def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None: + def construct( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> None: """Method for constructing within the context any required elements from subrequirements.""" new_config_path = interfaces.configuration.path_join(config_path, self.name) - num_layers_path = interfaces.configuration.path_join(new_config_path, "number_of_elements") + num_layers_path = interfaces.configuration.path_join( + new_config_path, "number_of_elements" + ) number_of_layers = context.config[num_layers_path] if not isinstance(number_of_layers, int): @@ -214,28 +264,36 @@ class LayerListRequirement(ComplexListRequirement): # Build all the layers that can be built for i in range(number_of_layers): layer_req = self.requirements.get(self.name + str(i), None) - if layer_req is not None and isinstance(layer_req, TranslationLayerRequirement): + if layer_req is not None and isinstance( + layer_req, TranslationLayerRequirement + ): layer_req.construct(context, new_config_path) def new_requirement(self, index) -> interfaces.configuration.RequirementInterface: """Constructs a new requirement based on the specified index.""" - return TranslationLayerRequirement(name = self.name + str(index), - description = "Layer for swap space", - optional = False) + return TranslationLayerRequirement( + name=self.name + str(index), + description="Layer for swap space", + optional=False, + ) -class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirementInterface, - interfaces.configuration.ConfigurableRequirementInterface): +class TranslationLayerRequirement( + interfaces.configuration.ConstructableRequirementInterface, + interfaces.configuration.ConfigurableRequirementInterface, +): """Class maintaining the limitations on what sort of translation layers are acceptable.""" - def __init__(self, - name: str, - description: str = None, - default: interfaces.configuration.ConfigSimpleType = None, - optional: bool = False, - oses: List = None, - architectures: List = None) -> None: + def __init__( + self, + name: str, + description: str = None, + default: interfaces.configuration.ConfigSimpleType = None, + optional: bool = False, + oses: List = None, + architectures: List = None, + ) -> None: """Constructs a Translation Layer Requirement. The configuration option's value will be the name of the layer once it exists in the store @@ -256,28 +314,46 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem self.architectures = architectures super().__init__(name, description, default, optional) - def unsatisfied(self, context: interfaces.context.ContextInterface, - config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: + def unsatisfied( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: """Validate that the value is a valid layer name and that the layer adheres to the requirements.""" config_path = interfaces.configuration.path_join(config_path, self.name) value = self.config_value(context, config_path, None) if isinstance(value, str): if value not in context.layers: - vollog.log(constants.LOGLEVEL_V, f"IndexError - Layer not found in memory space: {value}") + vollog.log( + constants.LOGLEVEL_V, + f"IndexError - Layer not found in memory space: {value}", + ) return {config_path: self} - if self.oses and context.layers[value].metadata.get('os', None) not in self.oses: - vollog.log(constants.LOGLEVEL_V, f"TypeError - Layer is not the required OS: {value}") + if ( + self.oses + and context.layers[value].metadata.get("os", None) not in self.oses + ): + vollog.log( + constants.LOGLEVEL_V, + f"TypeError - Layer is not the required OS: {value}", + ) return {config_path: self} - if (self.architectures - and context.layers[value].metadata.get('architecture', None) not in self.architectures): - vollog.log(constants.LOGLEVEL_V, f"TypeError - Layer is not the required Architecture: {value}") + if ( + self.architectures + and context.layers[value].metadata.get("architecture", None) + not in self.architectures + ): + vollog.log( + constants.LOGLEVEL_V, + f"TypeError - Layer is not the required Architecture: {value}", + ) return {config_path: self} return {} if value is not None: - vollog.log(constants.LOGLEVEL_V, - f"TypeError - Translation Layer Requirement only accepts string labels: {repr(value)}") + vollog.log( + constants.LOGLEVEL_V, + f"TypeError - Translation Layer Requirement only accepts string labels: {repr(value)}", + ) return {config_path: self} # TODO: check that the space in the context lives up to the requirements for arch/os etc @@ -285,10 +361,15 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem ### NOTE: This validate method has side effects (the dependencies can change)!!! self._validate_class(context, interfaces.configuration.parent_path(config_path)) - vollog.log(constants.LOGLEVEL_V, f"IndexError - No configuration provided: {config_path}") + vollog.log( + constants.LOGLEVEL_V, + f"IndexError - No configuration provided: {config_path}", + ) return {config_path: self} - def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None: + def construct( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> None: """Constructs the appropriate layer and adds it based on the class parameter.""" config_path = interfaces.configuration.path_join(config_path, self.name) @@ -303,8 +384,12 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if - not subreq.optional]): + [ + subreq.unsatisfied(context, config_path) + for subreq in self.requirements.values() + if not subreq.optional + ] + ): return None obj = self._construct_class(context, config_path, args) @@ -314,42 +399,57 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem # context.config[config_path] = obj.name return None - def build_configuration(self, context: interfaces.context.ContextInterface, _: str, - value: Any) -> interfaces.configuration.HierarchicalDict: + def build_configuration( + self, context: interfaces.context.ContextInterface, _: str, value: Any + ) -> interfaces.configuration.HierarchicalDict: """Builds the appropriate configuration for the specified requirement.""" return context.layers[value].build_configuration() -class SymbolTableRequirement(interfaces.configuration.ConstructableRequirementInterface, - interfaces.configuration.ConfigurableRequirementInterface): +class SymbolTableRequirement( + interfaces.configuration.ConstructableRequirementInterface, + interfaces.configuration.ConfigurableRequirementInterface, +): """Class maintaining the limitations on what sort of symbol spaces are acceptable.""" - def unsatisfied(self, context: interfaces.context.ContextInterface, - config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: + def unsatisfied( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: """Validate that the value is a valid within the symbol space of the provided context.""" config_path = interfaces.configuration.path_join(config_path, self.name) value = self.config_value(context, config_path, None) if not isinstance(value, str) and value is not None: - vollog.log(constants.LOGLEVEL_V, - f"TypeError - SymbolTableRequirement only accepts string labels: {repr(value)}") + vollog.log( + constants.LOGLEVEL_V, + f"TypeError - SymbolTableRequirement only accepts string labels: {repr(value)}", + ) return {config_path: self} if value and value in context.symbol_space: # This is an expected situation, so return rather than raise return {} elif value: - vollog.log(constants.LOGLEVEL_V, "IndexError - Value not present in the symbol space: {}".format(value - or "")) + vollog.log( + constants.LOGLEVEL_V, + "IndexError - Value not present in the symbol space: {}".format( + value or "" + ), + ) ### NOTE: This validate method has side effects (the dependencies can change)!!! self._validate_class(context, interfaces.configuration.parent_path(config_path)) - vollog.log(constants.LOGLEVEL_V, f"Symbol table requirement not yet fulfilled: {config_path}") + vollog.log( + constants.LOGLEVEL_V, + f"Symbol table requirement not yet fulfilled: {config_path}", + ) return {config_path: self} - def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None: + def construct( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> None: """Constructs the symbol space within the context based on the subrequirements.""" config_path = interfaces.configuration.path_join(config_path, self.name) @@ -359,14 +459,23 @@ class SymbolTableRequirement(interfaces.configuration.ConstructableRequirementIn args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if - not subreq.optional]): + [ + subreq.unsatisfied(context, config_path) + for subreq in self.requirements.values() + if not subreq.optional + ] + ): return None # Fill out the parameter for class creation - if not isinstance(self.requirements["class"], interfaces.configuration.ClassRequirement): - raise TypeError("Class requirement is not of type ClassRequirement: {}".format( - repr(self.requirements["class"]))) + if not isinstance( + self.requirements["class"], interfaces.configuration.ClassRequirement + ): + raise TypeError( + "Class requirement is not of type ClassRequirement: {}".format( + repr(self.requirements["class"]) + ) + ) cls = self.requirements["class"].cls if cls is None: return None @@ -380,23 +489,27 @@ class SymbolTableRequirement(interfaces.configuration.ConstructableRequirementIn context.symbol_space.append(obj) return None - def build_configuration(self, context: interfaces.context.ContextInterface, _: str, - value: Any) -> interfaces.configuration.HierarchicalDict: + def build_configuration( + self, context: interfaces.context.ContextInterface, _: str, value: Any + ) -> interfaces.configuration.HierarchicalDict: """Builds the appropriate configuration for the specified requirement.""" return context.symbol_space[value].build_configuration() class VersionRequirement(interfaces.configuration.RequirementInterface): - - def __init__(self, - name: str, - description: str = None, - default: bool = False, - optional: bool = False, - component: Type[interfaces.configuration.VersionableInterface] = None, - version: Optional[Tuple[int, ...]] = None) -> None: - super().__init__(name = name, description = description, default = default, optional = optional) + def __init__( + self, + name: str, + description: str = None, + default: bool = False, + optional: bool = False, + component: Type[interfaces.configuration.VersionableInterface] = None, + version: Optional[Tuple[int, ...]] = None, + ) -> None: + super().__init__( + name=name, description=description, default=default, optional=optional + ) if component is None: raise TypeError("Component cannot be None") self._component: Type[interfaces.configuration.VersionableInterface] = component @@ -404,17 +517,22 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): raise TypeError("Version cannot be None") self._version = version - def unsatisfied(self, context: interfaces.context.ContextInterface, - config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: + def unsatisfied( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} - context.config[interfaces.configuration.path_join(config_path, self.name)] = True + context.config[ + interfaces.configuration.path_join(config_path, self.name) + ] = True return {} @classmethod - def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]) -> bool: + def matches_required( + cls, required: Tuple[int, ...], version: Tuple[int, int, int] + ) -> bool: if len(required) > 0 and version[0] != required[0]: return False if len(required) > 1 and version[1] < required[1]: @@ -423,60 +541,87 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): class PluginRequirement(VersionRequirement): - - def __init__(self, - name: str, - description: str = None, - default: bool = False, - optional: bool = False, - plugin: Type[interfaces.plugins.PluginInterface] = None, - version: Optional[Tuple[int, ...]] = None) -> None: - super().__init__(name = name, - description = description, - default = default, - optional = optional, - component = plugin, - version = version) + def __init__( + self, + name: str, + description: str = None, + default: bool = False, + optional: bool = False, + plugin: Type[interfaces.plugins.PluginInterface] = None, + version: Optional[Tuple[int, ...]] = None, + ) -> None: + super().__init__( + name=name, + description=description, + default=default, + optional=optional, + component=plugin, + version=version, + ) -class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterface, - interfaces.configuration.ConfigurableRequirementInterface): - - def __init__(self, name: str, description: str = None, default: bool = False, - architectures: Optional[List[str]] = None, optional: bool = False): - super().__init__(name = name, description = description, default = default, optional = optional) - self.add_requirement(TranslationLayerRequirement(name = 'layer_name', architectures = architectures)) - self.add_requirement(SymbolTableRequirement(name = 'symbol_table_name')) +class ModuleRequirement( + interfaces.configuration.ConstructableRequirementInterface, + interfaces.configuration.ConfigurableRequirementInterface, +): + def __init__( + self, + name: str, + description: str = None, + default: bool = False, + architectures: Optional[List[str]] = None, + optional: bool = False, + ): + super().__init__( + name=name, description=description, default=default, optional=optional + ) + self.add_requirement( + TranslationLayerRequirement(name="layer_name", architectures=architectures) + ) + self.add_requirement(SymbolTableRequirement(name="symbol_table_name")) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - IntRequirement(name = 'offset'), + IntRequirement(name="offset"), ] - def unsatisfied(self, context: 'interfaces.context.ContextInterface', - config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: + def unsatisfied( + self, context: "interfaces.context.ContextInterface", config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: """Validate that the value is a valid module""" config_path = interfaces.configuration.path_join(config_path, self.name) value = self.config_value(context, config_path, None) if isinstance(value, str): if value not in context.modules: - vollog.log(constants.LOGLEVEL_V, f"IndexError - Module not found in context: {value}") + vollog.log( + constants.LOGLEVEL_V, + f"IndexError - Module not found in context: {value}", + ) return {config_path: self} return {} if value is not None: - vollog.log(constants.LOGLEVEL_V, - "TypeError - Module Requirement only accepts string labels: {}".format(repr(value))) + vollog.log( + constants.LOGLEVEL_V, + "TypeError - Module Requirement only accepts string labels: {}".format( + repr(value) + ), + ) return {config_path: self} result = {} for subreq in self._requirements: - req_unsatisfied = self._requirements[subreq].unsatisfied(context, config_path) + req_unsatisfied = self._requirements[subreq].unsatisfied( + context, config_path + ) if req_unsatisfied: result.update(req_unsatisfied) if not result: - vollog.log(constants.LOGLEVEL_V, f"IndexError - No configuration provided: {config_path}") + vollog.log( + constants.LOGLEVEL_V, + f"IndexError - No configuration provided: {config_path}", + ) result = {config_path: self} ### NOTE: This validate method has side effects (the dependencies can change)!!! @@ -485,7 +630,9 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa return result - def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None: + def construct( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> None: """Constructs the appropriate layer and adds it based on the class parameter.""" config_path = interfaces.configuration.path_join(config_path, self.name) @@ -499,8 +646,12 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if - not subreq.optional]): + [ + subreq.unsatisfied(context, config_path) + for subreq in self.requirements.values() + if not subreq.optional + ] + ): return None obj = self._construct_class(context, config_path, args) @@ -510,8 +661,9 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa # context.config[config_path] = obj.name return None - def build_configuration(self, context: 'interfaces.context.ContextInterface', _: str, - value: Any) -> interfaces.configuration.HierarchicalDict: + def build_configuration( + self, context: "interfaces.context.ContextInterface", _: str, value: Any + ) -> interfaces.configuration.HierarchicalDict: """Builds the appropriate configuration for the specified requirement.""" return context.modules[value].build_configuration() diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 95b365609..520fe0555 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -16,23 +16,27 @@ import volatility3.framework.constants.windows PLUGINS_PATH = [ os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "plugins")), - os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "plugins")) + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "plugins")), ] """Default list of paths to load plugins from (volatility3/plugins and volatility3/framework/plugins)""" SYMBOL_BASEPATHS = [ os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "symbols")), - os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "symbols")) + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "symbols")), ] """Default list of paths to load symbols from (volatility3/symbols and volatility3/framework/symbols)""" -ISF_EXTENSIONS = ['.json', '.json.xz', '.json.gz', '.json.bz2'] +ISF_EXTENSIONS = [".json", ".json.xz", ".json.gz", ".json.bz2"] """List of accepted extensions for ISF files""" -if hasattr(sys, 'frozen') and sys.frozen: +if hasattr(sys, "frozen") and sys.frozen: # Ensure we include the executable's directory as the base for plugins and symbols - PLUGINS_PATH = [os.path.abspath(os.path.join(os.path.dirname(sys.executable), 'plugins'))] + PLUGINS_PATH - SYMBOL_BASEPATHS = [os.path.abspath(os.path.join(os.path.dirname(sys.executable), 'symbols'))] + SYMBOL_BASEPATHS + PLUGINS_PATH = [ + os.path.abspath(os.path.join(os.path.dirname(sys.executable), "plugins")) + ] + PLUGINS_PATH + SYMBOL_BASEPATHS = [ + os.path.abspath(os.path.join(os.path.dirname(sys.executable), "symbols")) + ] + SYMBOL_BASEPATHS BANG = "!" """Constant used to delimit table names from type names when referring to a symbol""" @@ -45,10 +49,13 @@ VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature -PACKAGE_VERSION = ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + VERSION_SUFFIX +PACKAGE_VERSION = ( + ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + + VERSION_SUFFIX +) """The canonical version of the volatility3 package""" -AUTOMAGIC_CONFIG_PATH = 'automagic' +AUTOMAGIC_CONFIG_PATH = "automagic" """The root section within the context configuration for automagic values""" LOGLEVEL_V = 9 @@ -63,12 +70,14 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" -SQLITE_CACHE_PERIOD = '-3 days' +SQLITE_CACHE_PERIOD = "-3 days" """SQLite time modifier for how long each item is valid in the cache for""" -if sys.platform == 'win32': - CACHE_PATH = os.path.realpath(os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3")) -os.makedirs(CACHE_PATH, exist_ok = True) +if sys.platform == "win32": + CACHE_PATH = os.path.realpath( + os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") + ) +os.makedirs(CACHE_PATH, exist_ok=True) IDENTIFIERS_FILENAME = "identifier.cache" """Default location to record information about available identifiers""" @@ -81,12 +90,13 @@ BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" ProgressCallback = Optional[Callable[[float, str], None]] """Type information for ProgressCallback objects""" -OS_CATEGORIES = ['windows', 'mac', 'linux'] +OS_CATEGORIES = ["windows", "mac", "linux"] class Parallelism(enum.IntEnum): """An enumeration listing the different types of parallelism applied to volatility.""" + Off = 0 Threading = 1 Multiprocessing = 2 diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index c0f85593f..a1a1f5cf3 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -13,4 +13,4 @@ PAGE_SHIFT = 12 """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" # include/linux/sched.h -PF_KTHREAD = 0x00200000 # I'm a kernel thread +PF_KTHREAD = 0x00200000 # I'm a kernel thread diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 85a7d32b7..226a303dd 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -87,12 +87,14 @@ class Context(interfaces.context.ContextInterface): # ## Object Factory Functions - def object(self, - object_type: Union[str, interfaces.objects.Template], - layer_name: str, - offset: int, - native_layer_name: Optional[str] = None, - **arguments) -> interfaces.objects.ObjectInterface: + def object( + self, + object_type: Union[str, interfaces.objects.Template], + layer_name: str, + offset: int, + native_layer_name: Optional[str] = None, + **arguments, + ) -> interfaces.objects.ObjectInterface: """Object factory, takes a context, symbol, offset and optional layername. @@ -122,18 +124,24 @@ class Context(interfaces.context.ContextInterface): object_template = object_template.clone() object_template.update_vol(**arguments) - return object_template(context = self, - object_info = interfaces.objects.ObjectInformation(layer_name = layer_name, - offset = offset, - native_layer_name = native_layer_name, - size = object_template.size)) + return object_template( + context=self, + object_info=interfaces.objects.ObjectInformation( + layer_name=layer_name, + offset=offset, + native_layer_name=native_layer_name, + size=object_template.size, + ), + ) - def module(self, - module_name: str, - layer_name: str, - offset: int, - native_layer_name: Optional[str] = None, - size: Optional[int] = None) -> interfaces.context.ModuleInterface: + def module( + self, + module_name: str, + layer_name: str, + offset: int, + native_layer_name: Optional[str] = None, + size: Optional[int] = None, + ) -> interfaces.context.ModuleInterface: """Constructs a new os-independent module. Args: @@ -144,17 +152,21 @@ class Context(interfaces.context.ContextInterface): size: The size, in bytes, that the module occupies from offset location within the layer named layer_name """ if size: - return SizedModule.create(self, - module_name = module_name, - layer_name = layer_name, - offset = offset, - size = size, - native_layer_name = native_layer_name) - return Module.create(self, - module_name = module_name, - layer_name = layer_name, - offset = offset, - native_layer_name = native_layer_name) + return SizedModule.create( + self, + module_name=module_name, + layer_name=layer_name, + offset=offset, + size=size, + native_layer_name=native_layer_name, + ) + return Module.create( + self, + module_name=module_name, + layer_name=layer_name, + offset=offset, + native_layer_name=native_layer_name, + ) def get_module_wrapper(method: str) -> Callable: @@ -169,7 +181,13 @@ def get_module_wrapper(method: str) -> Callable: raise ValueError(f"Cannot reference another module when calling {method}") return getattr(self._context.symbol_space, method)(name) - for entry in ['__annotations__', '__doc__', '__module__', '__name__', '__qualname__']: + for entry in [ + "__annotations__", + "__doc__", + "__module__", + "__name__", + "__qualname__", + ]: proxy_interface = getattr(interfaces.context.ModuleInterface, method) if hasattr(proxy_interface, entry): setattr(wrapper, entry, getattr(proxy_interface, entry)) @@ -178,26 +196,27 @@ def get_module_wrapper(method: str) -> Callable: class Module(interfaces.context.ModuleInterface): - @classmethod - def create(cls, - context: interfaces.context.ContextInterface, - module_name: str, - layer_name: str, - offset: int, - **kwargs) -> 'Module': + def create( + cls, + context: interfaces.context.ContextInterface, + module_name: str, + layer_name: str, + offset: int, + **kwargs, + ) -> "Module": pathjoin = interfaces.configuration.path_join # Check if config_path is None free_module_name = context.modules.free_module_name(module_name) - config_path = kwargs.get('config_path', None) + config_path = kwargs.get("config_path", None) if config_path is None: - config_path = pathjoin('temporary', 'modules', free_module_name) + config_path = pathjoin("temporary", "modules", free_module_name) # Populate the configuration - context.config[pathjoin(config_path, 'layer_name')] = layer_name - context.config[pathjoin(config_path, 'offset')] = offset + context.config[pathjoin(config_path, "layer_name")] = layer_name + context.config[pathjoin(config_path, "offset")] = offset # This is important, since the module_name may be changed in case it is already in use - if 'symbol_table_name' not in kwargs: - kwargs['symbol_table_name'] = module_name + if "symbol_table_name" not in kwargs: + kwargs["symbol_table_name"] = module_name for arg in kwargs: context.config[pathjoin(config_path, arg)] = kwargs.get(arg, None) # Construct the object @@ -207,12 +226,14 @@ class Module(interfaces.context.ModuleInterface): # Add the module to the context modules collection return return_val - def object(self, - object_type: str, - offset: int = None, - native_layer_name: Optional[str] = None, - absolute: bool = False, - **kwargs) -> 'interfaces.objects.ObjectInterface': + def object( + self, + object_type: str, + offset: int = None, + native_layer_name: Optional[str] = None, + absolute: bool = False, + **kwargs, + ) -> "interfaces.objects.ObjectInterface": """Returns an object created using the symbol_table_name and layer_name of the Module. @@ -225,7 +246,9 @@ class Module(interfaces.context.ModuleInterface): if constants.BANG not in object_type: object_type = self.symbol_table_name + constants.BANG + object_type else: - raise ValueError("Cannot reference another module when constructing an object") + raise ValueError( + "Cannot reference another module when constructing an object" + ) if offset is None: raise TypeError("Offset must not be None for non-symbol objects") @@ -234,19 +257,23 @@ class Module(interfaces.context.ModuleInterface): offset += self._offset # Ensure we don't use a layer_name other than the module's, why would anyone do that? - if 'layer_name' in kwargs: - del kwargs['layer_name'] - return self._context.object(object_type = object_type, - layer_name = self._layer_name, - offset = offset, - native_layer_name = native_layer_name or self._native_layer_name, - **kwargs) + if "layer_name" in kwargs: + del kwargs["layer_name"] + return self._context.object( + object_type=object_type, + layer_name=self._layer_name, + offset=offset, + native_layer_name=native_layer_name or self._native_layer_name, + **kwargs, + ) - def object_from_symbol(self, - symbol_name: str, - native_layer_name: Optional[str] = None, - absolute: bool = False, - **kwargs) -> 'interfaces.objects.ObjectInterface': + def object_from_symbol( + self, + symbol_name: str, + native_layer_name: Optional[str] = None, + absolute: bool = False, + **kwargs, + ) -> "interfaces.objects.ObjectInterface": """Returns an object based on a specific symbol (containing type and offset information) and the layer_name of the Module. This will throw a ValueError if the symbol does not contain an associated type, or if @@ -261,7 +288,9 @@ class Module(interfaces.context.ModuleInterface): if constants.BANG not in symbol_name: symbol_name = self.symbol_table_name + constants.BANG + symbol_name else: - raise ValueError("Cannot reference another module when constructing an object") + raise ValueError( + "Cannot reference another module when constructing an object" + ) # Only set the offset if type is Symbol and we were given a name, not a template symbol_val = self._context.symbol_space.get_symbol(symbol_name) @@ -274,15 +303,17 @@ class Module(interfaces.context.ModuleInterface): raise TypeError(f"Symbol {symbol_val.name} has no associated type") # Ensure we don't use a layer_name other than the module's, why would anyone do that? - if 'layer_name' in kwargs: - del kwargs['layer_name'] + if "layer_name" in kwargs: + del kwargs["layer_name"] # Since type may be a template, we don't just call our own module method - return self._context.object(object_type = symbol_val.type, - layer_name = self._layer_name, - offset = offset, - native_layer_name = native_layer_name or self._native_layer_name, - **kwargs) + return self._context.object( + object_type=symbol_val.type, + layer_name=self._layer_name, + offset=offset, + native_layer_name=native_layer_name or self._native_layer_name, + **kwargs, + ) def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: """Returns the symbols within this module that live at the specified @@ -290,28 +321,30 @@ class Module(interfaces.context.ModuleInterface): if size < 0: raise ValueError("Size must be strictly non-negative") return list( - self._context.symbol_space.get_symbols_by_location(offset = offset - self._offset, - size = size, - table_name = self.symbol_table_name)) + self._context.symbol_space.get_symbols_by_location( + offset=offset - self._offset, + size=size, + table_name=self.symbol_table_name, + ) + ) @property def symbols(self): return self.context.symbol_space[self.symbol_table_name].symbols - get_symbol = get_module_wrapper('get_symbol') - get_type = get_module_wrapper('get_type') - get_enumeration = get_module_wrapper('get_enumeration') - has_symbol = get_module_wrapper('has_symbol') - has_type = get_module_wrapper('has_type') - has_enumeration = get_module_wrapper('has_enumeration') + get_symbol = get_module_wrapper("get_symbol") + get_type = get_module_wrapper("get_type") + get_enumeration = get_module_wrapper("get_enumeration") + has_symbol = get_module_wrapper("has_symbol") + has_type = get_module_wrapper("has_type") + has_enumeration = get_module_wrapper("has_enumeration") class SizedModule(Module): - @property def size(self) -> int: """Returns the size of the module (0 for unknown size)""" - size = self.config.get('size', 0) + size = self.config.get("size", 0) return size or 0 @property # type: ignore # FIXME: mypy #5107 @@ -326,8 +359,12 @@ class SizedModule(Module): layer = self._context.layers[self.layer_name] if not isinstance(layer, interfaces.layers.TranslationLayerInterface): raise TypeError("Hashing modules on non-TranslationLayers is not allowed") - return hashlib.md5(bytes(str(list(layer.mapping(self.offset, self.size, ignore_errors = True))), - 'utf-8')).hexdigest() + return hashlib.md5( + bytes( + str(list(layer.mapping(self.offset, self.size, ignore_errors=True))), + "utf-8", + ) + ).hexdigest() def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: """Returns the symbols within this module that live at the specified @@ -341,10 +378,12 @@ class ModuleCollection(interfaces.context.ModuleContainer): """Class to contain a collection of SizedModules and reason about their contents.""" - def __init__(self, modules: Optional[List[interfaces.context.ModuleInterface]] = None) -> None: + def __init__( + self, modules: Optional[List[interfaces.context.ModuleInterface]] = None + ) -> None: super().__init__(modules) - def deduplicate(self) -> 'ModuleCollection': + def deduplicate(self) -> "ModuleCollection": """Returns a new deduplicated ModuleCollection featuring no repeated modules (based on data hash) @@ -367,14 +406,17 @@ class ModuleCollection(interfaces.context.ModuleContainer): return prefix + str(count) @property - def modules(self) -> 'ModuleCollection': + def modules(self) -> "ModuleCollection": """A name indexed dictionary of modules using that name in this collection.""" vollog.warning( - "This method has been deprecated in favour of the ModuleCollection acting as a dictionary itself") + "This method has been deprecated in favour of the ModuleCollection acting as a dictionary itself" + ) return self - def get_module_symbols_by_absolute_location(self, offset: int, size: int = 0) -> Iterable[Tuple[str, List[str]]]: + def get_module_symbols_by_absolute_location( + self, offset: int, size: int = 0 + ) -> Iterable[Tuple[str, List[str]]]: """Returns a tuple of (module_name, list_of_symbol_names) for each module, where symbols live at the absolute offset in memory provided.""" @@ -383,16 +425,28 @@ class ModuleCollection(interfaces.context.ModuleContainer): for module_name in self._modules: module = self._modules[module_name] if isinstance(module, SizedModule): - if (offset <= module.offset + module.size) and (offset + size >= module.offset): - yield (module.name, module.get_symbols_by_absolute_location(offset, size)) + if (offset <= module.offset + module.size) and ( + offset + size >= module.offset + ): + yield ( + module.name, + module.get_symbols_by_absolute_location(offset, size), + ) class ConfigurableModule(Module, interfaces.configuration.ConfigurableInterface): - - def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str) -> None: - interfaces.configuration.ConfigurableInterface.__init__(self, context, config_path) - layer_name = self.config['layer_name'] - offset = self.config['offset'] - symbol_table_name = self.config['symbol_table_name'] - interfaces.configuration.ConfigurableInterface.__init__(self, context, config_path) - Module.__init__(self, context, name, layer_name, offset, symbol_table_name, layer_name) + def __init__( + self, context: interfaces.context.ContextInterface, config_path: str, name: str + ) -> None: + interfaces.configuration.ConfigurableInterface.__init__( + self, context, config_path + ) + layer_name = self.config["layer_name"] + offset = self.config["offset"] + symbol_table_name = self.config["symbol_table_name"] + interfaces.configuration.ConfigurableInterface.__init__( + self, context, config_path + ) + Module.__init__( + self, context, name, layer_name, offset, symbol_table_name, layer_name + ) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index a234a353a..f8701683b 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -30,7 +30,9 @@ class PluginRequirementException(VolatilityException): class SymbolError(VolatilityException): """Thrown when a symbol lookup has failed.""" - def __init__(self, symbol_name: Optional[str], table_name: Optional[str], *args) -> None: + def __init__( + self, symbol_name: Optional[str], table_name: Optional[str], *args + ) -> None: super().__init__(*args) self.symbol_name = symbol_name self.table_name = table_name @@ -63,7 +65,14 @@ class PagedInvalidAddressException(InvalidAddressException): that are invalid """ - def __init__(self, layer_name: str, invalid_address: int, invalid_bits: int, entry: int, *args) -> None: + def __init__( + self, + layer_name: str, + invalid_address: int, + invalid_bits: int, + entry: int, + *args, + ) -> None: super().__init__(layer_name, invalid_address, *args) self.invalid_bits = invalid_bits self.entry = entry @@ -77,8 +86,15 @@ class SwappedInvalidAddressException(PagedInvalidAddressException): the lookup that were invalid. """ - def __init__(self, layer_name: str, invalid_address: int, invalid_bits: int, entry: int, swap_offset: int, - *args) -> None: + def __init__( + self, + layer_name: str, + invalid_address: int, + invalid_bits: int, + entry: int, + swap_offset: int, + *args, + ) -> None: super().__init__(layer_name, invalid_address, invalid_bits, entry, *args) self.swap_offset = swap_offset @@ -88,14 +104,14 @@ class SymbolSpaceError(VolatilityException): class UnsatisfiedException(VolatilityException): - - def __init__(self, unsatisfied: Dict[str, interfaces.configuration.RequirementInterface]) -> None: + def __init__( + self, unsatisfied: Dict[str, interfaces.configuration.RequirementInterface] + ) -> None: super().__init__() self.unsatisfied = unsatisfied class MissingModuleException(VolatilityException): - def __init__(self, module: str, *args) -> None: super().__init__(*args) self.module = module @@ -109,4 +125,4 @@ class OfflineException(VolatilityException): self._url = url def __str__(self): - return f'Volatility 3 is offline: unable to access {self._url}' + return f"Volatility 3 is offline: unable to access {self._url}" diff --git a/volatility3/framework/interfaces/__init__.py b/volatility3/framework/interfaces/__init__.py index 05cf7d837..51d81d63a 100644 --- a/volatility3/framework/interfaces/__init__.py +++ b/volatility3/framework/interfaces/__init__.py @@ -12,5 +12,13 @@ components of volatility to write plugins. # Import the submodules we want people to be able to use without importing them themselves # This will also avoid namespace issues, because people can use interfaces.layers to # avoid clashing with the layers package -from volatility3.framework.interfaces import renderers, configuration, context, layers, objects, plugins, symbols, \ - automagic +from volatility3.framework.interfaces import ( + renderers, + configuration, + context, + layers, + objects, + plugins, + symbols, + automagic, +) diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index 4885645c3..fe1361b30 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -17,7 +17,9 @@ from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) -class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metaclass = ABCMeta): +class AutomagicInterface( + interfaces.configuration.ConfigurableInterface, metaclass=ABCMeta +): """Class that defines an automagic component that can help fulfill `Requirements` @@ -43,33 +45,52 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla exclusion_list = [] """A list of plugin categories (typically operating systems) which the plugin will not operate on""" - def __init__(self, context: interfaces.context.ContextInterface, config_path: str, *args, **kwargs) -> None: + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + *args, + **kwargs + ) -> None: super().__init__(context, config_path) for requirement in self.get_requirements(): - if not isinstance(requirement, (interfaces.configuration.SimpleTypeRequirement, - requirements.ChoiceRequirement, requirements.ListRequirement, - requirements.VersionRequirement)): + if not isinstance( + requirement, + ( + interfaces.configuration.SimpleTypeRequirement, + requirements.ChoiceRequirement, + requirements.ListRequirement, + requirements.VersionRequirement, + ), + ): raise TypeError( - "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement, ListRequirement or VersionRequirement") + "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement, ListRequirement or VersionRequirement" + ) - def __call__(self, - context: interfaces.context.ContextInterface, - config_path: str, - requirement: interfaces.configuration.RequirementInterface, - progress_callback: constants.ProgressCallback = None) -> Optional[List[Any]]: + def __call__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement: interfaces.configuration.RequirementInterface, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[List[Any]]: """Runs the automagic over the configurable.""" return [] # TODO: requirement_type can be made UnionType[Type[T], Tuple[Type[T], ...]] # once mypy properly supports Tuples in instance - def find_requirements(self, - context: interfaces.context.ContextInterface, - config_path: str, - requirement_root: interfaces.configuration.RequirementInterface, - requirement_type: Union[Tuple[Type[interfaces.configuration.RequirementInterface], ...], - Type[interfaces.configuration.RequirementInterface]], - shortcut: bool = True) -> List[Tuple[str, interfaces.configuration.RequirementInterface]]: + def find_requirements( + self, + context: interfaces.context.ContextInterface, + config_path: str, + requirement_root: interfaces.configuration.RequirementInterface, + requirement_type: Union[ + Tuple[Type[interfaces.configuration.RequirementInterface], ...], + Type[interfaces.configuration.RequirementInterface], + ], + shortcut: bool = True, + ) -> List[Tuple[str, interfaces.configuration.RequirementInterface]]: """Determines if there is actually an unfulfilled `Requirement` waiting. @@ -85,7 +106,9 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla Returns: A list of tuples containing the config_path, sub_config_path and requirement identifying the unsatisfied `Requirements` """ - sub_config_path = interfaces.configuration.path_join(config_path, requirement_root.name) + sub_config_path = interfaces.configuration.path_join( + config_path, requirement_root.name + ) results: List[Tuple[str, interfaces.configuration.RequirementInterface]] = [] recurse = not shortcut if isinstance(requirement_root, requirement_type): @@ -95,11 +118,13 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla recurse = True if recurse: for subreq in requirement_root.requirements.values(): - results += self.find_requirements(context, sub_config_path, subreq, requirement_type, shortcut) + results += self.find_requirements( + context, sub_config_path, subreq, requirement_type, shortcut + ) return results -class StackerLayerInterface(metaclass = ABCMeta): +class StackerLayerInterface(metaclass=ABCMeta): """Class that takes a lower layer and attempts to build on it. stack_order determines the order (from low to high) that stacking @@ -113,10 +138,12 @@ class StackerLayerInterface(metaclass = ABCMeta): """The list operating systems/first-level plugin hierarchy that should exclude this stacker""" @classmethod - def stack(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: """Method to determine whether this builder can operate on the named layer. If so, modify the context appropriately. @@ -135,4 +162,5 @@ class StackerLayerInterface(metaclass = ABCMeta): @classmethod def stacker_slow_warning(cls): vollog.warning( - "Reads to this layer are slow, it's recommended to use the layerwriter plugin once to produce a raw file") + "Reads to this layer are slow, it's recommended to use the layerwriter plugin once to produce a raw file" + ) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index e271ef6d4..3bb3cb019 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -23,7 +23,19 @@ import random import string import sys from abc import ABCMeta, abstractmethod -from typing import Any, ClassVar, Dict, Generator, Iterator, List, Optional, Type, Union, Tuple, Set +from typing import ( + Any, + ClassVar, + Dict, + Generator, + Iterator, + List, + Optional, + Type, + Union, + Tuple, + Set, +) from volatility3 import classproperty, framework from volatility3.framework import constants, interfaces @@ -68,9 +80,11 @@ class HierarchicalDict(collections.abc.Mapping): """The core of configuration data, it is a mapping class that stores keys within itself, and also stores lower hierarchies.""" - def __init__(self, - initial_dict: Dict[str, 'SimpleTypeRequirement'] = None, - separator: str = CONFIG_SEPARATOR) -> None: + def __init__( + self, + initial_dict: Dict[str, "SimpleTypeRequirement"] = None, + separator: str = CONFIG_SEPARATOR, + ) -> None: """ Args: initial_dict: A dictionary to populate the HierarchicalDict with initially @@ -80,7 +94,7 @@ class HierarchicalDict(collections.abc.Mapping): raise TypeError(f"Separator must be a one character string: {separator}") self._separator = separator self._data: Dict[str, ConfigSimpleType] = {} - self._subdict: Dict[str, 'HierarchicalDict'] = {} + self._subdict: Dict[str, "HierarchicalDict"] = {} if isinstance(initial_dict, str): initial_dict = json.loads(initial_dict) if isinstance(initial_dict, dict): @@ -88,7 +102,8 @@ class HierarchicalDict(collections.abc.Mapping): self[k] = v elif initial_dict is not None: raise TypeError( - f"Initial_dict must be a dictionary or JSON string containing a dictionary: {initial_dict}") + f"Initial_dict must be a dictionary or JSON string containing a dictionary: {initial_dict}" + ) def __eq__(self, other): """Define equality between HierarchicalDicts""" @@ -109,7 +124,7 @@ class HierarchicalDict(collections.abc.Mapping): """Returns the first division of a key based on the dict separator, or the full key if the separator is not present.""" if self.separator in key: - return key[:key.index(self.separator)] + return key[: key.index(self.separator)] else: return key @@ -117,8 +132,8 @@ class HierarchicalDict(collections.abc.Mapping): """Returns all but the first division of a key based on the dict separator, or None if the separator is not in the key.""" if self.separator in key: - return key[key.index(self.separator) + 1:] - return '' + return key[key.index(self.separator) + 1 :] + return "" def __iter__(self) -> Iterator[Any]: """Returns an iterator object that supports the iterator protocol.""" @@ -156,7 +171,9 @@ class HierarchicalDict(collections.abc.Mapping): def _setitem(self, key: str, value: Any, is_data: bool = True) -> None: """Set an item or appends a whole subtree at a key location.""" if self.separator in key: - subdict = self._subdict.get(self._key_head(key), HierarchicalDict(separator = self.separator)) + subdict = self._subdict.get( + self._key_head(key), HierarchicalDict(separator=self.separator) + ) subdict._setitem(self._key_tail(key), value, is_data) self._subdict[self._key_head(key)] = subdict else: @@ -166,7 +183,9 @@ class HierarchicalDict(collections.abc.Mapping): if not isinstance(value, HierarchicalDict): raise TypeError( "HierarchicalDicts can only store HierarchicalDicts within their structure: {}".format( - type(value))) + type(value) + ) + ) self._subdict[key] = value def _sanitize_value(self, value: Any) -> ConfigSimpleType: @@ -185,7 +204,9 @@ class HierarchicalDict(collections.abc.Mapping): for element in value: element_value = self._sanitize_value(element) if isinstance(element_value, list): - raise TypeError("Configuration list types cannot contain list types") + raise TypeError( + "Configuration list types cannot contain list types" + ) if element_value is not None: new_list.append(element_value) return new_list @@ -220,7 +241,7 @@ class HierarchicalDict(collections.abc.Mapping): """Returns the length of all items.""" return len(self._data) + sum([len(subdict) for subdict in self._subdict]) - def branch(self, key: str) -> 'HierarchicalDict': + def branch(self, key: str) -> "HierarchicalDict": """Returns the HierarchicalDict housed under the key. This differs from the data property, in that it is directed by the `key`, and all layers under that key are @@ -241,10 +262,12 @@ class HierarchicalDict(collections.abc.Mapping): else: return self._subdict[key] except KeyError: - self._setitem(key = key, value = HierarchicalDict(separator = self.separator), is_data = False) + self._setitem( + key=key, value=HierarchicalDict(separator=self.separator), is_data=False + ) return HierarchicalDict() - def splice(self, key: str, value: 'HierarchicalDict') -> None: + def splice(self, key: str, value: "HierarchicalDict") -> None: """Splices an existing HierarchicalDictionary under a specific key. This can be thought of as an inverse of :func:`branch`, although @@ -255,7 +278,9 @@ class HierarchicalDict(collections.abc.Mapping): raise TypeError("Splice requires a string key and HierarchicalDict value") self._setitem(key, value, False) - def merge(self, key: str, value: 'HierarchicalDict', overwrite: bool = False) -> None: + def merge( + self, key: str, value: "HierarchicalDict", overwrite: bool = False + ) -> None: """Acts similarly to splice, but maintains previous values. If overwrite is true, then entries in the new value are used over those that exist within key already @@ -274,7 +299,7 @@ class HierarchicalDict(collections.abc.Mapping): else: self[key + self._separator + item] = value[item] - def clone(self) -> 'HierarchicalDict': + def clone(self) -> "HierarchicalDict": """Duplicates the configuration, allowing changes without affecting the original. @@ -285,10 +310,12 @@ class HierarchicalDict(collections.abc.Mapping): def __str__(self) -> str: """Turns the Hierarchical dict into a string representation.""" - return json.dumps(dict([(key, self[key]) for key in sorted(self.generator())]), indent = 2) + return json.dumps( + dict([(key, self[key]) for key in sorted(self.generator())]), indent=2 + ) -class RequirementInterface(metaclass = ABCMeta): +class RequirementInterface(metaclass=ABCMeta): """Class that defines a requirement. A requirement is a means for plugins and other framework components to request specific configuration data. @@ -300,11 +327,13 @@ class RequirementInterface(metaclass = ABCMeta): as :class:`TranslationLayerRequirement`, :class:`SymbolTableRequirement` and :class:`ClassRequirement` """ - def __init__(self, - name: str, - description: str = None, - default: ConfigSimpleType = None, - optional: bool = False) -> None: + def __init__( + self, + name: str, + description: str = None, + default: ConfigSimpleType = None, + optional: bool = False, + ) -> None: """ Args: @@ -315,7 +344,9 @@ class RequirementInterface(metaclass = ABCMeta): """ super().__init__() if CONFIG_SEPARATOR in name: - raise ValueError(f"Name cannot contain the config-hierarchy divider ({CONFIG_SEPARATOR})") + raise ValueError( + f"Name cannot contain the config-hierarchy divider ({CONFIG_SEPARATOR})" + ) self._name = name self._description = description or "" self._default = default @@ -363,10 +394,12 @@ class RequirementInterface(metaclass = ABCMeta): """Sets the optional value for a requirement.""" self._optional = bool(value) - def config_value(self, - context: 'interfaces.context.ContextInterface', - config_path: str, - default: ConfigSimpleType = None) -> ConfigSimpleType: + def config_value( + self, + context: "interfaces.context.ContextInterface", + config_path: str, + default: ConfigSimpleType = None, + ) -> ConfigSimpleType: """Returns the value for this Requirement from its config path. Args: @@ -378,12 +411,12 @@ class RequirementInterface(metaclass = ABCMeta): # Child operations @property - def requirements(self) -> Dict[str, 'RequirementInterface']: + def requirements(self) -> Dict[str, "RequirementInterface"]: """Returns a dictionary of all the child requirements, indexed by name.""" return self._requirements.copy() - def add_requirement(self, requirement: 'RequirementInterface') -> None: + def add_requirement(self, requirement: "RequirementInterface") -> None: """Adds a child to the list of requirements. Args: @@ -391,7 +424,7 @@ class RequirementInterface(metaclass = ABCMeta): """ self._requirements[requirement.name] = requirement - def remove_requirement(self, requirement: 'RequirementInterface') -> None: + def remove_requirement(self, requirement: "RequirementInterface") -> None: """Removes a child from the list of requirements. Args: @@ -399,8 +432,9 @@ class RequirementInterface(metaclass = ABCMeta): """ del self._requirements[requirement.name] - def unsatisfied_children(self, context: 'interfaces.context.ContextInterface', - config_path: str) -> Dict[str, 'RequirementInterface']: + def unsatisfied_children( + self, context: "interfaces.context.ContextInterface", config_path: str + ) -> Dict[str, "RequirementInterface"]: """Method that will validate all child requirements. Args: @@ -413,14 +447,17 @@ class RequirementInterface(metaclass = ABCMeta): result = {} for requirement in self.requirements.values(): if not requirement.optional: - subresult = requirement.unsatisfied(context, path_join(config_path, self._name)) + subresult = requirement.unsatisfied( + context, path_join(config_path, self._name) + ) result.update(subresult) return result # Validation routines @abstractmethod - def unsatisfied(self, context: 'interfaces.context.ContextInterface', - config_path: str) -> Dict[str, 'RequirementInterface']: + def unsatisfied( + self, context: "interfaces.context.ContextInterface", config_path: str + ) -> Dict[str, "RequirementInterface"]: """Method to validate the value stored at config_path for the configuration object against a context. @@ -438,6 +475,7 @@ class RequirementInterface(metaclass = ABCMeta): class SimpleTypeRequirement(RequirementInterface): """Class to represent a single simple type (such as a boolean, a string, an integer or a series of bytes)""" + instance_type: ClassVar[Type] = bool def add_requirement(self, requirement: RequirementInterface): @@ -450,8 +488,9 @@ class SimpleTypeRequirement(RequirementInterface): children.""" raise TypeError("Instance Requirements cannot have subrequirements") - def unsatisfied(self, context: 'interfaces.context.ContextInterface', - config_path: str) -> Dict[str, RequirementInterface]: + def unsatisfied( + self, context: "interfaces.context.ContextInterface", config_path: str + ) -> Dict[str, RequirementInterface]: """Validates the instance requirement based upon its `instance_type`.""" config_path = path_join(config_path, self.name) @@ -460,8 +499,10 @@ class SimpleTypeRequirement(RequirementInterface): if not isinstance(value, self.instance_type): vollog.log( constants.LOGLEVEL_V, - "TypeError - {} requirements only accept {} type: {}".format(self.name, self.instance_type.__name__, - repr(value))) + "TypeError - {} requirements only accept {} type: {}".format( + self.name, self.instance_type.__name__, repr(value) + ), + ) return {config_path: self} return {} @@ -489,8 +530,9 @@ class ClassRequirement(RequirementInterface): class name.""" return self._cls - def unsatisfied(self, context: 'interfaces.context.ContextInterface', - config_path: str) -> Dict[str, RequirementInterface]: + def unsatisfied( + self, context: "interfaces.context.ContextInterface", config_path: str + ) -> Dict[str, RequirementInterface]: """Checks to see if a class can be recovered.""" config_path = path_join(config_path, self.name) @@ -499,8 +541,8 @@ class ClassRequirement(RequirementInterface): if value is not None and isinstance(value, str): if "." in value: # TODO: consider importing the prefix - module = sys.modules.get(value[:value.rindex(".")], None) - class_name = value[value.rindex(".") + 1:] + module = sys.modules.get(value[: value.rindex(".")], None) + class_name = value[value.rindex(".") + 1 :] if hasattr(module, class_name): self._cls = getattr(module, class_name) else: @@ -528,7 +570,9 @@ class ConstructableRequirementInterface(RequirementInterface): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.add_requirement(ClassRequirement("class", "Class of the constructable requirement")) + self.add_requirement( + ClassRequirement("class", "Class of the constructable requirement") + ) self._current_class_requirements: Set[Any] = set() def __eq__(self, other): @@ -537,7 +581,9 @@ class ConstructableRequirementInterface(RequirementInterface): return super().__eq__(other) @abstractmethod - def construct(self, context: 'interfaces.context.ContextInterface', config_path: str) -> None: + def construct( + self, context: "interfaces.context.ContextInterface", config_path: str + ) -> None: """Method for constructing within the context any required elements from subrequirements. @@ -546,7 +592,9 @@ class ConstructableRequirementInterface(RequirementInterface): config_path: The configuration path for the specific instance of this constructable """ - def _validate_class(self, context: 'interfaces.context.ContextInterface', config_path: str) -> None: + def _validate_class( + self, context: "interfaces.context.ContextInterface", config_path: str + ) -> None: """Method to check if the class Requirement is valid and if so populate the other requirements (but no need to validate, since we're invalid already) @@ -555,9 +603,11 @@ class ConstructableRequirementInterface(RequirementInterface): context: The context object containing the configuration data for the constructable config_path: The configuration path for the specific instance of this constructable """ - class_req = self.requirements['class'] + class_req = self.requirements["class"] subreq_config_path = path_join(config_path, self.name) - if not class_req.unsatisfied(context, subreq_config_path) and isinstance(class_req, ClassRequirement): + if not class_req.unsatisfied(context, subreq_config_path) and isinstance( + class_req, ClassRequirement + ): # We have a class, and since it's validated we can construct our requirements from it if issubclass(class_req.cls, ConfigurableInterface): # In case the class has changed, clear out the old requirements @@ -569,10 +619,12 @@ class ConstructableRequirementInterface(RequirementInterface): self._current_class_requirements.add(requirement.name) self.add_requirement(requirement) - def _construct_class(self, - context: 'interfaces.context.ContextInterface', - config_path: str, - requirement_dict: Dict[str, object] = None) -> Optional['interfaces.objects.ObjectInterface']: + def _construct_class( + self, + context: "interfaces.context.ContextInterface", + config_path: str, + requirement_dict: Dict[str, object] = None, + ) -> Optional["interfaces.objects.ObjectInterface"]: """Constructs the class, handing args and the subrequirements as parameters to __init__""" if self.requirements["class"].unsatisfied(context, config_path): @@ -605,16 +657,22 @@ class ConstructableRequirementInterface(RequirementInterface): class ConfigurableRequirementInterface(RequirementInterface): """Simple Abstract class to provide build_required_config.""" - def build_configuration(self, context: 'interfaces.context.ContextInterface', config_path: str, - value: Any) -> HierarchicalDict: + def build_configuration( + self, + context: "interfaces.context.ContextInterface", + config_path: str, + value: Any, + ) -> HierarchicalDict: """Proxies to a ConfigurableInterface if necessary.""" -class ConfigurableInterface(metaclass = ABCMeta): +class ConfigurableInterface(metaclass=ABCMeta): """Class to allow objects to have requirements and read configuration data from the context config tree.""" - def __init__(self, context: 'interfaces.context.ContextInterface', config_path: str) -> None: + def __init__( + self, context: "interfaces.context.ContextInterface", config_path: str + ) -> None: """Basic initializer that allows configurables to access their own config settings.""" super().__init__() @@ -623,7 +681,7 @@ class ConfigurableInterface(metaclass = ABCMeta): self._config_cache: Optional[HierarchicalDict] = None @property - def context(self) -> 'interfaces.context.ContextInterface': + def context(self) -> "interfaces.context.ContextInterface": """The context object that this configurable belongs to/configuration is stored in.""" return self._context @@ -660,11 +718,16 @@ class ConfigurableInterface(metaclass = ABCMeta): for req in self.get_requirements(): value = self.config.get(req.name, None) # Do not include the name of constructed classes - if value is not None and not isinstance(req, ConstructableRequirementInterface): + if value is not None and not isinstance( + req, ConstructableRequirementInterface + ): result[req.name] = value if isinstance(req, ConfigurableRequirementInterface): if value is not None: - result.splice(req.name, req.build_configuration(self.context, self.config_path, value)) + result.splice( + req.name, + req.build_configuration(self.context, self.config_path, value), + ) return result @classmethod @@ -674,8 +737,9 @@ class ConfigurableInterface(metaclass = ABCMeta): return [] @classmethod - def unsatisfied(cls, context: 'interfaces.context.ContextInterface', - config_path: str) -> Dict[str, RequirementInterface]: + def unsatisfied( + cls, context: "interfaces.context.ContextInterface", config_path: str + ) -> Dict[str, RequirementInterface]: """Returns a list of the names of all unsatisfied requirements. Since a satisfied set of requirements will return [], it can be used in tests as follows: @@ -694,7 +758,12 @@ class ConfigurableInterface(metaclass = ABCMeta): return result @classmethod - def make_subconfig(cls, context: 'interfaces.context.ContextInterface', base_config_path: str, **kwargs) -> str: + def make_subconfig( + cls, + context: "interfaces.context.ContextInterface", + base_config_path: str, + **kwargs, + ) -> str: """Convenience function to allow constructing a new randomly generated sub-configuration path, containing each element from kwargs. @@ -706,8 +775,10 @@ class ConfigurableInterface(metaclass = ABCMeta): Returns: str: The newly generated full configuration path """ - random_config_dict = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) - for _ in range(8)) + random_config_dict = "".join( + random.SystemRandom().choice(string.ascii_uppercase + string.digits) + for _ in range(8) + ) new_config_path = path_join(base_config_path, random_config_dict) # TODO: Check that the new_config_path is empty, although it's not critical if it's not since the values are merged in @@ -716,7 +787,9 @@ class ConfigurableInterface(metaclass = ABCMeta): # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a simple type for k, v in kwargs.items(): if not isinstance(v, (int, str, bool, float, bytes)): - raise TypeError("Config values passed to make_subconfig can only be simple types") + raise TypeError( + "Config values passed to make_subconfig can only be simple types" + ) context.config[path_join(new_config_path, k)] = v return new_config_path @@ -729,6 +802,7 @@ class VersionableInterface: All version number should use semantic versioning """ + _version: Tuple[int, int, int] = (0, 0, 0) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index b8470ae47..7e385746d 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -19,7 +19,7 @@ from typing import Optional, Union, Dict, List, Iterable from volatility3.framework import interfaces, exceptions -class ContextInterface(metaclass = ABCMeta): +class ContextInterface(metaclass=ABCMeta): """All context-like objects must adhere to the following interface. This interface is present to avoid import dependency cycles. @@ -32,12 +32,12 @@ class ContextInterface(metaclass = ABCMeta): @property @abstractmethod - def config(self) -> 'interfaces.configuration.HierarchicalDict': + def config(self) -> "interfaces.configuration.HierarchicalDict": """Returns the configuration object for this context.""" @property @abstractmethod - def symbol_space(self) -> 'interfaces.symbols.SymbolSpaceInterface': + def symbol_space(self) -> "interfaces.symbols.SymbolSpaceInterface": """Returns the symbol_space for the context. This object must support the :class:`~volatility3.framework.interfaces.symbols.SymbolSpaceInterface` @@ -47,11 +47,11 @@ class ContextInterface(metaclass = ABCMeta): @property @abstractmethod - def modules(self) -> 'ModuleContainer': + def modules(self) -> "ModuleContainer": """Returns the memory object for the context.""" raise NotImplementedError("ModuleContainer has not been implemented.") - def add_module(self, module: 'interfaces.context.ModuleInterface'): + def add_module(self, module: "interfaces.context.ModuleInterface"): """Adds a named module to the context. Args: @@ -65,11 +65,11 @@ class ContextInterface(metaclass = ABCMeta): @property @abstractmethod - def layers(self) -> 'interfaces.layers.LayerContainer': + def layers(self) -> "interfaces.layers.LayerContainer": """Returns the memory object for the context.""" raise NotImplementedError("LayerContainer has not been implemented.") - def add_layer(self, layer: 'interfaces.layers.DataLayerInterface'): + def add_layer(self, layer: "interfaces.layers.DataLayerInterface"): """Adds a named translation layer to the context memory. Args: @@ -80,12 +80,14 @@ class ContextInterface(metaclass = ABCMeta): # ## Object Factory Functions @abstractmethod - def object(self, - object_type: Union[str, 'interfaces.objects.Template'], - layer_name: str, - offset: int, - native_layer_name: str = None, - **arguments): + def object( + self, + object_type: Union[str, "interfaces.objects.Template"], + layer_name: str, + offset: int, + native_layer_name: str = None, + **arguments, + ): """Object factory, takes a context, symbol, offset and optional layer_name. @@ -102,7 +104,7 @@ class ContextInterface(metaclass = ABCMeta): A fully constructed object """ - def clone(self) -> 'ContextInterface': + def clone(self) -> "ContextInterface": """Produce a clone of the context (and configuration), allowing modifications to be made without affecting any mutable objects in the original. @@ -112,12 +114,14 @@ class ContextInterface(metaclass = ABCMeta): """ return copy.deepcopy(self) - def module(self, - module_name: str, - layer_name: str, - offset: int, - native_layer_name: Optional[str] = None, - size: Optional[int] = None) -> 'ModuleInterface': + def module( + self, + module_name: str, + layer_name: str, + offset: int, + native_layer_name: Optional[str] = None, + size: Optional[int] = None, + ) -> "ModuleInterface": """Create a module object. A module object is associated with a symbol table, and acts like a context, but offsets locations by a known value @@ -142,10 +146,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): This object is OS-independent. """ - def __init__(self, - context: ContextInterface, - config_path: str, - name: str) -> None: + def __init__(self, context: ContextInterface, config_path: str, name: str) -> None: """Constructs a new os-independent module. Args: @@ -158,35 +159,43 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): @property def _layer_name(self) -> str: - return self.config['layer_name'] + return self.config["layer_name"] @property def _offset(self) -> int: - return self.config['offset'] + return self.config["offset"] @property def _native_layer_name(self) -> str: - return self.config.get('native_layer_name', self._layer_name) + return self.config.get("native_layer_name", self._layer_name) @property def _symbol_table_name(self) -> str: - return self.config.get('symbol_table_name', self._module_name) + return self.config.get("symbol_table_name", self._module_name) - def build_configuration(self) -> 'interfaces.configuration.HierarchicalDict': + def build_configuration(self) -> "interfaces.configuration.HierarchicalDict": """Builds the configuration dictionary for this specific Module""" config = super().build_configuration() - config['offset'] = self.config['offset'] - subconfigs = {'symbol_table_name': self.context.symbol_space[self.symbol_table_name].build_configuration(), - 'layer_name': self.context.layers[self.layer_name].build_configuration()} + config["offset"] = self.config["offset"] + subconfigs = { + "symbol_table_name": self.context.symbol_space[ + self.symbol_table_name + ].build_configuration(), + "layer_name": self.context.layers[self.layer_name].build_configuration(), + } if self.layer_name != self._native_layer_name: - subconfigs['native_layer_name'] = self.context.layers[self._native_layer_name].build_configuration() + subconfigs["native_layer_name"] = self.context.layers[ + self._native_layer_name + ].build_configuration() for subconfig in subconfigs: for req in subconfigs[subconfig]: - config[interfaces.configuration.path_join(subconfig, req)] = subconfigs[subconfig][req] + config[interfaces.configuration.path_join(subconfig, req)] = subconfigs[ + subconfig + ][req] return config @@ -217,12 +226,14 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): return self._symbol_table_name @abstractmethod - def object(self, - object_type: str, - offset: int = None, - native_layer_name: Optional[str] = None, - absolute: bool = False, - **kwargs) -> 'interfaces.objects.ObjectInterface': + def object( + self, + object_type: str, + offset: int = None, + native_layer_name: Optional[str] = None, + absolute: bool = False, + **kwargs, + ) -> "interfaces.objects.ObjectInterface": """Returns an object created using the symbol_table_name and layer_name of the Module. @@ -237,11 +248,13 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): """ @abstractmethod - def object_from_symbol(self, - symbol_name: str, - native_layer_name: Optional[str] = None, - absolute: bool = False, - **kwargs) -> 'interfaces.objects.ObjectInterface': + def object_from_symbol( + self, + symbol_name: str, + native_layer_name: Optional[str] = None, + absolute: bool = False, + **kwargs, + ) -> "interfaces.objects.ObjectInterface": """Returns an object created using the symbol_table_name and layer_name of the Module. @@ -259,13 +272,13 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol = self.get_symbol(name) return self.offset + symbol.address - def get_type(self, name: str) -> 'interfaces.objects.Template': + def get_type(self, name: str) -> "interfaces.objects.Template": """Returns a type from the module's symbol table.""" - def get_symbol(self, name: str) -> 'interfaces.symbols.SymbolInterface': + def get_symbol(self, name: str) -> "interfaces.symbols.SymbolInterface": """Returns a symbol object from the module's symbol table.""" - def get_enumeration(self, name: str) -> 'interfaces.objects.Template': + def get_enumeration(self, name: str) -> "interfaces.objects.Template": """Returns an enumeration from the module's symbol table.""" def has_type(self, name: str) -> bool: @@ -306,7 +319,9 @@ class ModuleContainer(collections.abc.Mapping): module: the module to add to the list of modules (based on module.name) """ if module.name in self._modules: - raise exceptions.VolatilityException(f"Module already exists: {module.name}") + raise exceptions.VolatilityException( + f"Module already exists: {module.name}" + ) self._modules[module.name] = module def __delitem__(self, name: str) -> None: diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 7ff110c6e..a3c31a953 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -22,11 +22,13 @@ from volatility3.framework import constants, exceptions, interfaces vollog = logging.getLogger(__name__) -ProgressValue = Union['DummyProgress', multiprocessing.managers.ValueProxy] +ProgressValue = Union["DummyProgress", multiprocessing.managers.ValueProxy] IteratorValue = Tuple[List[Tuple[str, int, int]], int] -class ScannerInterface(interfaces.configuration.VersionableInterface, metaclass = ABCMeta): +class ScannerInterface( + interfaces.configuration.VersionableInterface, metaclass=ABCMeta +): """Class for layer scanners that return locations of particular values from within the data. @@ -52,6 +54,7 @@ class ScannerInterface(interfaces.configuration.VersionableInterface, metaclass in either their own class or the context. This will allow the scanner to be run in parallel against multiple blocks. """ + thread_safe = False _required_framework_version = (2, 0, 0) @@ -64,11 +67,11 @@ class ScannerInterface(interfaces.configuration.VersionableInterface, metaclass self._layer_name: Optional[str] = None @property - def context(self) -> Optional['interfaces.context.ContextInterface']: + def context(self) -> Optional["interfaces.context.ContextInterface"]: return self._context @context.setter - def context(self, ctx: 'interfaces.context.ContextInterface') -> None: + def context(self, ctx: "interfaces.context.ContextInterface") -> None: """Stores the context locally in case the scanner needs to access the layer.""" self._context = ctx @@ -94,20 +97,24 @@ class ScannerInterface(interfaces.configuration.VersionableInterface, metaclass """ -class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metaclass = ABCMeta): +class DataLayerInterface( + interfaces.configuration.ConfigurableInterface, metaclass=ABCMeta +): """A Layer that directly holds data (and does not translate it). This is effectively a leaf node in a layer tree. It directly accesses a data source and exposes it within volatility. """ - _direct_metadata: Mapping = {'architecture': 'Unknown', 'os': 'Unknown'} + _direct_metadata: Mapping = {"architecture": "Unknown", "os": "Unknown"} - def __init__(self, - context: 'interfaces.context.ContextInterface', - config_path: str, - name: str, - metadata: Optional[Dict[str, Any]] = None) -> None: + def __init__( + self, + context: "interfaces.context.ContextInterface", + config_path: str, + name: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: super().__init__(context, config_path) self._name = name self._metadata = metadata or {} @@ -199,11 +206,13 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla # ## General scanning methods - def scan(self, - context: interfaces.context.ContextInterface, - scanner: ScannerInterface, - progress_callback: constants.ProgressCallback = None, - sections: Iterable[Tuple[int, int]] = None) -> Iterable[Any]: + def scan( + self, + context: interfaces.context.ContextInterface, + scanner: ScannerInterface, + progress_callback: constants.ProgressCallback = None, + sections: Iterable[Tuple[int, int]] = None, + ) -> Iterable[Any]: """Scans a Translation layer by chunk. Note: this will skip missing/unmappable chunks of memory @@ -224,7 +233,9 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla scanner.layer_name = self.name if sections is None: - sections = [(self.minimum_address, self.maximum_address - self.minimum_address)] + sections = [ + (self.minimum_address, self.maximum_address - self.minimum_address) + ] sections = list(self._coalesce_sections(sections)) @@ -232,13 +243,18 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla progress: ProgressValue = DummyProgress() scan_iterator = functools.partial(self._scan_iterator, scanner, sections) scan_metric = self._scan_metric(scanner, sections) - if not scanner.thread_safe or constants.PARALLELISM == constants.Parallelism.Off: + if ( + not scanner.thread_safe + or constants.PARALLELISM == constants.Parallelism.Off + ): progress = DummyProgress() scan_chunk = functools.partial(self._scan_chunk, scanner, progress) for value in scan_iterator(): if progress_callback: - progress_callback(scan_metric(progress.value), - f"Scanning {self.name} using {scanner.__class__.__name__}") + progress_callback( + scan_metric(progress.value), + f"Scanning {self.name} using {scanner.__class__.__name__}", + ) yield from scan_chunk(value) else: progress = multiprocessing.Manager().Value("Q", 0) @@ -252,8 +268,10 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla while not result.ready(): if progress_callback: # Run the progress_callback - progress_callback(scan_metric(progress.value), - f"Scanning {self.name} using {scanner.__class__.__name__}") + progress_callback( + scan_metric(progress.value), + f"Scanning {self.name} using {scanner.__class__.__name__}", + ) # Ensures we don't burn CPU cycles going round in a ready waiting loop # without delaying the user too long between progress updates/results result.wait(0.1) @@ -262,10 +280,16 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla except Exception as e: # We don't care the kind of exception, so catch and report on everything, yielding nothing further vollog.debug(f"Scan Failure: {str(e)}") - vollog.log(constants.LOGLEVEL_VVV, - "\n".join(traceback.TracebackException.from_exception(e).format(chain = True))) + vollog.log( + constants.LOGLEVEL_VVV, + "\n".join( + traceback.TracebackException.from_exception(e).format(chain=True) + ), + ) - def _coalesce_sections(self, sections: Iterable[Tuple[int, int]]) -> Iterable[Tuple[int, int]]: + def _coalesce_sections( + self, sections: Iterable[Tuple[int, int]] + ) -> Iterable[Tuple[int, int]]: """Take a list of (start, length) sections and coalesce any adjacent sections.""" result: List[Tuple[int, int]] = [] @@ -283,7 +307,10 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla if first_start + first_length < self.minimum_address: result = result[1:] elif first_start < self.minimum_address: - result[0] = (self.minimum_address, (first_start + first_length) - self.minimum_address) + result[0] = ( + self.minimum_address, + (first_start + first_length) - self.minimum_address, + ) while result and result[-1] > (self.maximum_address, 0): last_start, last_length = result[-1] if last_start > self.maximum_address: @@ -292,8 +319,9 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla result[1] = (last_start, self.maximum_address - last_start) return result - def _scan_iterator(self, scanner: 'ScannerInterface', sections: Iterable[Tuple[int, - int]]) -> Iterable[IteratorValue]: + def _scan_iterator( + self, scanner: "ScannerInterface", sections: Iterable[Tuple[int, int]] + ) -> Iterable[IteratorValue]: """Iterator that indicates which blocks in the layer are to be read by for the scanning. @@ -303,7 +331,12 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla assumed to have no holes """ for section_start, section_length in sections: - offset, mapped_offset, length, layer_name = section_start, section_start, section_length, self.name + offset, mapped_offset, length, layer_name = ( + section_start, + section_start, + section_length, + self.name, + ) while length > 0: chunk_size = min(length, scanner.chunk_size + scanner.overlap) yield [(layer_name, mapped_offset, chunk_size)], offset + chunk_size @@ -315,16 +348,23 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla offset += chunk_size # We ignore the type due to the iterator_value, actually it only needs to match the output from _scan_iterator - def _scan_chunk(self, scanner: 'ScannerInterface', progress: 'ProgressValue', - iterator_value: IteratorValue) -> List[Any]: + def _scan_chunk( + self, + scanner: "ScannerInterface", + progress: "ProgressValue", + iterator_value: IteratorValue, + ) -> List[Any]: data_to_scan, chunk_end = iterator_value - data = b'' + data = b"" for layer_name, address, chunk_size in data_to_scan: try: data += self.context.layers[layer_name].read(address, chunk_size) except exceptions.InvalidAddressException: - vollog.debug("Invalid address in layer {} found scanning {} at address {:x}".format( - layer_name, self.name, address)) + vollog.debug( + "Invalid address in layer {} found scanning {} at address {:x}".format( + layer_name, self.name, address + ) + ) if len(data) > scanner.chunk_size + scanner.overlap: vollog.debug(f"Scan chunk too large: {hex(len(data))}") @@ -332,7 +372,9 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla progress.value = chunk_end return list(scanner(data, chunk_end - len(data))) - def _scan_metric(self, _scanner: 'ScannerInterface', sections: List[Tuple[int, int]]) -> Callable[[int], float]: + def _scan_metric( + self, _scanner: "ScannerInterface", sections: List[Tuple[int, int]] + ) -> Callable[[int], float]: if not sections: raise ValueError("Sections have no size, nothing to scan") @@ -357,11 +399,15 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla @property def metadata(self) -> Mapping: """Returns a ReadOnly copy of the metadata published by this layer.""" - maps = [self.context.layers[layer_name].metadata for layer_name in self.dependencies] - return interfaces.objects.ReadOnlyMapping(collections.ChainMap(self._metadata, self._direct_metadata, *maps)) + maps = [ + self.context.layers[layer_name].metadata for layer_name in self.dependencies + ] + return interfaces.objects.ReadOnlyMapping( + collections.ChainMap(self._metadata, self._direct_metadata, *maps) + ) -class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): +class TranslationLayerInterface(DataLayerInterface, metaclass=ABCMeta): """Provides a layer that translates or transforms another layer or layers. Translation layers always depend on another layer (typically @@ -370,10 +416,9 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): """ @abstractmethod - def mapping(self, - offset: int, - length: int, - ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + def mapping( + self, offset: int, length: int, ignore_errors: bool = False + ) -> Iterable[Tuple[int, int, int, int, str]]: """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer) mappings. @@ -390,7 +435,9 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): """Returns a list of layer names that this layer translates onto.""" return [] - def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes: + def _decode_data( + self, data: bytes, mapped_offset: int, offset: int, output_length: int + ) -> bytes: """Decodes any necessary data. Note, additional data may need to be read from the lower layer, such as lookup tables or similar. The data provided to this layer is purely that data which encompasses the requested data range. @@ -405,7 +452,9 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): The data to be read from the underlying layer.""" return data - def _encode_data(self, layer_name: str, mapped_offset: int, offset: int, value: bytes) -> bytes: + def _encode_data( + self, layer_name: str, mapped_offset: int, offset: int, value: bytes + ) -> bytes: """Encodes any necessary data. Args: @@ -420,28 +469,41 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): # ## Read/Write functions for mapped pages - @functools.lru_cache(maxsize = 512) + @functools.lru_cache(maxsize=512) def read(self, offset: int, length: int, pad: bool = False) -> bytes: """Reads an offset for length bytes and returns 'bytes' (not 'str') of length size.""" current_offset = offset - output: bytes = b'' - for (layer_offset, sublength, mapped_offset, mapped_length, layer) in self.mapping(offset, - length, - ignore_errors = pad): + output: bytes = b"" + for ( + layer_offset, + sublength, + mapped_offset, + mapped_length, + layer, + ) in self.mapping(offset, length, ignore_errors=pad): if not pad and layer_offset > current_offset: raise exceptions.InvalidAddressException( - self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}") + self.name, + current_offset, + f"Layer {self.name} cannot map offset: {current_offset}", + ) elif layer_offset > current_offset: output += b"\x00" * (layer_offset - current_offset) current_offset = layer_offset # The layer_offset can be less than the current_offset in non-linearly mapped layers # it does not suggest an overlap, but that the data is in an encoded block if mapped_length > 0: - unprocessed_data = self._context.layers.read(layer, mapped_offset, mapped_length, pad) - processed_data = self._decode_data(unprocessed_data, mapped_offset, layer_offset, sublength) + unprocessed_data = self._context.layers.read( + layer, mapped_offset, mapped_length, pad + ) + processed_data = self._decode_data( + unprocessed_data, mapped_offset, layer_offset, sublength + ) if len(processed_data) != sublength: - raise ValueError("ProcessedData length does not match expected length of chunk") + raise ValueError( + "ProcessedData length does not match expected length of chunk" + ) output += processed_data current_offset += sublength return output + (b"\x00" * (length - len(output))) @@ -451,21 +513,36 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): underlying mapping.""" current_offset = offset length = len(value) - for (layer_offset, sublength, mapped_offset, mapped_length, layer) in self.mapping(offset, length): + for ( + layer_offset, + sublength, + mapped_offset, + mapped_length, + layer, + ) in self.mapping(offset, length): if layer_offset > current_offset: raise exceptions.InvalidAddressException( - self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}") + self.name, + current_offset, + f"Layer {self.name} cannot map offset: {current_offset}", + ) - value_chunk = value[layer_offset - offset:layer_offset - offset + sublength] - new_data = self._encode_data(layer, mapped_offset, layer_offset, value_chunk) + value_chunk = value[ + layer_offset - offset : layer_offset - offset + sublength + ] + new_data = self._encode_data( + layer, mapped_offset, layer_offset, value_chunk + ) self._context.layers.write(layer, mapped_offset, new_data) current_offset += len(new_data) - def _scan_iterator(self, - scanner: 'ScannerInterface', - sections: Iterable[Tuple[int, int]], - linear: bool = False) -> Iterable[IteratorValue]: + def _scan_iterator( + self, + scanner: "ScannerInterface", + sections: Iterable[Tuple[int, int]], + linear: bool = False, + ) -> Iterable[IteratorValue]: """Iterator that indicates which blocks in the layer are to be read by for the scanning. @@ -483,7 +560,9 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): # For each section, find out which bits of its exists and where they map to # This is faster than cutting the entire space into scan_chunk sized blocks and then # finding out what exists (particularly if most of the space isn't mapped) - for mapped in self.mapping(section_start, section_length, ignore_errors = True): + for mapped in self.mapping( + section_start, section_length, ignore_errors=True + ): offset, sublength, mapped_offset, mapped_length, layer_name = mapped # Setup the variables for this block @@ -506,7 +585,10 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): # Halfway through a chunk, finish the chunk, then take more if chunk_position != chunk_start: - chunk_size = min(chunk_position - chunk_start, scanner.chunk_size + scanner.overlap) + chunk_size = min( + chunk_position - chunk_start, + scanner.chunk_size + scanner.overlap, + ) output += [(return_name, chunk_position + conversion, chunk_size)] chunk_start = chunk_position + chunk_size chunk_position = chunk_start @@ -519,8 +601,12 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): chunk_position = chunk_start # Take from chunk_position as far as the block can go, # or as much left of a scanner chunk as we can - chunk_size = min(block_end - chunk_position, - scanner.chunk_size + scanner.overlap - (chunk_position - chunk_start)) + chunk_size = min( + block_end - chunk_position, + scanner.chunk_size + + scanner.overlap + - (chunk_position - chunk_start), + ) output += [(return_name, chunk_position + conversion, chunk_size)] chunk_start = chunk_position + chunk_size chunk_position = chunk_start @@ -568,12 +654,20 @@ class LayerContainer(collections.abc.Mapping): layer: the layer to add to the list of layers (based on layer.name) """ if layer.name in self._layers: - raise exceptions.LayerException(layer.name, f"Layer already exists: {layer.name}") + raise exceptions.LayerException( + layer.name, f"Layer already exists: {layer.name}" + ) if isinstance(layer, TranslationLayerInterface): - missing_list = [sublayer for sublayer in layer.dependencies if sublayer not in self._layers] + missing_list = [ + sublayer + for sublayer in layer.dependencies + if sublayer not in self._layers + ] if missing_list: raise exceptions.LayerException( - layer.name, f"Layer {layer.name} has unmet dependencies: {', '.join(missing_list)}") + layer.name, + f"Layer {layer.name} has unmet dependencies: {', '.join(missing_list)}", + ) self._layers[layer.name] = layer def del_layer(self, name: str) -> None: @@ -585,11 +679,16 @@ class LayerContainer(collections.abc.Mapping): name: The name of the layer to delete """ for layer in self._layers: - depend_list = [superlayer for superlayer in self._layers if name in self._layers[layer].dependencies] + depend_list = [ + superlayer + for superlayer in self._layers + if name in self._layers[layer].dependencies + ] if depend_list: raise exceptions.LayerException( self._layers[layer].name, - f"Layer {self._layers[layer].name} is depended upon: {', '.join(depend_list)}") + f"Layer {self._layers[layer].name} is depended upon: {', '.join(depend_list)}", + ) self._layers[name].destroy() del self._layers[name] diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 1df13ca81..ab568b927 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -28,11 +28,13 @@ class ReadOnlyMapping(collections.abc.Mapping): def __getattr__(self, attr: str) -> Any: """Returns the item as an attribute.""" - if attr == '_dict': + if attr == "_dict": return super().__getattribute__(attr) if attr in self._dict: return self._dict[attr] - raise AttributeError(f"Object has no attribute: {self.__class__.__name__}.{attr}") + raise AttributeError( + f"Object has no attribute: {self.__class__.__name__}.{attr}" + ) def __getitem__(self, name: str) -> Any: """Returns the item requested.""" @@ -61,13 +63,15 @@ class ObjectInformation(ReadOnlyMapping): in a single place. These values are based on the :class:`ReadOnlyMapping` class, to prevent their modification. """ - def __init__(self, - layer_name: str, - offset: int, - member_name: Optional[str] = None, - parent: Optional['ObjectInterface'] = None, - native_layer_name: Optional[str] = None, - size: Optional[int] = None): + def __init__( + self, + layer_name: str, + offset: int, + member_name: Optional[str] = None, + parent: Optional["ObjectInterface"] = None, + native_layer_name: Optional[str] = None, + size: Optional[int] = None, + ): """Constructs a container for basic information about an object. Args: @@ -78,22 +82,29 @@ class ObjectInformation(ReadOnlyMapping): native_layer_name: If this object references other objects (such as a pointer), what layer those objects live in size: The size that the whole structure consumes in bytes """ - super().__init__({ - 'layer_name': layer_name, - 'offset': offset, - 'member_name': member_name, - 'parent': parent, - 'native_layer_name': native_layer_name or layer_name, - 'size': size - }) + super().__init__( + { + "layer_name": layer_name, + "offset": offset, + "member_name": member_name, + "parent": parent, + "native_layer_name": native_layer_name or layer_name, + "size": size, + } + ) -class ObjectInterface(metaclass = abc.ABCMeta): +class ObjectInterface(metaclass=abc.ABCMeta): """A base object required to be the ancestor of every object used in volatility.""" - def __init__(self, context: 'interfaces.context.ContextInterface', type_name: str, object_info: 'ObjectInformation', - **kwargs) -> None: + def __init__( + self, + context: "interfaces.context.ContextInterface", + type_name: str, + object_info: "ObjectInformation", + **kwargs, + ) -> None: """Constructs an Object adhering to the ObjectInterface. Args: @@ -116,7 +127,7 @@ class ObjectInterface(metaclass = abc.ABCMeta): mask = context.layers[object_info.layer_name].address_mask normalized_offset = object_info.offset & mask - vol_info_dict = {'type_name': type_name, 'offset': normalized_offset} + vol_info_dict = {"type_name": type_name, "offset": normalized_offset} self._vol = collections.ChainMap({}, vol_info_dict, object_info, kwargs) self._context = context @@ -143,13 +154,17 @@ class ObjectInterface(metaclass = abc.ABCMeta): KeyError: If the table_name is not valid within the object's context """ if constants.BANG not in self.vol.type_name: - raise ValueError(f"Unable to determine table for symbol: {self.vol.type_name}") - table_name = self.vol.type_name[:self.vol.type_name.index(constants.BANG)] + raise ValueError( + f"Unable to determine table for symbol: {self.vol.type_name}" + ) + table_name = self.vol.type_name[: self.vol.type_name.index(constants.BANG)] if table_name not in self._context.symbol_space: - raise KeyError(f"Symbol table not found in context's symbol_space for symbol: {self.vol.type_name}") + raise KeyError( + f"Symbol table not found in context's symbol_space for symbol: {self.vol.type_name}" + ) return table_name - def cast(self, new_type_name: str, **additional) -> 'ObjectInterface': + def cast(self, new_type_name: str, **additional) -> "ObjectInterface": """Returns a new object at the offset and from the layer that the current object inhabits. @@ -163,13 +178,15 @@ class ObjectInterface(metaclass = abc.ABCMeta): object_template = self._context.symbol_space.get_type(new_type_name) object_template = object_template.clone() object_template.update_vol(**additional) - object_info = ObjectInformation(layer_name = self.vol.layer_name, - offset = self.vol.offset, - member_name = self.vol.member_name, - parent = self.vol.parent, - native_layer_name = self.vol.native_layer_name, - size = object_template.size) - return object_template(context = self._context, object_info = object_info) + object_info = ObjectInformation( + layer_name=self.vol.layer_name, + offset=self.vol.offset, + member_name=self.vol.member_name, + parent=self.vol.parent, + native_layer_name=self.vol.native_layer_name, + size=object_template.size, + ) + return object_template(context=self._context, object_info=object_info) def has_member(self, member_name: str) -> bool: """Returns whether the object would contain a member called @@ -201,7 +218,7 @@ class ObjectInterface(metaclass = abc.ABCMeta): """ return all([self.has_valid_member(member_name) for member_name in member_names]) - class VolTemplateProxy(metaclass = abc.ABCMeta): + class VolTemplateProxy(metaclass=abc.ABCMeta): """A container for proxied methods that the ObjectTemplate of this object will call. This is primarily to keep methods together for easy organization/management, there is no significant need for it to be a @@ -214,41 +231,52 @@ class ObjectInterface(metaclass = abc.ABCMeta): to control how their templates respond without needing to write new templates for each and every potential object type. """ + _methods: List[str] = [] @classmethod @abc.abstractmethod - def size(cls, template: 'Template') -> int: + def size(cls, template: "Template") -> int: """Returns the size of the template object.""" @classmethod @abc.abstractmethod - def children(cls, template: 'Template') -> List['Template']: + def children(cls, template: "Template") -> List["Template"]: """Returns the children of the template.""" return [] @classmethod @abc.abstractmethod - def replace_child(cls, template: 'Template', old_child: 'Template', new_child: 'Template') -> None: + def replace_child( + cls, template: "Template", old_child: "Template", new_child: "Template" + ) -> None: """Substitutes the old_child for the new_child.""" - raise KeyError(f"Template does not contain any children to replace: {template.vol.type_name}") + raise KeyError( + f"Template does not contain any children to replace: {template.vol.type_name}" + ) @classmethod @abc.abstractmethod - def relative_child_offset(cls, template: 'Template', child: str) -> int: + def relative_child_offset(cls, template: "Template", child: str) -> int: """Returns the relative offset from the head of the parent data to the child member.""" - raise KeyError(f"Template does not contain any children: {template.vol.type_name}") + raise KeyError( + f"Template does not contain any children: {template.vol.type_name}" + ) @classmethod @abc.abstractmethod - def child_template(cls, template: 'Template', child: str) -> 'interfaces.objects.Template': + def child_template( + cls, template: "Template", child: str + ) -> "interfaces.objects.Template": """Returns the template of the child member from the parent.""" - raise KeyError(f"Template does not contain any children: {template.vol.type_name}") + raise KeyError( + f"Template does not contain any children: {template.vol.type_name}" + ) @classmethod @abc.abstractmethod - def has_member(cls, template: 'Template', member_name: str) -> bool: + def has_member(cls, template: "Template", member_name: str) -> bool: """Returns whether the object would contain a member called member_name.""" return False @@ -282,7 +310,9 @@ class Template: # Allow the updating of template arguments whilst still in template form super().__init__() empty_dict: Dict[str, Any] = {} - self._vol = collections.ChainMap(empty_dict, arguments, {'type_name': type_name}) + self._vol = collections.ChainMap( + empty_dict, arguments, {"type_name": type_name} + ) @property def vol(self) -> ReadOnlyMapping: @@ -292,7 +322,7 @@ class Template: return ReadOnlyMapping(self._vol) @property - def children(self) -> List['Template']: + def children(self) -> List["Template"]: """The children of this template (such as member types, sub-types and base-types where they are relevant). @@ -311,11 +341,11 @@ class Template: offset.""" @abc.abstractmethod - def child_template(self, child: str) -> 'interfaces.objects.Template': + def child_template(self, child: str) -> "interfaces.objects.Template": """Returns the `child` member template from its parent.""" @abc.abstractmethod - def replace_child(self, old_child: 'Template', new_child: 'Template') -> None: + def replace_child(self, old_child: "Template", new_child: "Template") -> None: """Replaces `old_child` with `new_child` in the list of children.""" @abc.abstractmethod @@ -323,7 +353,7 @@ class Template: """Returns whether the object would contain a member called `member_name`""" - def clone(self) -> 'Template': + def clone(self) -> "Template": """Returns a copy of the original Template as constructed (without `update_vol` additions having been made)""" clone = self.__class__(**self._vol.parents.new_child()) @@ -337,11 +367,16 @@ class Template: def __getattr__(self, attr: str) -> Any: """Exposes any other values stored in ._vol as attributes (for example, enumeration choices)""" - if attr != '_vol': + if attr != "_vol": if attr in self._vol: return self._vol[attr] - raise AttributeError(f"{self.__class__.__name__} object has no attribute {attr}") + raise AttributeError( + f"{self.__class__.__name__} object has no attribute {attr}" + ) - def __call__(self, context: 'interfaces.context.ContextInterface', - object_info: ObjectInformation) -> ObjectInterface: + def __call__( + self, + context: "interfaces.context.ContextInterface", + object_info: ObjectInformation, + ) -> ObjectInterface: """Constructs the object.""" diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 983232cf0..0de109c5e 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -64,7 +64,9 @@ class FileHandlerInterface(io.RawIOBase): if exc_type is None and exc_value is None and traceback is None: self.close() else: - vollog.warning(f"File {self._preferred_filename} could not be written: {str(exc_value)}") + vollog.warning( + f"File {self._preferred_filename} could not be written: {str(exc_value)}" + ) self.close() @@ -82,9 +84,11 @@ class FileHandlerInterface(io.RawIOBase): # The plugin runs and produces a TreeGrid output -class PluginInterface(interfaces.configuration.ConfigurableInterface, - interfaces.configuration.VersionableInterface, - metaclass = ABCMeta): +class PluginInterface( + interfaces.configuration.ConfigurableInterface, + interfaces.configuration.VersionableInterface, + metaclass=ABCMeta, +): """Class that defines the basic interface that all Plugins must maintain. The constructor must only take a `context` and `config_path`, so @@ -97,10 +101,12 @@ class PluginInterface(interfaces.configuration.ConfigurableInterface, _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" - def __init__(self, - context: interfaces.context.ContextInterface, - config_path: str, - progress_callback: constants.ProgressCallback = None) -> None: + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + progress_callback: constants.ProgressCallback = None, + ) -> None: """ Args: @@ -114,7 +120,9 @@ class PluginInterface(interfaces.configuration.ConfigurableInterface, # the validation doesn't need to be repeated over and over again by externals if self.unsatisfied(context, config_path): vollog.warning("Plugin failed validation") - raise exceptions.PluginRequirementException("The plugin configuration failed to validate") + raise exceptions.PluginRequirementException( + "The plugin configuration failed to validate" + ) # Populate any optional defaults for requirement in self.get_requirements(): if requirement.name not in self.config: diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index 9368009a9..b13de1834 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -12,14 +12,26 @@ suitable output. import datetime from abc import abstractmethod, ABCMeta from collections import abc -from typing import Any, Callable, ClassVar, Generator, List, NamedTuple, Optional, TypeVar, Type, Tuple, Union +from typing import ( + Any, + Callable, + ClassVar, + Generator, + List, + NamedTuple, + Optional, + TypeVar, + Type, + Tuple, + Union, +) -Column = NamedTuple('Column', [('name', str), ('type', Any)]) +Column = NamedTuple("Column", [("name", str), ("type", Any)]) RenderOption = Any -class Renderer(metaclass = ABCMeta): +class Renderer(metaclass=ABCMeta): """Class that defines the interface that all output renderers must support.""" @@ -32,12 +44,12 @@ class Renderer(metaclass = ABCMeta): """Returns a list of rendering options.""" @abstractmethod - def render(self, grid: 'TreeGrid') -> None: + def render(self, grid: "TreeGrid") -> None: """Takes a grid object and renders it based on the object's preferences.""" -class ColumnSortKey(metaclass = ABCMeta): +class ColumnSortKey(metaclass=ABCMeta): ascending: bool = True @abstractmethod @@ -46,14 +58,13 @@ class ColumnSortKey(metaclass = ABCMeta): function.""" -class TreeNode(abc.Sequence, metaclass = ABCMeta): - +class TreeNode(abc.Sequence, metaclass=ABCMeta): def __init__(self, path, treegrid, parent, values): """Initializes the TreeNode.""" @property @abstractmethod - def values(self) -> List['BaseTypes']: + def values(self) -> List["BaseTypes"]: """Returns the list of values from the particular node, based on column index.""" @@ -69,7 +80,7 @@ class TreeNode(abc.Sequence, metaclass = ABCMeta): @property @abstractmethod - def parent(self) -> Optional['TreeNode']: + def parent(self) -> Optional["TreeNode"]: """Returns the parent node of this node or None.""" @property @@ -94,9 +105,12 @@ class BaseAbsentValue(object): class Disassembly(object): """A class to indicate that the bytes provided should be disassembled (based on the architecture)""" - possible_architectures = ['intel', 'intel64', 'arm', 'arm64'] - def __init__(self, data: bytes, offset: int = 0, architecture: str = 'intel64') -> None: + possible_architectures = ["intel", "intel64", "arm", "arm64"] + + def __init__( + self, data: bytes, offset: int = 0, architecture: str = "intel64" + ) -> None: self.data = data self.architecture = None if architecture in self.possible_architectures: @@ -110,13 +124,20 @@ class Disassembly(object): # contain the types that the validator will accept (which would not include the base) _Type = TypeVar("_Type") -BaseTypes = Union[Type[int], Type[str], Type[float], Type[bytes], Type[datetime.datetime], Type[BaseAbsentValue], - Type[Disassembly]] +BaseTypes = Union[ + Type[int], + Type[str], + Type[float], + Type[bytes], + Type[datetime.datetime], + Type[BaseAbsentValue], + Type[Disassembly], +] ColumnsType = List[Tuple[str, BaseTypes]] VisitorSignature = Callable[[TreeNode, _Type], _Type] -class TreeGrid(object, metaclass = ABCMeta): +class TreeGrid(object, metaclass=ABCMeta): """Class providing the interface for a TreeGrid (which contains TreeNodes) The structure of a TreeGrid is designed to maintain the structure of the tree in a single object. @@ -129,7 +150,14 @@ class TreeGrid(object, metaclass = ABCMeta): and to create cycles. """ - base_types: ClassVar[Tuple] = (int, str, float, bytes, datetime.datetime, Disassembly) + base_types: ClassVar[Tuple] = ( + int, + str, + float, + bytes, + datetime.datetime, + Disassembly, + ) def __init__(self, columns: ColumnsType, generator: Generator) -> None: """Constructs a TreeGrid object using a specific set of columns. @@ -149,10 +177,12 @@ class TreeGrid(object, metaclass = ABCMeta): """Method used to sanitize column names for TreeNodes.""" @abstractmethod - def populate(self, - function: VisitorSignature = None, - initial_accumulator: Any = None, - fail_on_errors: bool = True) -> Optional[Exception]: + def populate( + self, + function: VisitorSignature = None, + initial_accumulator: Any = None, + fail_on_errors: bool = True, + ) -> Optional[Exception]: """Populates the tree by consuming the TreeGrid's construction generator Func is called on every node, so can be used to create output on demand. @@ -196,11 +226,13 @@ class TreeGrid(object, metaclass = ABCMeta): return node.path_depth @abstractmethod - def visit(self, - node: Optional[TreeNode], - function: VisitorSignature, - initial_accumulator: _Type, - sort_key: ColumnSortKey = None) -> None: + def visit( + self, + node: Optional[TreeNode], + function: VisitorSignature, + initial_accumulator: _Type, + sort_key: ColumnSortKey = None, + ) -> None: """Visits all the nodes in a tree, calling function on each one. function should have the signature function(node, accumulator) and return new_accumulator diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index 1ad30cfdf..b645f5cd1 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -16,11 +16,13 @@ from volatility3.framework.interfaces.configuration import RequirementInterface class SymbolInterface: """Contains information about a named location in a program's memory.""" - def __init__(self, - name: str, - address: int, - type: Optional[objects.Template] = None, - constant_data: Optional[bytes] = None) -> None: + def __init__( + self, + name: str, + address: int, + type: Optional[objects.Template] = None, + constant_data: Optional[bytes] = None, + ) -> None: """ Args: @@ -31,7 +33,9 @@ class SymbolInterface: """ self._name = name if constants.BANG in self._name: - raise ValueError(f"Symbol names cannot contain the symbol differentiator ({constants.BANG})") + raise ValueError( + f"Symbol names cannot contain the symbol differentiator ({constants.BANG})" + ) # Scope can be added at a later date self._location = None @@ -50,7 +54,7 @@ class SymbolInterface: # Objects and ObjectTemplates should *always* get a type_name when they're constructed, so allow the IndexError if self.type is None: return None - return self.type.vol['type_name'] + return self.type.vol["type_name"] @property def type(self) -> Optional[objects.Template]: @@ -78,11 +82,13 @@ class BaseSymbolTableInterface: Note: table_mapping is a rarely used feature (since symbol tables are typically self-contained) """ - def __init__(self, - name: str, - native_types: 'NativeTableInterface', - table_mapping: Optional[Dict[str, str]] = None, - class_types: Optional[Mapping[str, Type[objects.ObjectInterface]]] = None) -> None: + def __init__( + self, + name: str, + native_types: "NativeTableInterface", + table_mapping: Optional[Dict[str, str]] = None, + class_types: Optional[Mapping[str, Type[objects.ObjectInterface]]] = None, + ) -> None: """ Args: @@ -110,44 +116,54 @@ class BaseSymbolTableInterface: If the symbol isn't found, it raises a SymbolError exception """ - raise NotImplementedError("Abstract property get_symbol not implemented by subclass.") + raise NotImplementedError( + "Abstract property get_symbol not implemented by subclass." + ) @property def symbols(self) -> Iterable[str]: """Returns an iterator of the Symbol names.""" - raise NotImplementedError("Abstract property symbols not implemented by subclass.") + raise NotImplementedError( + "Abstract property symbols not implemented by subclass." + ) # ## Required Type functions @property def types(self) -> Iterable[str]: """Returns an iterator of the Symbol type names.""" - raise NotImplementedError("Abstract property types not implemented by subclass.") + raise NotImplementedError( + "Abstract property types not implemented by subclass." + ) def get_type(self, name: str) -> objects.Template: """Resolves a symbol name into an object template. If the symbol isn't found it raises a SymbolError exception """ - raise NotImplementedError("Abstract method get_type not implemented by subclass.") + raise NotImplementedError( + "Abstract method get_type not implemented by subclass." + ) # ## Required Symbol enumeration functions @property def enumerations(self) -> Iterable[Any]: """Returns an iterator of the Enumeration names.""" - raise NotImplementedError("Abstract property enumerations not implemented by subclass.") + raise NotImplementedError( + "Abstract property enumerations not implemented by subclass." + ) # ## Native Type Handler @property - def natives(self) -> 'NativeTableInterface': + def natives(self) -> "NativeTableInterface": """Returns None or a NativeTable for handling space specific native types.""" return self._native_types @natives.setter - def natives(self, value: 'NativeTableInterface') -> None: + def natives(self, value: "NativeTableInterface") -> None: """Checks the natives value and then applies it internally. WARNING: This allows changing the underlying size of all the other types referenced in the SymbolTable @@ -167,7 +183,9 @@ class BaseSymbolTableInterface: """ raise NotImplementedError("Abstract method set_type_class not implemented yet.") - def optional_set_type_class(self, name: str, clazz: Type[objects.ObjectInterface]) -> bool: + def optional_set_type_class( + self, name: str, clazz: Type[objects.ObjectInterface] + ) -> bool: """Calls the set_type_class function but does not throw an exception. Returns whether setting the type class was successful. Args: @@ -176,7 +194,7 @@ class BaseSymbolTableInterface: """ try: self.set_type_class(name, clazz) - + return True except ValueError: return False @@ -206,8 +224,10 @@ class BaseSymbolTableInterface: # This allows for searching with and without the table name (in case multiple tables contain # the same symbol name and we've not specifically been told which one) symbol = self.get_symbol(symbol_name) - if symbol.type_name is not None and (symbol.type_name == type_name or - (symbol.type_name.endswith(constants.BANG + type_name))): + if symbol.type_name is not None and ( + symbol.type_name == type_name + or (symbol.type_name.endswith(constants.BANG + type_name)) + ): yield symbol.name def get_symbols_by_location(self, offset: int, size: int = 0) -> Iterable[str]: @@ -216,11 +236,15 @@ class BaseSymbolTableInterface: if size < 0: raise ValueError("Size must be strictly non-negative") if not self._sort_symbols: - self._sort_symbols = sorted([(self.get_symbol(sn).address, sn) for sn in self.symbols]) + self._sort_symbols = sorted( + [(self.get_symbol(sn).address, sn) for sn in self.symbols] + ) sort_symbols = self._sort_symbols result = bisect.bisect_left(sort_symbols, (offset, "")) - while result < len(sort_symbols) and \ - (sort_symbols[result][0] >= offset and sort_symbols[result][0] <= offset + size): + while result < len(sort_symbols) and ( + sort_symbols[result][0] >= offset + and sort_symbols[result][0] <= offset + size + ): yield sort_symbols[result][1] result += 1 @@ -247,7 +271,9 @@ class SymbolSpaceInterface(collections.abc.Mapping): """Returns all symbols based on the type of the symbol.""" @abstractmethod - def get_symbols_by_location(self, offset: int, size: int = 0, table_name: Optional[str] = None) -> Iterable[str]: + def get_symbols_by_location( + self, offset: int, size: int = 0, table_name: Optional[str] = None + ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" @abstractmethod @@ -281,17 +307,21 @@ class SymbolSpaceInterface(collections.abc.Mapping): """Adds a symbol_list to the end of the space.""" -class SymbolTableInterface(BaseSymbolTableInterface, configuration.ConfigurableInterface, ABC): +class SymbolTableInterface( + BaseSymbolTableInterface, configuration.ConfigurableInterface, ABC +): """Handles a table of symbols.""" # FIXME: native_types and table_mapping aren't recorded in the configuration - def __init__(self, - context: 'interfaces.context.ContextInterface', - config_path: str, - name: str, - native_types: 'NativeTableInterface', - table_mapping: Optional[Dict[str, str]] = None, - class_types: Optional[Mapping[str, Type[objects.ObjectInterface]]] = None) -> None: + def __init__( + self, + context: "interfaces.context.ContextInterface", + config_path: str, + name: str, + native_types: "NativeTableInterface", + table_mapping: Optional[Dict[str, str]] = None, + class_types: Optional[Mapping[str, Type[objects.ObjectInterface]]] = None, + ) -> None: """Instantiates an SymbolTable based on an IntermediateSymbolFormat JSON file. This is validated against the appropriate schema. @@ -305,9 +335,11 @@ class SymbolTableInterface(BaseSymbolTableInterface, configuration.ConfigurableI class_types: A dictionary of type names and classes that override StructType when they are instantiated """ configuration.ConfigurableInterface.__init__(self, context, config_path) - BaseSymbolTableInterface.__init__(self, name, native_types, table_mapping, class_types = class_types) + BaseSymbolTableInterface.__init__( + self, name, native_types, table_mapping, class_types=class_types + ) - def build_configuration(self) -> 'configuration.HierarchicalDict': + def build_configuration(self) -> "configuration.HierarchicalDict": config = super().build_configuration() # Symbol Tables are constructable, and therefore require a class configuration variable @@ -317,9 +349,13 @@ class SymbolTableInterface(BaseSymbolTableInterface, configuration.ConfigurableI @classmethod def get_requirements(cls) -> List[RequirementInterface]: return super().get_requirements() + [ - requirements.IntRequirement(name = 'symbol_mask', description = 'Address mask for symbols', optional = True, - default = 0), - ] + requirements.IntRequirement( + name="symbol_mask", + description="Address mask for symbols", + optional=True, + default=0, + ), + ] class NativeTableInterface(BaseSymbolTableInterface): @@ -333,7 +369,9 @@ class NativeTableInterface(BaseSymbolTableInterface): return [] def get_enumeration(self, name: str) -> objects.Template: - raise exceptions.SymbolError(name, self.name, "NativeTables never hold enumerations") + raise exceptions.SymbolError( + name, self.name, "NativeTables never hold enumerations" + ) @property def enumerations(self) -> Iterable[str]: diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index f31737232..b12fdd01c 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -37,13 +37,19 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer): @classmethod def _check_header(cls, layer: interfaces.layers.DataLayerInterface): header_structure = " None: base_layer = self.context.layers[self._base_layer] @@ -52,24 +58,38 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer): avml_header_structure = " Tuple[ - List[Tuple[int, int, int, int, bool]], int]: + def _read_snappy_frames( + self, data: bytes, expected_length: int + ) -> Tuple[List[Tuple[int, int, int, int, bool]], int]: """ Reads a framed-format snappy stream @@ -84,41 +104,62 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer): decompressed_len = 0 offset = 0 crc_len = 4 - frame_header_struct = '> 8 - if frame_type == 0xff: - if data[offset + frame_header_len:offset + frame_header_len + frame_size] != b'sNaPpY': + frame_header = data[offset : offset + frame_header_len] + frame_header_val = struct.unpack("> 8 + if frame_type == 0xFF: + if ( + data[ + offset + + frame_header_len : offset + + frame_header_len + + frame_size + ] + != b"sNaPpY" + ): raise ValueError(f"Snappy header missing at offset: {offset}") elif frame_type in [0x00, 0x01]: # CRC + (Un)compressed data mapped_start = offset + frame_header_len # frame_crc = data[mapped_start: mapped_start + crc_len] - frame_data = data[mapped_start + crc_len: mapped_start + frame_size] + frame_data = data[ + mapped_start + crc_len : mapped_start + frame_size + ] if frame_type == 0x00: # Compressed data frame_data = snappy.decompress(frame_data) # TODO: Verify CRC - segments.append((decompressed_len, mapped_start + crc_len, len(frame_data), frame_size - crc_len, - frame_type == 0x00)) + segments.append( + ( + decompressed_len, + mapped_start + crc_len, + len(frame_data), + frame_size - crc_len, + frame_type == 0x00, + ) + ) decompressed_len += len(frame_data) elif frame_type in range(0x2, 0x80): # Unskippable - raise exceptions.LayerException(f"Unskippable chunk of type {frame_type} found: {offset}") + raise exceptions.LayerException( + f"Unskippable chunk of type {frame_type} found: {offset}" + ) offset += frame_header_len + frame_size return segments, offset - def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes: + def _decode_data( + self, data: bytes, mapped_offset: int, offset: int, output_length: int + ) -> bytes: start_offset, _, _, _ = self._find_segment(offset) if self._compressed[mapped_offset]: decoded_data = snappy.decompress(data) else: decoded_data = data - decoded_data = decoded_data[offset - start_offset:] + decoded_data = decoded_data[offset - start_offset :] decoded_data = decoded_data[:output_length] return decoded_data @@ -127,14 +168,18 @@ class AVMLStacker(interfaces.automagic.StackerLayerInterface): stack_order = 10 @classmethod - def stack(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: try: AVMLLayer._check_header(context.layers[layer_name]) except exceptions.LayerException: return None new_name = context.layers.free_layer_name("AVMLLayer") - context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name + context.config[ + interfaces.configuration.path_join(new_name, "base_layer") + ] = layer_name return AVMLLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 6194501ee..64166cfba 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -27,16 +27,18 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): provides = {"type": "physical"} SIGNATURE = 0x45474150 - VALIDDUMP = 0x504d5544 + VALIDDUMP = 0x504D5544 - crashdump_json = 'crash' + crashdump_json = "crash" supported_dumptypes = [0x01, 0x05] # we need 0x5 for 32-bit bitmaps - dump_header_name = '_DUMP_HEADER' + dump_header_name = "_DUMP_HEADER" - _magic_struct = struct.Struct(' None: + def __init__( + self, context: interfaces.context.ContextInterface, config_path: str, name: str + ) -> None: # Construct these so we can use self.config self._context = context @@ -46,15 +48,18 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): self._base_layer = self.config["base_layer"] # Create a custom SymbolSpace - self._crash_table_name = intermed.IntermediateSymbolTable.create(context, self._config_path, 'windows', - self.crashdump_json) + self._crash_table_name = intermed.IntermediateSymbolTable.create( + context, self._config_path, "windows", self.crashdump_json + ) # the _SUMMARY_DUMP is shared between 32- and 64-bit - self._crash_common_table_name = intermed.IntermediateSymbolTable.create(context, - self._config_path, - 'windows', - 'crash_common', - class_types = crash.class_types) + self._crash_common_table_name = intermed.IntermediateSymbolTable.create( + context, + self._config_path, + "windows", + "crash_common", + class_types=crash.class_types, + ) # Check Header hdr_layer = self._context.layers[self._base_layer] @@ -71,21 +76,30 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): # Verify that it is a supported format if header.DumpType not in self.supported_dumptypes: - vollog.log(constants.LOGLEVEL_VVVV, f"unsupported dump format 0x{header.DumpType:x}") - raise WindowsCrashDumpFormatException(name, f"unsupported dump format 0x{header.DumpType:x}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"unsupported dump format 0x{header.DumpType:x}", + ) + raise WindowsCrashDumpFormatException( + name, f"unsupported dump format 0x{header.DumpType:x}" + ) # Then call the super, which will call load_segments (which needs the base_layer before it'll work) super().__init__(context, config_path, name) def get_header(self) -> interfaces.objects.ObjectInterface: - return self.context.object(self._crash_table_name + constants.BANG + self.dump_header_name, - offset = 0, - layer_name = self._base_layer) + return self.context.object( + self._crash_table_name + constants.BANG + self.dump_header_name, + offset=0, + layer_name=self._base_layer, + ) def get_summary_header(self) -> interfaces.objects.ObjectInterface: - return self.context.object(self._crash_common_table_name + constants.BANG + "_SUMMARY_DUMP", - offset = 0x1000 * self.headerpages, - layer_name = self._base_layer) + return self.context.object( + self._crash_common_table_name + constants.BANG + "_SUMMARY_DUMP", + offset=0x1000 * self.headerpages, + layer_name=self._base_layer, + ) def _load_segments(self) -> None: """Loads up the segments from the meta_layer.""" @@ -93,15 +107,25 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): segments = [] if self.dump_type == 0x1: - header = self.context.object(self._crash_table_name + constants.BANG + self.dump_header_name, - offset = 0, - layer_name = self._base_layer) + header = self.context.object( + self._crash_table_name + constants.BANG + self.dump_header_name, + offset=0, + layer_name=self._base_layer, + ) offset = self.headerpages - header.PhysicalMemoryBlockBuffer.Run.count = header.PhysicalMemoryBlockBuffer.NumberOfRuns + header.PhysicalMemoryBlockBuffer.Run.count = ( + header.PhysicalMemoryBlockBuffer.NumberOfRuns + ) for run in header.PhysicalMemoryBlockBuffer.Run: segments.append( - (run.BasePage * 0x1000, offset * 0x1000, run.PageCount * 0x1000, run.PageCount * 0x1000)) + ( + run.BasePage * 0x1000, + offset * 0x1000, + run.PageCount * 0x1000, + run.PageCount * 0x1000, + ) + ) offset += run.PageCount elif self.dump_type == 0x05: @@ -118,7 +142,14 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): if first_bit is not None: last_bit = ((outer_index - 1) * 32) + 31 segment_length = (last_bit - first_bit + 1) * 0x1000 - segments.append((first_bit * 0x1000, first_offset, segment_length, segment_length)) + segments.append( + ( + first_bit * 0x1000, + first_offset, + segment_length, + segment_length, + ) + ) first_bit = None elif buffer_long[outer_index] == 0xFFFFFFFF: if first_bit is None: @@ -135,48 +166,74 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): offset = offset + 0x1000 else: if first_bit is not None: - segment_length = ((bit_addr - 1) - first_bit + 1) * 0x1000 - segments.append((first_bit * 0x1000, first_offset, segment_length, segment_length)) + segment_length = ( + (bit_addr - 1) - first_bit + 1 + ) * 0x1000 + segments.append( + ( + first_bit * 0x1000, + first_offset, + segment_length, + segment_length, + ) + ) first_bit = None last_bit_seen = (outer_index * 32) + 31 if first_bit is not None: segment_length = (last_bit_seen - first_bit + 1) * 0x1000 - segments.append((first_bit * 0x1000, first_offset, segment_length, segment_length)) + segments.append( + (first_bit * 0x1000, first_offset, segment_length, segment_length) + ) else: - vollog.log(constants.LOGLEVEL_VVVV, f"unsupported dump format 0x{self.dump_type:x}") - raise WindowsCrashDumpFormatException(self.name, f"unsupported dump format 0x{self.dump_type:x}") + vollog.log( + constants.LOGLEVEL_VVVV, f"unsupported dump format 0x{self.dump_type:x}" + ) + raise WindowsCrashDumpFormatException( + self.name, f"unsupported dump format 0x{self.dump_type:x}" + ) if len(segments) == 0: - raise WindowsCrashDumpFormatException(self.name, f"No Crash segments defined in {self._base_layer}") + raise WindowsCrashDumpFormatException( + self.name, f"No Crash segments defined in {self._base_layer}" + ) else: # report the segments for debugging. this is valuable for dev/troubleshooting but # not important enough for a dedicated plugin. for idx, (start_position, mapped_offset, length, _) in enumerate(segments): vollog.log( constants.LOGLEVEL_VVVV, - "Segment {}: Position {:#x} Offset {:#x} Length {:#x}".format(idx, start_position, mapped_offset, - length)) + "Segment {}: Position {:#x} Offset {:#x} Length {:#x}".format( + idx, start_position, mapped_offset, length + ), + ) self._segments = segments @classmethod - def check_header(cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0) -> Tuple[int, int]: + def check_header( + cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0 + ) -> Tuple[int, int]: # Verify the Window's crash dump file magic try: header_data = base_layer.read(offset, cls._magic_struct.size) except exceptions.InvalidAddressException: - raise WindowsCrashDumpFormatException(base_layer.name, - f"Crashdump header not found at offset {offset}") + raise WindowsCrashDumpFormatException( + base_layer.name, f"Crashdump header not found at offset {offset}" + ) (signature, validdump) = cls._magic_struct.unpack(header_data) if signature != cls.SIGNATURE: raise WindowsCrashDumpFormatException( - base_layer.name, f"Bad signature 0x{signature:x} at file offset 0x{offset:x}") + base_layer.name, + f"Bad signature 0x{signature:x} at file offset 0x{offset:x}", + ) if validdump != cls.VALIDDUMP: - raise WindowsCrashDumpFormatException(base_layer.name, - f"Invalid dump 0x{validdump:x} at file offset 0x{offset:x}") + raise WindowsCrashDumpFormatException( + base_layer.name, + f"Invalid dump 0x{validdump:x} at file offset 0x{offset:x}", + ) return signature, validdump @@ -188,8 +245,8 @@ class WindowsCrashDump64Layer(WindowsCrashDump32Layer): """ VALIDDUMP = 0x34365544 - crashdump_json = 'crash64' - dump_header_name = '_DUMP_HEADER64' + crashdump_json = "crash64" + dump_header_name = "_DUMP_HEADER64" supported_dumptypes = [0x1, 0x05] headerpages = 2 @@ -198,14 +255,18 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): stack_order = 11 @classmethod - def stack(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: for layer in [WindowsCrashDump32Layer, WindowsCrashDump64Layer]: with contextlib.suppress(WindowsCrashDumpFormatException): layer.check_header(context.layers[layer_name]) new_name = context.layers.free_layer_name(layer.__name__) - context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name + context.config[ + interfaces.configuration.path_join(new_name, "base_layer") + ] = layer_name return layer(context, new_name, new_name) return None diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index 4eb93a1c8..bcafa9aed 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -18,50 +18,82 @@ class ElfFormatException(exceptions.LayerException): class Elf64Layer(segmented.SegmentedLayer): """A layer that supports the Elf64 format as documented at: http://ftp.openwatcom.org/devel/docs/elf-64-gen.pdf""" + _header_struct = struct.Struct(" None: + def __init__( + self, context: interfaces.context.ContextInterface, config_path: str, name: str + ) -> None: # Create a custom SymbolSpace - self._elf_table_name = intermed.IntermediateSymbolTable.create(context, config_path, 'linux', 'elf') + self._elf_table_name = intermed.IntermediateSymbolTable.create( + context, config_path, "linux", "elf" + ) super().__init__(context, config_path, name) def _load_segments(self) -> None: """Load the segments from based on the PT_LOAD segments of the Elf64 format""" - ehdr = self.context.object(self._elf_table_name + constants.BANG + "Elf64_Ehdr", - layer_name = self._base_layer, - offset = 0) + ehdr = self.context.object( + self._elf_table_name + constants.BANG + "Elf64_Ehdr", + layer_name=self._base_layer, + offset=0, + ) segments = [] for pindex in range(ehdr.e_phnum): - phdr = self.context.object(self._elf_table_name + constants.BANG + "Elf64_Phdr", - layer_name = self._base_layer, - offset = ehdr.e_phoff + (pindex * ehdr.e_phentsize)) + phdr = self.context.object( + self._elf_table_name + constants.BANG + "Elf64_Phdr", + layer_name=self._base_layer, + offset=ehdr.e_phoff + (pindex * ehdr.e_phentsize), + ) # We only want PT_TYPES with valid sizes - if phdr.p_type.lookup() == "PT_LOAD" and phdr.p_filesz == phdr.p_memsz and phdr.p_filesz > 0: + if ( + phdr.p_type.lookup() == "PT_LOAD" + and phdr.p_filesz == phdr.p_memsz + and phdr.p_filesz > 0 + ): # Cast these to ints to ensure the offsets don't need reconstructing - segments.append((int(phdr.p_paddr), int(phdr.p_offset), int(phdr.p_memsz), int(phdr.p_memsz))) + segments.append( + ( + int(phdr.p_paddr), + int(phdr.p_offset), + int(phdr.p_memsz), + int(phdr.p_memsz), + ) + ) if len(segments) == 0: - raise ElfFormatException(self.name, f"No ELF segments defined in {self._base_layer}") + raise ElfFormatException( + self.name, f"No ELF segments defined in {self._base_layer}" + ) self._segments = segments @classmethod - def _check_header(cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0) -> bool: + def _check_header( + cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0 + ) -> bool: try: header_data = base_layer.read(offset, cls._header_struct.size) except exceptions.InvalidAddressException: - raise ElfFormatException(base_layer.name, - f"Offset 0x{offset:0x} does not exist within the base layer") - (magic, elf_class, elf_data_encoding, elf_version) = cls._header_struct.unpack(header_data) + raise ElfFormatException( + base_layer.name, + f"Offset 0x{offset:0x} does not exist within the base layer", + ) + (magic, elf_class, elf_data_encoding, elf_version) = cls._header_struct.unpack( + header_data + ) if magic != cls.MAGIC: - raise ElfFormatException(base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}") + raise ElfFormatException( + base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}" + ) if elf_class != cls.ELF_CLASS: - raise ElfFormatException(base_layer.name, f"ELF class is not 64-bit (2): {elf_class:d}") + raise ElfFormatException( + base_layer.name, f"ELF class is not 64-bit (2): {elf_class:d}" + ) # Virtualbox uses an ELF version of 0, which isn't to specification, but is ok to deal with return True @@ -70,10 +102,12 @@ class Elf64Stacker(interfaces.automagic.StackerLayerInterface): stack_order = 10 @classmethod - def stack(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: try: if not Elf64Layer._check_header(context.layers[layer_name]): return None @@ -81,6 +115,8 @@ class Elf64Stacker(interfaces.automagic.StackerLayerInterface): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("Elf64Layer") - context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name + context.config[ + interfaces.configuration.path_join(new_name, "base_layer") + ] = layer_name return Elf64Layer(context, new_name, new_name) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 723e4143b..dce207fd5 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -28,28 +28,39 @@ class Intel(linear.LinearlyMappedLayer): # NOTE: _maxphyaddr is MAXPHYADDR as defined in the Intel specs *NOT* the maximum physical address _maxphyaddr = 32 _maxvirtaddr = _maxphyaddr - _structure = [('page directory', 10, False), ('page table', 10, True)] - _direct_metadata = collections.ChainMap({'architecture': 'Intel32'}, {'mapped': True}, - interfaces.layers.TranslationLayerInterface._direct_metadata) + _structure = [("page directory", 10, False), ("page table", 10, True)] + _direct_metadata = collections.ChainMap( + {"architecture": "Intel32"}, + {"mapped": True}, + interfaces.layers.TranslationLayerInterface._direct_metadata, + ) - def __init__(self, - context: interfaces.context.ContextInterface, - config_path: str, - name: str, - metadata: Optional[Dict[str, Any]] = None) -> None: - super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__( + context=context, config_path=config_path, name=name, metadata=metadata + ) self._base_layer = self.config["memory_layer"] self._swap_layers: List[str] = [] self._page_map_offset = self.config["page_map_offset"] # Assign constants self._initial_position = min(self._maxvirtaddr, self._bits_per_register) - 1 - self._initial_entry = self._mask(self._page_map_offset, self._initial_position, 0) | 0x1 + self._initial_entry = ( + self._mask(self._page_map_offset, self._initial_position, 0) | 0x1 + ) self._entry_size = struct.calcsize(self._entry_format) self._entry_number = self.page_size // self._entry_size # These can vary depending on the type of space - self._index_shift = int(math.ceil(math.log2(struct.calcsize(self._entry_format)))) + self._index_shift = int( + math.ceil(math.log2(struct.calcsize(self._entry_format))) + ) @classproperty @functools.lru_cache() @@ -86,7 +97,7 @@ class Intel(linear.LinearlyMappedLayer): """Returns the bits of a value between highbit and lowbit inclusive.""" high_mask = (1 << (high_bit + 1)) - 1 low_mask = (1 << low_bit) - 1 - mask = (high_mask ^ low_mask) + mask = high_mask ^ low_mask # print(high_bit, low_bit, bin(mask), bin(value)) return value & mask @@ -106,9 +117,16 @@ class Intel(linear.LinearlyMappedLayer): # Now we're done if not self._page_is_valid(entry): - raise exceptions.PagedInvalidAddressException(self.name, offset, position + 1, entry, - f"Page Fault at entry {hex(entry)} in page entry") - page = self._mask(entry, self._maxphyaddr - 1, position + 1) | self._mask(offset, position, 0) + raise exceptions.PagedInvalidAddressException( + self.name, + offset, + position + 1, + entry, + f"Page Fault at entry {hex(entry)} in page entry", + ) + page = self._mask(entry, self._maxphyaddr - 1, position + 1) | self._mask( + offset, position, 0 + ) return page, 1 << (position + 1), self._base_layer @@ -124,20 +142,30 @@ class Intel(linear.LinearlyMappedLayer): entry = self._initial_entry if self.minimum_address > offset > self.maximum_address: - raise exceptions.PagedInvalidAddressException(self.name, offset, position + 1, entry, - "Entry outside virtual address range: " + hex(entry)) + raise exceptions.PagedInvalidAddressException( + self.name, + offset, + position + 1, + entry, + "Entry outside virtual address range: " + hex(entry), + ) # Run through the offset in various chunks for (name, size, large_page) in self._structure: # Check we're valid if not self._page_is_valid(entry): - raise exceptions.PagedInvalidAddressException(self.name, offset, position + 1, entry, - "Page Fault at entry " + hex(entry) + " in table " + name) + raise exceptions.PagedInvalidAddressException( + self.name, + offset, + position + 1, + entry, + "Page Fault at entry " + hex(entry) + " in table " + name, + ) # Check if we're a large page if large_page and (entry & (1 << 7)): # Mask off the PAT bit if entry & (1 << 12): - entry -= (1 << 12) + entry -= 1 << 12 # We're a large page, the rest is finished below # If we want to implement PSE-36, it would need to be done here break @@ -147,33 +175,51 @@ class Intel(linear.LinearlyMappedLayer): index = self._mask(offset, start, position + 1) >> (position + 1) # Grab the base address of the table we'll be getting the next entry from - base_address = self._mask(entry, self._maxphyaddr - 1, size + self._index_shift) + base_address = self._mask( + entry, self._maxphyaddr - 1, size + self._index_shift + ) table = self._get_valid_table(base_address) if table is None: - raise exceptions.PagedInvalidAddressException(self.name, offset, position + 1, entry, - "Page Fault at entry " + hex(entry) + " in table " + name) + raise exceptions.PagedInvalidAddressException( + self.name, + offset, + position + 1, + entry, + "Page Fault at entry " + hex(entry) + " in table " + name, + ) # Read the data for the next entry - entry_data = table[(index << self._index_shift):(index << self._index_shift) + self._entry_size] + entry_data = table[ + (index << self._index_shift) : (index << self._index_shift) + + self._entry_size + ] if INTEL_TRANSLATION_DEBUGGING: vollog.log( - constants.LOGLEVEL_VVVV, "Entry {} at index {} gives data {} as {}".format( - hex(entry), hex(index), hex(struct.unpack(self._entry_format, entry_data)[0]), name)) + constants.LOGLEVEL_VVVV, + "Entry {} at index {} gives data {} as {}".format( + hex(entry), + hex(index), + hex(struct.unpack(self._entry_format, entry_data)[0]), + name, + ), + ) # Read out the new entry from memory - entry, = struct.unpack(self._entry_format, entry_data) + (entry,) = struct.unpack(self._entry_format, entry_data) return entry, position @functools.lru_cache(1025) def _get_valid_table(self, base_address: int) -> Optional[bytes]: """Extracts the table, validates it and returns it if it's valid.""" - table = self._context.layers.read(self._base_layer, base_address, self.page_size) + table = self._context.layers.read( + self._base_layer, base_address, self.page_size + ) # If the table is entirely duplicates, then mark the whole table as bad - if (table == table[:self._entry_size] * self._entry_number): + if table == table[: self._entry_size] * self._entry_number: return None return table @@ -182,27 +228,36 @@ class Intel(linear.LinearlyMappedLayer): address.""" try: # TODO: Consider reimplementing this, since calls to mapping can call is_valid - return all([ - self._context.layers[layer].is_valid(mapped_offset) - for _, _, mapped_offset, _, layer in self.mapping(offset, length) - ]) + return all( + [ + self._context.layers[layer].is_valid(mapped_offset) + for _, _, mapped_offset, _, layer in self.mapping(offset, length) + ] + ) except exceptions.InvalidAddressException: return False - def mapping(self, - offset: int, - length: int, - ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + def mapping( + self, offset: int, length: int, ignore_errors: bool = False + ) -> Iterable[Tuple[int, int, int, int, str]]: """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer) mappings. This allows translation layers to provide maps of contiguous regions in one layer """ - stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = stashed_map_layer = None - for offset, size, mapped_offset, mapped_size, map_layer in self._mapping(offset, length, ignore_errors): - if stashed_offset is None or (stashed_offset + stashed_size != offset) or ( - stashed_mapped_offset + stashed_mapped_size != mapped_offset) or (stashed_map_layer != map_layer): + stashed_offset = ( + stashed_mapped_offset + ) = stashed_size = stashed_mapped_size = stashed_map_layer = None + for offset, size, mapped_offset, mapped_size, map_layer in self._mapping( + offset, length, ignore_errors + ): + if ( + stashed_offset is None + or (stashed_offset + stashed_size != offset) + or (stashed_mapped_offset + stashed_mapped_size != mapped_offset) + or (stashed_map_layer != map_layer) + ): # The block isn't contiguous if stashed_offset is not None: yield stashed_offset, stashed_size, stashed_mapped_offset, stashed_mapped_size, stashed_map_layer @@ -217,14 +272,18 @@ class Intel(linear.LinearlyMappedLayer): stashed_size += size stashed_mapped_size += mapped_size # Yield whatever's left - if (stashed_offset is not None and stashed_mapped_offset is not None and stashed_size is not None - and stashed_mapped_size is not None and stashed_map_layer is not None): + if ( + stashed_offset is not None + and stashed_mapped_offset is not None + and stashed_size is not None + and stashed_mapped_size is not None + and stashed_map_layer is not None + ): yield stashed_offset, stashed_size, stashed_mapped_offset, stashed_mapped_size, stashed_map_layer - def _mapping(self, - offset: int, - length: int, - ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + def _mapping( + self, offset: int, length: int, ignore_errors: bool = False + ) -> Iterable[Tuple[int, int, int, int, str]]: """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer) mappings. @@ -235,7 +294,9 @@ class Intel(linear.LinearlyMappedLayer): try: mapped_offset, _, layer_name = self._translate(offset) if not self._context.layers[layer_name].is_valid(mapped_offset): - raise exceptions.InvalidAddressException(layer_name = layer_name, invalid_address = mapped_offset) + raise exceptions.InvalidAddressException( + layer_name=layer_name, invalid_address=mapped_offset + ) except exceptions.InvalidAddressException: if not ignore_errors: raise @@ -246,9 +307,16 @@ class Intel(linear.LinearlyMappedLayer): try: chunk_offset, page_size, layer_name = self._translate(offset) chunk_size = min(page_size - (chunk_offset % page_size), length) - if not self._context.layers[layer_name].is_valid(chunk_offset, chunk_size): - raise exceptions.InvalidAddressException(layer_name = layer_name, invalid_address = chunk_offset) - except (exceptions.PagedInvalidAddressException, exceptions.InvalidAddressException) as excp: + if not self._context.layers[layer_name].is_valid( + chunk_offset, chunk_size + ): + raise exceptions.InvalidAddressException( + layer_name=layer_name, invalid_address=chunk_offset + ) + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ) as excp: if not ignore_errors: raise # We can jump more if we know where the page fault failed @@ -256,7 +324,7 @@ class Intel(linear.LinearlyMappedLayer): mask = (1 << excp.invalid_bits) - 1 else: mask = (1 << self._page_size_in_bits) - 1 - length_diff = (mask + 1 - (offset & mask)) + length_diff = mask + 1 - (offset & mask) length -= length_diff offset += length_diff else: @@ -273,11 +341,13 @@ class Intel(linear.LinearlyMappedLayer): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'memory_layer', optional = False), - requirements.LayerListRequirement(name = 'swap_layers', optional = True), - requirements.IntRequirement(name = 'page_map_offset', optional = False), - requirements.IntRequirement(name = 'kernel_virtual_offset', optional = True), - requirements.StringRequirement(name = 'kernel_banner', optional = True) + requirements.TranslationLayerRequirement( + name="memory_layer", optional=False + ), + requirements.LayerListRequirement(name="swap_layers", optional=True), + requirements.IntRequirement(name="page_map_offset", optional=False), + requirements.IntRequirement(name="kernel_virtual_offset", optional=True), + requirements.StringRequirement(name="kernel_banner", optional=True), ] @@ -289,25 +359,34 @@ class IntelPAE(Intel): _bits_per_register = 32 _maxphyaddr = 40 _maxvirtaddr = 32 - _structure = [('page directory pointer', 2, False), ('page directory', 9, True), ('page table', 9, True)] - _direct_metadata = collections.ChainMap({'pae': True}, Intel._direct_metadata) + _structure = [ + ("page directory pointer", 2, False), + ("page directory", 9, True), + ("page table", 9, True), + ] + _direct_metadata = collections.ChainMap({"pae": True}, Intel._direct_metadata) class Intel32e(Intel): """Class for handling 64-bit (32-bit extensions) for Intel architectures.""" - _direct_metadata = collections.ChainMap({'architecture': 'Intel64'}, Intel._direct_metadata) + _direct_metadata = collections.ChainMap( + {"architecture": "Intel64"}, Intel._direct_metadata + ) _entry_format = " bool: """Returns whether a particular page is valid based on its entry. @@ -321,7 +400,9 @@ class WindowsMixin(Intel): """ return bool((entry & 1) or ((entry & 1 << 11) and not entry & 1 << 10)) - def _translate_swap(self, layer: Intel, offset: int, bit_offset: int) -> Tuple[int, int, str]: + def _translate_swap( + self, layer: Intel, offset: int, bit_offset: int + ) -> Tuple[int, int, str]: try: return super()._translate(offset) except exceptions.PagedInvalidAddressException as excp: @@ -331,19 +412,27 @@ class WindowsMixin(Intel): unknown_bit = bool(entry & (1 << 7)) n = (entry >> 1) & 0xF vbit = bool(entry & 1) - if (not tbit and not pbit and not vbit and unknown_bit) and ((entry >> bit_offset) != 0): + if (not tbit and not pbit and not vbit and unknown_bit) and ( + (entry >> bit_offset) != 0 + ): swap_offset = entry >> bit_offset << excp.invalid_bits - if layer.config.get('swap_layers', False): + if layer.config.get("swap_layers", False): swap_layer_name = layer.config.get( - interfaces.configuration.path_join('swap_layers', 'swap_layers' + str(n)), None) + interfaces.configuration.path_join( + "swap_layers", "swap_layers" + str(n) + ), + None, + ) if swap_layer_name: return swap_offset, 1 << excp.invalid_bits, swap_layer_name - raise exceptions.SwappedInvalidAddressException(layer_name = excp.layer_name, - invalid_address = excp.invalid_address, - invalid_bits = excp.invalid_bits, - entry = excp.entry, - swap_offset = swap_offset) + raise exceptions.SwappedInvalidAddressException( + layer_name=excp.layer_name, + invalid_address=excp.invalid_address, + invalid_bits=excp.invalid_bits, + entry=excp.entry, + swap_offset=swap_offset, + ) raise @@ -351,13 +440,11 @@ class WindowsMixin(Intel): class WindowsIntel(WindowsMixin, Intel): - def _translate(self, offset): return self._translate_swap(self, offset, self._page_size_in_bits) class WindowsIntelPAE(WindowsMixin, IntelPAE): - def _translate(self, offset: int) -> Tuple[int, int, str]: return self._translate_swap(self, offset, self._bits_per_register) diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index fb0442cfe..73700dd3c 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -9,6 +9,7 @@ from typing import Optional, Any, List try: import leechcorepyc + HAS_LEECHCORE = True except ImportError: HAS_LEECHCORE = False @@ -66,7 +67,7 @@ if HAS_LEECHCORE: """ return bool(self._handle) - def seek(self, offset, whence = io.SEEK_SET): + def seek(self, offset, whence=io.SEEK_SET): if whence == io.SEEK_SET: self._cursor = offset elif whence == io.SEEK_CUR: @@ -91,9 +92,14 @@ if HAS_LEECHCORE: output = [] for entry in self.handle.memmap: - if entry['base'] + entry['size'] <= chunk_start or entry['base'] >= chunk_start + chunk_size: + if ( + entry["base"] + entry["size"] <= chunk_start + or entry["base"] >= chunk_start + chunk_size + ): continue - output += [(max(entry['base'], chunk_start), min(entry['size'], chunk_size))] + output += [ + (max(entry["base"], chunk_start), min(entry["size"], chunk_size)) + ] chunk_start = output[-1][0] + output[-1][1] chunk_size = max(0, size - chunk_start) @@ -114,14 +120,16 @@ if HAS_LEECHCORE: if len(data) > size: data = data[:size] else: - data = data + b'\x00' * (size - len(data)) + data = data + b"\x00" * (size - len(data)) self._cursor += len(data) if not len(data): - raise exceptions.InvalidAddressException('LeechCore layer read failure', self._cursor + len(data)) + raise exceptions.InvalidAddressException( + "LeechCore layer read failure", self._cursor + len(data) + ) return data def readline(self, __size: Optional[int] = ...) -> bytes: - data = b'' + data = b"" while __size > self._chunk_size or __size < 0: data += self.read(self._chunk_size) index = data.find(b"\n") @@ -159,20 +167,18 @@ if HAS_LEECHCORE: def closed(self): return self._handle - class LeechCoreHandler(resources.VolatilityHandler): - """Handler for the invented `leechcore` scheme. This is an unofficial scheme and not registered with IANA - """ + """Handler for the invented `leechcore` scheme. This is an unofficial scheme and not registered with IANA""" @classmethod def non_cached_schemes(cls) -> List[str]: """We need to turn caching *off* for a live filesystem""" - return ['leechcore'] + return ["leechcore"] @staticmethod def default_open(req: urllib.request.Request) -> Optional[Any]: """Handles the request if it's the leechcore scheme.""" - if req.type == 'leechcore': - device_uri = '://'.join(req.full_url.split('://')[1:]) + if req.type == "leechcore": + device_uri = "://".join(req.full_url.split("://")[1:]) return LeechCoreFile(device_uri) return None diff --git a/volatility3/framework/layers/lime.py b/volatility3/framework/layers/lime.py index 4f4a66f18..28d646640 100644 --- a/volatility3/framework/layers/lime.py +++ b/volatility3/framework/layers/lime.py @@ -20,14 +20,16 @@ class LimeLayer(segmented.SegmentedLayer): are large holes in the physical layer """ - MAGIC = 0x4c694d45 + MAGIC = 0x4C694D45 VERSION = 1 # Magic[4], Version[4], Start[8], End[8], Reserved[8] # XXX move this to a custom SymbolSpace? - _header_struct = struct.Struct(' None: + def __init__( + self, context: interfaces.context.ContextInterface, config_path: str, name: str + ) -> None: super().__init__(context, config_path, name) # The base class loads the segments on initialization, but otherwise this must to get the right min/max addresses @@ -45,31 +47,45 @@ class LimeLayer(segmented.SegmentedLayer): if start < maxaddr or end < start: raise LimeFormatException( - self.name, f"Bad start/end 0x{start:x}/0x{end:x} at file offset 0x{offset:x}") + self.name, + f"Bad start/end 0x{start:x}/0x{end:x} at file offset 0x{offset:x}", + ) segment_length = end - start + 1 - segments.append((start, offset + header_size, segment_length, segment_length)) + segments.append( + (start, offset + header_size, segment_length, segment_length) + ) maxaddr = end offset = offset + header_size + segment_length if len(segments) == 0: - raise LimeFormatException(self.name, f"No LiME segments defined in {self._base_layer}") + raise LimeFormatException( + self.name, f"No LiME segments defined in {self._base_layer}" + ) self._segments = segments @classmethod - def _check_header(cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0) -> Tuple[int, int]: + def _check_header( + cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0 + ) -> Tuple[int, int]: try: header_data = base_layer.read(offset, cls._header_struct.size) except exceptions.InvalidAddressException: - raise LimeFormatException(base_layer.name, - f"Offset 0x{offset:0x} does not exist within the base layer") + raise LimeFormatException( + base_layer.name, + f"Offset 0x{offset:0x} does not exist within the base layer", + ) (magic, version, start, end, reserved) = cls._header_struct.unpack(header_data) if magic != cls.MAGIC: - raise LimeFormatException(base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}") + raise LimeFormatException( + base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}" + ) if version != cls.VERSION: - raise LimeFormatException(base_layer.name, - f"Unexpected version {version:d} at file offset 0x{offset:x}") + raise LimeFormatException( + base_layer.name, + f"Unexpected version {version:d} at file offset 0x{offset:x}", + ) return start, end @@ -77,14 +93,18 @@ class LimeStacker(interfaces.automagic.StackerLayerInterface): stack_order = 10 @classmethod - def stack(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: try: LimeLayer._check_header(context.layers[layer_name]) except LimeFormatException: return None new_name = context.layers.free_layer_name("LimeLayer") - context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name + context.config[ + interfaces.configuration.path_join(new_name, "base_layer") + ] = layer_name return LimeLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/linear.py b/volatility3/framework/layers/linear.py index 383f3d558..19203eb66 100644 --- a/volatility3/framework/layers/linear.py +++ b/volatility3/framework/layers/linear.py @@ -14,41 +14,54 @@ class LinearlyMappedLayer(interfaces.layers.TranslationLayerInterface): ### Translation layer convenience function - def translate(self, offset: int, ignore_errors: bool = False) -> Tuple[Optional[int], Optional[str]]: + def translate( + self, offset: int, ignore_errors: bool = False + ) -> Tuple[Optional[int], Optional[str]]: mapping = list(self.mapping(offset, 0, ignore_errors)) if len(mapping) == 1: original_offset, _, mapped_offset, _, layer = mapping[0] if original_offset != offset: - raise exceptions.LayerException(self.name, - f"Layer {self.name} claims to map linearly but does not") + raise exceptions.LayerException( + self.name, f"Layer {self.name} claims to map linearly but does not" + ) else: if ignore_errors: # We should only hit this if we ignored errors, but check anyway return None, None - raise exceptions.InvalidAddressException(self.name, offset, - f"Cannot translate {offset} in layer {self.name}") + raise exceptions.InvalidAddressException( + self.name, offset, f"Cannot translate {offset} in layer {self.name}" + ) return mapped_offset, layer # ## Read/Write functions for mapped pages # Redefine read here for speed reasons (so we don't call a processing method - @functools.lru_cache(maxsize = 512) + @functools.lru_cache(maxsize=512) def read(self, offset: int, length: int, pad: bool = False) -> bytes: """Reads an offset for length bytes and returns 'bytes' (not 'str') of length size.""" current_offset = offset output: List[bytes] = [] - for (offset, _, mapped_offset, mapped_length, layer) in self.mapping(offset, length, ignore_errors = pad): + for (offset, _, mapped_offset, mapped_length, layer) in self.mapping( + offset, length, ignore_errors=pad + ): if not pad and offset > current_offset: raise exceptions.InvalidAddressException( - self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}") + self.name, + current_offset, + f"Layer {self.name} cannot map offset: {current_offset}", + ) elif offset > current_offset: output += [b"\x00" * (offset - current_offset)] current_offset = offset elif offset < current_offset: - raise exceptions.LayerException(self.name, "Mapping returned an overlapping element") + raise exceptions.LayerException( + self.name, "Mapping returned an overlapping element" + ) if mapped_length > 0: - output += [self._context.layers.read(layer, mapped_offset, mapped_length, pad)] + output += [ + self._context.layers.read(layer, mapped_offset, mapped_length, pad) + ] current_offset += mapped_length recovered_data = b"".join(output) return recovered_data + b"\x00" * (length - len(recovered_data)) @@ -61,15 +74,22 @@ class LinearlyMappedLayer(interfaces.layers.TranslationLayerInterface): for (offset, _, mapped_offset, length, layer) in self.mapping(offset, length): if offset > current_offset: raise exceptions.InvalidAddressException( - self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}") + self.name, + current_offset, + f"Layer {self.name} cannot map offset: {current_offset}", + ) elif offset < current_offset: - raise exceptions.LayerException(self.name, "Mapping returned an overlapping element") + raise exceptions.LayerException( + self.name, "Mapping returned an overlapping element" + ) self._context.layers.write(layer, mapped_offset, value[:length]) value = value[length:] current_offset += length - def _scan_iterator(self, - scanner: 'interfaces.layers.ScannerInterface', - sections: Iterable[Tuple[int, int]], - linear: bool = True) -> Iterable[interfaces.layers.IteratorValue]: + def _scan_iterator( + self, + scanner: "interfaces.layers.ScannerInterface", + sections: Iterable[Tuple[int, int]], + linear: bool = True, + ) -> Iterable[interfaces.layers.IteratorValue]: return super()._scan_iterator(scanner, sections, linear) diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 02fc570bc..76c645e92 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -21,15 +21,19 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer): "BIG_MSF_HDR": "Microsoft C/C++ MSF 7.00\r\n\x1a\x44\x53", } - def __init__(self, - context: 'interfaces.context.ContextInterface', - config_path: str, - name: str, - metadata: Optional[Dict[str, Any]] = None) -> None: + def __init__( + self, + context: "interfaces.context.ContextInterface", + config_path: str, + name: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: super().__init__(context, config_path, name, metadata) self._base_layer = self.config["base_layer"] - self._pdb_symbol_table = intermed.IntermediateSymbolTable.create(context, self._config_path, 'windows', 'pdb') + self._pdb_symbol_table = intermed.IntermediateSymbolTable.create( + context, self._config_path, "windows", "pdb" + ) response = self._check_header() if response is None: raise PDBFormatException(name, "Could not find a suitable header") @@ -46,56 +50,79 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer): return # Recover the root table, by recovering the root table index table... - module = self.context.module(self.pdb_symbol_table, self._base_layer, offset = 0) + module = self.context.module(self.pdb_symbol_table, self._base_layer, offset=0) entry_size = module.get_type("unsigned long").size - root_table_num_pages = math.ceil(self._header.StreamInfo.StreamInfoSize / self._header.PageSize) - root_index_size = math.ceil((root_table_num_pages * entry_size) / self._header.PageSize) - root_index = module.object(object_type = "array", - offset = self._header.vol.size, - count = root_index_size, - subtype = module.get_type("unsigned long")) - root_index_layer_name = self.create_stream_from_pages("root_index", self._header.StreamInfo.StreamInfoSize, - [x for x in root_index]) + root_table_num_pages = math.ceil( + self._header.StreamInfo.StreamInfoSize / self._header.PageSize + ) + root_index_size = math.ceil( + (root_table_num_pages * entry_size) / self._header.PageSize + ) + root_index = module.object( + object_type="array", + offset=self._header.vol.size, + count=root_index_size, + subtype=module.get_type("unsigned long"), + ) + root_index_layer_name = self.create_stream_from_pages( + "root_index", + self._header.StreamInfo.StreamInfoSize, + [x for x in root_index], + ) - module = self.context.module(self.pdb_symbol_table, root_index_layer_name, offset = 0) - root_pages = module.object(object_type = "array", - offset = 0, - count = root_table_num_pages, - subtype = module.get_type("unsigned long")) - root_layer_name = self.create_stream_from_pages("root", self._header.StreamInfo.StreamInfoSize, - [x for x in root_pages]) + module = self.context.module( + self.pdb_symbol_table, root_index_layer_name, offset=0 + ) + root_pages = module.object( + object_type="array", + offset=0, + count=root_table_num_pages, + subtype=module.get_type("unsigned long"), + ) + root_layer_name = self.create_stream_from_pages( + "root", self._header.StreamInfo.StreamInfoSize, [x for x in root_pages] + ) - module = self.context.module(self.pdb_symbol_table, root_layer_name, offset = 0) - num_streams = module.object(object_type = "unsigned long", offset = 0) - stream_sizes = module.object(object_type = "array", - offset = entry_size, - count = num_streams, - subtype = module.get_type("unsigned long")) + module = self.context.module(self.pdb_symbol_table, root_layer_name, offset=0) + num_streams = module.object(object_type="unsigned long", offset=0) + stream_sizes = module.object( + object_type="array", + offset=entry_size, + count=num_streams, + subtype=module.get_type("unsigned long"), + ) current_offset = (num_streams + 1) * entry_size for stream in range(num_streams): list_size = math.ceil(stream_sizes[stream] / self.page_size) - if list_size == 0 or stream_sizes[stream] == 0xffffffff: + if list_size == 0 or stream_sizes[stream] == 0xFFFFFFFF: self._streams[stream] = None else: - stream_page_list = module.object(object_type = "array", - offset = current_offset, - count = list_size, - subtype = module.get_type("unsigned long")) - current_offset += (list_size * entry_size) - self._streams[stream] = self.create_stream_from_pages("stream" + str(stream), stream_sizes[stream], - [x for x in stream_page_list]) + stream_page_list = module.object( + object_type="array", + offset=current_offset, + count=list_size, + subtype=module.get_type("unsigned long"), + ) + current_offset += list_size * entry_size + self._streams[stream] = self.create_stream_from_pages( + "stream" + str(stream), + stream_sizes[stream], + [x for x in stream_page_list], + ) - def create_stream_from_pages(self, stream_name: str, maximum_size: int, pages: List[int]) -> str: + def create_stream_from_pages( + self, stream_name: str, maximum_size: int, pages: List[int] + ) -> str: # Construct a root layer based on a number of pages layer_name = self.name + "_" + stream_name path_join = interfaces.configuration.path_join config_path = path_join(self.config_path, stream_name) - self.context.config[path_join(config_path, 'base_layer')] = self.name - self.context.config[path_join(config_path, 'pages')] = pages - self.context.config[path_join(config_path, 'maximum_size')] = maximum_size + self.context.config[path_join(config_path, "base_layer")] = self.name + self.context.config[path_join(config_path, "pages")] = pages + self.context.config[path_join(config_path, "maximum_size")] = maximum_size layer = PdbMSFStream(self.context, config_path, layer_name) self.context.layers.add_layer(layer) return layer_name @@ -107,7 +134,10 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer): header_type = self.pdb_symbol_table + constants.BANG + header current_header = self.context.object(header_type, self._base_layer, 0) if utility.array_to_string(current_header.Magic) == self._headers[header]: - if not (current_header.PageSize < 0x100 or current_header.PageSize > (128 * 0x10000)): + if not ( + current_header.PageSize < 0x100 + or current_header.PageSize > (128 * 0x10000) + ): return header, current_header return None @@ -123,7 +153,9 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [requirements.TranslationLayerRequirement(name = 'base_layer', optional = False)] + return [ + requirements.TranslationLayerRequirement(name="base_layer", optional=False) + ] @property def maximum_address(self) -> int: @@ -136,13 +168,12 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer): def is_valid(self, offset: int, length: int = 1) -> bool: return self.context.layers[self._base_layer].is_valid(offset, length) - def mapping(self, - offset: int, - length: int, - ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + def mapping( + self, offset: int, length: int, ignore_errors: bool = False + ) -> Iterable[Tuple[int, int, int, int, str]]: yield offset, length, offset, length, self._base_layer - def get_stream(self, index) -> Optional['PdbMSFStream']: + def get_stream(self, index) -> Optional["PdbMSFStream"]: self.read_streams() if index not in self._streams: raise PDBFormatException(self.name, "Stream not present") @@ -154,12 +185,13 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer): class PdbMSFStream(linear.LinearlyMappedLayer): - - def __init__(self, - context: 'interfaces.context.ContextInterface', - config_path: str, - name: str, - metadata: Optional[Dict[str, Any]] = None) -> None: + def __init__( + self, + context: "interfaces.context.ContextInterface", + config_path: str, + name: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: super().__init__(context, config_path, name, metadata) self._base_layer = self.config["base_layer"] self._pages = self.config.get("pages", None) @@ -180,28 +212,31 @@ class PdbMSFStream(linear.LinearlyMappedLayer): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ListRequirement(name = 'pages', element_type = int, min_elements = 1), - requirements.TranslationLayerRequirement(name = 'base_layer'), - requirements.IntRequirement(name = 'maximum_size') + requirements.ListRequirement( + name="pages", element_type=int, min_elements=1 + ), + requirements.TranslationLayerRequirement(name="base_layer"), + requirements.IntRequirement(name="maximum_size"), ] - def mapping(self, - offset: int, - length: int, - ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + def mapping( + self, offset: int, length: int, ignore_errors: bool = False + ) -> Iterable[Tuple[int, int, int, int, str]]: returned = 0 page_size = self._pdb_layer.page_size while length > 0: page = math.floor((offset + returned) / page_size) - page_position = ((offset + returned) % page_size) + page_position = (offset + returned) % page_size chunk_size = min(page_size - page_position, length) if page >= self._pages_len: if not ignore_errors: - raise exceptions.InvalidAddressException(layer_name = self.name, - invalid_address = offset + returned) + raise exceptions.InvalidAddressException( + layer_name=self.name, invalid_address=offset + returned + ) else: - yield offset + returned, chunk_size, (self._pages[page] * - page_size) + page_position, chunk_size, self._base_layer + yield offset + returned, chunk_size, ( + self._pages[page] * page_size + ) + page_position, chunk_size, self._base_layer returned += chunk_size length -= chunk_size @@ -218,13 +253,17 @@ class PdbMSFStream(linear.LinearlyMappedLayer): @property def maximum_address(self) -> int: - return self.config.get('maximum_size', len(self._pages) * self._pdb_layer.page_size) + return self.config.get( + "maximum_size", len(self._pages) * self._pdb_layer.page_size + ) @property def _pdb_layer(self) -> PdbMultiStreamFormat: if self._base_layer not in self._context.layers: - raise PDBFormatException(self._base_layer, - f"No PdbMultiStreamFormat layer found: {self._base_layer}") + raise PDBFormatException( + self._base_layer, + f"No PdbMultiStreamFormat layer found: {self._base_layer}", + ) result = self._context.layers[self._base_layer] if isinstance(result, PdbMultiStreamFormat): return result diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index b09055c90..70dd9541a 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -16,13 +16,17 @@ class BufferDataLayer(interfaces.layers.DataLayerInterface): """A DataLayer class backed by a buffer in memory, designed for testing and swift data access.""" - def __init__(self, - context: interfaces.context.ContextInterface, - config_path: str, - name: str, - buffer: bytes, - metadata: Optional[Dict[str, Any]] = None) -> None: - super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + buffer: bytes, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__( + context=context, config_path=config_path, name=name, metadata=metadata + ) self._buffer = buffer @property @@ -37,8 +41,10 @@ class BufferDataLayer(interfaces.layers.DataLayerInterface): def is_valid(self, offset: int, length: int = 1) -> bool: """Returns whether the offset is valid or not.""" - return bool(self.minimum_address <= offset <= self.maximum_address - and self.minimum_address <= offset + length - 1 <= self.maximum_address) + return bool( + self.minimum_address <= offset <= self.maximum_address + and self.minimum_address <= offset + length - 1 <= self.maximum_address + ) def read(self, address: int, length: int, pad: bool = False) -> bytes: """Reads the data from the buffer.""" @@ -46,26 +52,30 @@ class BufferDataLayer(interfaces.layers.DataLayerInterface): invalid_address = address if self.minimum_address < address <= self.maximum_address: invalid_address = self.maximum_address + 1 - raise exceptions.InvalidAddressException(self.name, invalid_address, - "Offset outside of the buffer boundaries") - return self._buffer[address:address + length] + raise exceptions.InvalidAddressException( + self.name, invalid_address, "Offset outside of the buffer boundaries" + ) + return self._buffer[address : address + length] def write(self, address: int, data: bytes): """Writes the data from to the buffer.""" - self._buffer = self._buffer[:address] + data + self._buffer[address + len(data):] + self._buffer = ( + self._buffer[:address] + data + self._buffer[address + len(data) :] + ) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # No real requirements (only the buffer). Need to figure out if there's a better way of representing this return [ - requirements.BytesRequirement(name = 'buffer', - description = "The direct bytes to interact with", - optional = False) + requirements.BytesRequirement( + name="buffer", + description="The direct bytes to interact with", + optional=False, + ) ] class DummyLock: - def __enter__(self) -> None: pass @@ -76,12 +86,16 @@ class DummyLock: class FileLayer(interfaces.layers.DataLayerInterface): """a DataLayer backed by a file on the filesystem.""" - def __init__(self, - context: interfaces.context.ContextInterface, - config_path: str, - name: str, - metadata: Optional[Dict[str, Any]] = None) -> None: - super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__( + context=context, config_path=config_path, name=name, metadata=metadata + ) self._write_warning = False self._location = self.config["location"] @@ -133,8 +147,10 @@ class FileLayer(interfaces.layers.DataLayerInterface): """Returns whether the offset is valid or not.""" if length <= 0: raise ValueError("Length must be positive") - return bool(self.minimum_address <= offset <= self.maximum_address - and self.minimum_address <= offset + length - 1 <= self.maximum_address) + return bool( + self.minimum_address <= offset <= self.maximum_address + and self.minimum_address <= offset + length - 1 <= self.maximum_address + ) def read(self, offset: int, length: int, pad: bool = False) -> bytes: """Reads from the file at offset for length.""" @@ -142,8 +158,9 @@ class FileLayer(interfaces.layers.DataLayerInterface): invalid_address = offset if self.minimum_address < offset <= self.maximum_address: invalid_address = self.maximum_address + 1 - raise exceptions.InvalidAddressException(self.name, invalid_address, - "Offset outside of the buffer boundaries") + raise exceptions.InvalidAddressException( + self.name, invalid_address, "Offset outside of the buffer boundaries" + ) # TODO: implement locking for multi-threading with self._lock: @@ -152,10 +169,13 @@ class FileLayer(interfaces.layers.DataLayerInterface): if len(data) < length: if pad: - data += (b"\x00" * (length - len(data))) + data += b"\x00" * (length - len(data)) else: raise exceptions.InvalidAddressException( - self.name, offset + len(data), "Could not read sufficient bytes from the " + self.name + " file") + self.name, + offset + len(data), + "Could not read sufficient bytes from the " + self.name + " file", + ) return data def write(self, offset: int, data: bytes) -> None: @@ -172,8 +192,11 @@ class FileLayer(interfaces.layers.DataLayerInterface): invalid_address = offset if self.minimum_address < offset <= self.maximum_address: invalid_address = self.maximum_address + 1 - raise exceptions.InvalidAddressException(self.name, invalid_address, - "Data segment outside of the " + self.name + " file boundaries") + raise exceptions.InvalidAddressException( + self.name, + invalid_address, + "Data segment outside of the " + self.name + " file boundaries", + ) with self._lock: self._file.seek(offset) self._file.write(data) @@ -196,4 +219,4 @@ class FileLayer(interfaces.layers.DataLayerInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [requirements.StringRequirement(name = 'location', optional = False)] + return [requirements.StringRequirement(name="location", optional=False)] diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 907116e99..829354987 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -26,7 +26,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): QEVM_SUBSECTION = 0x05 QEVM_VMDESCRIPTION = 0x06 QEVM_CONFIGURATION = 0x07 - QEVM_SECTION_FOOTER = 0x7e + QEVM_SECTION_FOOTER = 0x7E HASH_PTE_SIZE_64 = 16 SEGMENT_FLAG_COMPRESS = 0x02 @@ -56,57 +56,86 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): distro_re = r"(\w+[\d{1,2}\.]*)" - pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), - re.compile(r"^pc-i440fx-[01]\.\d$"): (0xe0000000, 0xe0000000, 0x100000000), - re.compile(r"^pc-q35-\d\.\d$"): (0xb0000000, 0x80000000, 0x100000000), - re.compile(r"^microvm$"): (0xc0000000, 0xc0000000, 0x100000000), - re.compile(r"^xen$"): (0xf0000000, 0xf0000000, 0x100000000), - re.compile(r"^pc-i440fx-" + distro_re + r"$"): (0xe0000000, 0xc0000000, 0x100000000), - re.compile(r"^pc-q35-" + distro_re + r"$"): (0xb0000000, 0x80000000, 0x100000000), - } + pci_hole_table = { + re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): ( + 0xE0000000, + 0xC0000000, + 0x100000000, + ), + re.compile(r"^pc-i440fx-[01]\.\d$"): (0xE0000000, 0xE0000000, 0x100000000), + re.compile(r"^pc-q35-\d\.\d$"): (0xB0000000, 0x80000000, 0x100000000), + re.compile(r"^microvm$"): (0xC0000000, 0xC0000000, 0x100000000), + re.compile(r"^xen$"): (0xF0000000, 0xF0000000, 0x100000000), + re.compile(r"^pc-i440fx-" + distro_re + r"$"): ( + 0xE0000000, + 0xC0000000, + 0x100000000, + ), + re.compile(r"^pc-q35-" + distro_re + r"$"): ( + 0xB0000000, + 0x80000000, + 0x100000000, + ), + } - def __init__(self, - context: interfaces.context.ContextInterface, - config_path: str, - name: str, - metadata: Optional[Dict[str, Any]] = None) -> None: - self._qemu_table_name = intermed.IntermediateSymbolTable.create(context, config_path, 'generic', 'qemu') + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + self._qemu_table_name = intermed.IntermediateSymbolTable.create( + context, config_path, "generic", "qemu" + ) self._configuration = None self._architecture = None self._compressed: Set[int] = set() - self._current_segment_name = b'' + self._current_segment_name = b"" self._pci_hole_start = 0 self._pci_hole_end = 0 self._pci_hole_minimum = 0 - super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) + super().__init__( + context=context, config_path=config_path, name=name, metadata=metadata + ) @classmethod - def _check_header(cls, base_layer: interfaces.layers.DataLayerInterface, name: str = ''): + def _check_header( + cls, base_layer: interfaces.layers.DataLayerInterface, name: str = "" + ): header = base_layer.read(0, 8) - if header[:4] != b'\x51\x45\x56\x4D': - raise exceptions.LayerException(name, 'No QEMU magic bytes') - if header[4:] != b'\x00\x00\x00\x03': - raise exceptions.LayerException(name, 'Unsupported QEMU version found') + if header[:4] != b"\x51\x45\x56\x4D": + raise exceptions.LayerException(name, "No QEMU magic bytes") + if header[4:] != b"\x00\x00\x00\x03": + raise exceptions.LayerException(name, "Unsupported QEMU version found") vollog.debug("QEVM header found") - def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, name: str) -> Any: + def _read_configuration( + self, base_layer: interfaces.layers.DataLayerInterface, name: str + ) -> Any: """Reads the JSON configuration from the end of the file""" chunk_size = 4096 - data = b'' - for i in range(base_layer.maximum_address, base_layer.minimum_address, -chunk_size): + data = b"" + for i in range( + base_layer.maximum_address, base_layer.minimum_address, -chunk_size + ): if i != base_layer.maximum_address: - data = (base_layer.read(i, chunk_size) + data).rstrip(b'\x00') - if b'\x00' in data: - last_null_byte = data.rfind(b'\x00') - start_of_json = data.find(b'{', last_null_byte) + data = (base_layer.read(i, chunk_size) + data).rstrip(b"\x00") + if b"\x00" in data: + last_null_byte = data.rfind(b"\x00") + start_of_json = data.find(b"{", last_null_byte) if start_of_json >= 0: data = data[start_of_json:] return json.loads(data) # No JSON configuration found at the end of the file, return empty dict return dict() - raise exceptions.LayerException(name, "Invalid JSON configuration at the end of the file") + raise exceptions.LayerException( + name, "Invalid JSON configuration at the end of the file" + ) - def _get_ram_segments(self, index: int, page_size: int) -> Tuple[List[Tuple[int, int, int, int]], int]: + def _get_ram_segments( + self, index: int, page_size: int + ) -> Tuple[List[Tuple[int, int, int, int]], int]: """Recovers the new index and any sections of memory from a ram section""" done = None segments = [] @@ -116,7 +145,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): while not done: # Use struct.unpack here for performance improvements - addr = struct.unpack('>Q', base_layer.read(index, 8))[0] + addr = struct.unpack(">Q", base_layer.read(index, 8))[0] # Flags are stored in the n least significant bits, where n equals the bit-length of pagesize flags = addr & (page_size - 1) @@ -129,43 +158,59 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): addr += self._pci_hole_end - self._pci_hole_start if flags & self.SEGMENT_FLAG_MEM_SIZE: - namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', - offset = index, - layer_name = self._base_layer) + namelen = self._context.object( + self._qemu_table_name + constants.BANG + "unsigned char", + offset=index, + layer_name=self._base_layer, + ) while namelen != 0: - total_size = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', - offset = index + 1 + namelen, - layer_name = self._base_layer) + total_size = self._context.object( + self._qemu_table_name + constants.BANG + "unsigned long long", + offset=index + 1 + namelen, + layer_name=self._base_layer, + ) size_array[base_layer.read(index + 1, namelen)] = total_size index += 1 + namelen + 8 - namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', - offset = index, - layer_name = self._base_layer) - highest_possible_maximum = max([x[0] for x in self.pci_hole_table.values()]) + 1 - if size_array.get(b'pc.ram', highest_possible_maximum) < self._pci_hole_minimum: + namelen = self._context.object( + self._qemu_table_name + constants.BANG + "unsigned char", + offset=index, + layer_name=self._base_layer, + ) + highest_possible_maximum = ( + max([x[0] for x in self.pci_hole_table.values()]) + 1 + ) + if ( + size_array.get(b"pc.ram", highest_possible_maximum) + < self._pci_hole_minimum + ): # Turns off the pci_hole if it's not supposed to be there vollog.debug( - f"QEVM turning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}") + f"QEVM turning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}" + ) self._pci_hole_start, self._pci_hole_end = 0, 0 if flags & (self.SEGMENT_FLAG_COMPRESS | self.SEGMENT_FLAG_PAGE): if not (flags & self.SEGMENT_FLAG_CONTINUE): - namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', - offset = index, - layer_name = self._base_layer) + namelen = self._context.object( + self._qemu_table_name + constants.BANG + "unsigned char", + offset=index, + layer_name=self._base_layer, + ) self._current_segment_name = base_layer.read(index + 1, namelen) index += 1 + namelen if flags & self.SEGMENT_FLAG_COMPRESS: - if self._current_segment_name == b'pc.ram': + if self._current_segment_name == b"pc.ram": segments.append((addr, index, page_size, 1)) self._compressed.add(addr) index += 1 else: - if self._current_segment_name == b'pc.ram': + if self._current_segment_name == b"pc.ram": segments.append((addr, index, page_size, page_size)) index += page_size if flags & self.SEGMENT_FLAG_XBZRLE: - raise exceptions.LayerException(self.name, "XBZRLE compression not supported") + raise exceptions.LayerException( + self.name, "XBZRLE compression not supported" + ) if flags & self.SEGMENT_FLAG_EOS: done = True return segments, index @@ -187,88 +232,136 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if not self._architecture: self._architecture = self._fallback_determine_architecture() if self._architecture is None: - vollog.log(constants.LOGLEVEL_VV, f"QEVM architecture could not be determined") + vollog.log( + constants.LOGLEVEL_VV, + f"QEVM architecture could not be determined", + ) # Once all segments have been read, determine the PCI hole if any for regex in self.pci_hole_table: if regex.match(self._architecture): - self._pci_hole_minimum, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] - vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") + ( + self._pci_hole_minimum, + self._pci_hole_start, + self._pci_hole_end, + ) = self.pci_hole_table[regex] + vollog.log( + constants.LOGLEVEL_VVVV, + f"QEVM architecture detected as: {self._architecture}", + ) break else: - vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"QEVM unknown architecture found: {self._architecture}", + ) arch_detected = True - section_byte = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned char', - offset = index, - layer_name = self._base_layer) + section_byte = self.context.object( + self._qemu_table_name + constants.BANG + "unsigned char", + offset=index, + layer_name=self._base_layer, + ) index += 1 if section_byte == self.QEVM_CONFIGURATION: - section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', - offset = index, - layer_name = self._base_layer) - self._architecture = self.context.object(self._qemu_table_name + constants.BANG + 'string', - offset = index + 4, layer_name = self._base_layer, - max_length = section_len) + section_len = self.context.object( + self._qemu_table_name + constants.BANG + "unsigned long", + offset=index, + layer_name=self._base_layer, + ) + self._architecture = self.context.object( + self._qemu_table_name + constants.BANG + "string", + offset=index + 4, + layer_name=self._base_layer, + max_length=section_len, + ) index += 4 + section_len - elif section_byte == self.QEVM_SECTION_START or section_byte == self.QEVM_SECTION_FULL: - section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', - offset = index, - layer_name = self._base_layer) + elif ( + section_byte == self.QEVM_SECTION_START + or section_byte == self.QEVM_SECTION_FULL + ): + section_id = self.context.object( + self._qemu_table_name + constants.BANG + "unsigned long", + offset=index, + layer_name=self._base_layer, + ) current_section_id = section_id index += 4 - name_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned char', - offset = index, - layer_name = self._base_layer) + name_len = self.context.object( + self._qemu_table_name + constants.BANG + "unsigned char", + offset=index, + layer_name=self._base_layer, + ) index += 1 - name = self.context.object(self._qemu_table_name + constants.BANG + 'string', - offset = index, - layer_name = self._base_layer, - max_length = name_len) + name = self.context.object( + self._qemu_table_name + constants.BANG + "string", + offset=index, + layer_name=self._base_layer, + max_length=name_len, + ) index += name_len # instance_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', # offset = index, # layer_name = self._base_layer) index += 4 - version_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', - offset = index, - layer_name = self._base_layer) + version_id = self.context.object( + self._qemu_table_name + constants.BANG + "unsigned long", + offset=index, + layer_name=self._base_layer, + ) index += 4 # Store section info for handling QEVM_SECTION_PARTs later on - section_info[current_section_id] = {'name': name, 'version_id': version_id} + section_info[current_section_id] = { + "name": name, + "version_id": version_id, + } # Read additional data index = self.extract_data(index, name, version_id) - elif section_byte == self.QEVM_SECTION_PART or section_byte == self.QEVM_SECTION_END: - section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', - offset = index, - layer_name = self._base_layer) + elif ( + section_byte == self.QEVM_SECTION_PART + or section_byte == self.QEVM_SECTION_END + ): + section_id = self.context.object( + self._qemu_table_name + constants.BANG + "unsigned long", + offset=index, + layer_name=self._base_layer, + ) current_section_id = section_id index += 4 # Read additional data - index = self.extract_data(index, section_info[current_section_id]['name'], - section_info[current_section_id]['version_id']) + index = self.extract_data( + index, + section_info[current_section_id]["name"], + section_info[current_section_id]["version_id"], + ) elif section_byte == self.QEVM_SECTION_FOOTER: - section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', - offset = index, - layer_name = self._base_layer) + section_id = self.context.object( + self._qemu_table_name + constants.BANG + "unsigned long", + offset=index, + layer_name=self._base_layer, + ) index += 4 if section_id != current_section_id: raise exceptions.LayerException( - self._name, f'QEMU section footer mismatch: {current_section_id} and {section_id}') + self._name, + f"QEMU section footer mismatch: {current_section_id} and {section_id}", + ) elif section_byte == self.QEVM_EOF: pass else: - raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}') + raise exceptions.LayerException( + self._name, f"QEMU unknown section encountered: {section_byte}" + ) def _fallback_determine_architecture(self) -> str: - architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|\w+[\d{1,2}\.]*)' + architecture_pattern = rb"pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|\w+[\d{1,2}\.]*)" default_suffix = "-2.0" base_layer = self.context.layers[self._base_layer] vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") res = scanners.RegExScanner(architecture_pattern) - for offset in base_layer.scan(context = self.context, scanner = res): + for offset in base_layer.scan(context=self.context, scanner=res): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) architecture = regex_results.group().decode() @@ -276,80 +369,102 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): # If that does not work, look in configuration JSON for devices specific to a certain architecture architecture = None - for device in self._configuration.get('devices', []): - device_name = device.get('vmsd_name', '').lower() - if 'i440fx' in device_name or 'piix' in device_name: - architecture = 'pc-i440fx' + default_suffix + for device in self._configuration.get("devices", []): + device_name = device.get("vmsd_name", "").lower() + if "i440fx" in device_name or "piix" in device_name: + architecture = "pc-i440fx" + default_suffix break - elif 'ich9' in device_name: - architecture = 'pc-q35' + default_suffix + elif "ich9" in device_name: + architecture = "pc-q35" + default_suffix break if architecture: - vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}') + vollog.log( + constants.LOGLEVEL_VVV, + f"Architecture version unknown, default used: {default_suffix}", + ) return architecture # Still haven't found architecture, switch to fallback-method - architecture_pattern = rb'Standard PC \((i440FX|Q35)' + architecture_pattern = rb"Standard PC \((i440FX|Q35)" res = scanners.RegExScanner(architecture_pattern) - for offset in base_layer.scan(context = self.context, scanner = res): + for offset in base_layer.scan(context=self.context, scanner=res): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) - architecture = "pc-" + regex_results.groups()[0].decode().lower() + default_suffix - vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}') + architecture = ( + "pc-" + regex_results.groups()[0].decode().lower() + default_suffix + ) + vollog.log( + constants.LOGLEVEL_VVV, + f"Architecture version unknown, default used: {default_suffix}", + ) return architecture vollog.warning("Could not determine QEMU target architecture!") return None def extract_data(self, index, name, version_id): - if name == 'ram': + if name == "ram": if version_id != 4: - raise exceptions.LayerException(f"QEMU unknown RAM version_id {version_id}") - new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', 4096)) + raise exceptions.LayerException( + f"QEMU unknown RAM version_id {version_id}" + ) + new_segments, index = self._get_ram_segments( + index, self._configuration.get("page_size", 4096) + ) self._segments += new_segments - elif name == 'spapr/htab': + elif name == "spapr/htab": if version_id != 1: - raise exceptions.LayerException(f"QEMU unknown HTAB version_id {version_id}") - header = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', - offset = index, - layer_name = self._base_layer) + raise exceptions.LayerException( + f"QEMU unknown HTAB version_id {version_id}" + ) + header = self.context.object( + self._qemu_table_name + constants.BANG + "unsigned long", + offset=index, + layer_name=self._base_layer, + ) index += 4 if header == 0: htab_index = -1 htab_n_valid = 0 htab_n_invalid = 0 while htab_index != 0 and htab_n_valid != 0 and htab_n_invalid != 0: - htab = self.context.object(self._qemu_table_name + constants.BANG + 'htab', - offset = index, - layer_name = self._base_layer) + htab = self.context.object( + self._qemu_table_name + constants.BANG + "htab", + offset=index, + layer_name=self._base_layer, + ) htab_index, htab_n_valid, htab_n_invalid = htab index += 8 + (htab_n_valid * self.HASH_PTE_SIZE_64) - elif name == 'dirty-bitmap': + elif name == "dirty-bitmap": index += 1 - elif name == 'pbs-state': - section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', - offset = index, - layer_name = self._base_layer) + elif name == "pbs-state": + section_len = self.context.object( + self._qemu_table_name + constants.BANG + "unsigned long long", + offset=index, + layer_name=self._base_layer, + ) index += 8 + section_len return index - def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes: + def _decode_data( + self, data: bytes, mapped_offset: int, offset: int, output_length: int + ) -> bytes: """Takes the full segment from the base_layer that the data occurs in, checks whether it's compressed (by locating it in the segment list and verifying if that address is compressed), then reading/expanding the data, and finally cutting it to the right size. Offset may be the address requested rather than the location of the starting data. It is the responsibility of the layer to turn the provided data chunk into the right portion of data necessary. """ - page_size = self._configuration.get('page_size', 4096) + page_size = self._configuration.get("page_size", 4096) # start_offset equals the highest multiple of pagesize <= offset # (We assume that page_size is a power of 2) start_offset = offset ^ (offset & (page_size - 1)) if start_offset in self._compressed: - data = (data * page_size) - result = data[offset - start_offset:output_length + offset - start_offset] + data = data * page_size + result = data[offset - start_offset : output_length + offset - start_offset] return result - @functools.lru_cache(maxsize = 512) + @functools.lru_cache(maxsize=512) def read(self, offset: int, length: int, pad: bool = False) -> bytes: return super().read(offset, length, pad) @@ -358,16 +473,20 @@ class QemuStacker(interfaces.automagic.StackerLayerInterface): stack_order = 10 @classmethod - def stack(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: try: QemuSuspendLayer._check_header(context.layers[layer_name]) except exceptions.LayerException: return None new_name = context.layers.free_layer_name("QemuSuspendLayer") - context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name + context.config[ + interfaces.configuration.path_join(new_name, "base_layer") + ] = layer_name layer = QemuSuspendLayer(context, new_name, new_name) cls.stacker_slow_warning() return layer diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index ec7aed217..660e0a299 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -7,7 +7,10 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.configuration import requirements -from volatility3.framework.configuration.requirements import IntRequirement, TranslationLayerRequirement +from volatility3.framework.configuration.requirements import ( + IntRequirement, + TranslationLayerRequirement, +) from volatility3.framework.exceptions import InvalidAddressException from volatility3.framework.layers import linear from volatility3.framework.symbols import intermed @@ -25,35 +28,49 @@ class RegistryInvalidIndex(exceptions.LayerException): class RegistryHive(linear.LinearlyMappedLayer): - - def __init__(self, - context: interfaces.context.ContextInterface, - config_path: str, - name: str, - metadata: Optional[Dict[str, Any]] = None) -> None: - super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__( + context=context, config_path=config_path, name=name, metadata=metadata + ) self._base_layer = self.config["base_layer"] self._hive_offset = self.config["hive_offset"] self._table_name = self.config["nt_symbols"] self._page_size = 1 << 12 - self._reg_table_name = intermed.IntermediateSymbolTable.create(context, self._config_path, 'windows', - 'registry') + self._reg_table_name = intermed.IntermediateSymbolTable.create( + context, self._config_path, "windows", "registry" + ) - cmhive = self.context.object(self._table_name + constants.BANG + "_CMHIVE", self._base_layer, self._hive_offset) + cmhive = self.context.object( + self._table_name + constants.BANG + "_CMHIVE", + self._base_layer, + self._hive_offset, + ) self._cmhive_name = cmhive.get_name() self.hive = cmhive.Hive # TODO: Check the checksum - if self.hive.Signature != 0xbee0bee0: + if self.hive.Signature != 0xBEE0BEE0: raise RegistryFormatException( - self.name, f"Registry hive at {self._hive_offset} does not have a valid signature") + self.name, + f"Registry hive at {self._hive_offset} does not have a valid signature", + ) # Win10 17063 introduced the Registry process to map most hives. Check # if it exists and update RegistryHive._base_layer - for proc in pslist.PsList.list_processes(self.context, self.config['base_layer'], self.config['nt_symbols']): - proc_name = proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, errors = 'replace') + for proc in pslist.PsList.list_processes( + self.context, self.config["base_layer"], self.config["nt_symbols"] + ): + proc_name = proc.ImageFileName.cast( + "string", max_length=proc.ImageFileName.vol.count, errors="replace" + ) if proc_name == "Registry" and proc.InheritedFromUniqueProcessId == 4: proc_layer_name = proc.add_process_layer() self._base_layer = proc_layer_name @@ -66,16 +83,23 @@ class RegistryHive(linear.LinearlyMappedLayer): self._hive_maxaddr_non_volatile = self.hive.Storage[0].Length self._hive_maxaddr_volatile = self.hive.Storage[1].Length self._maxaddr = 0x80000000 | self._hive_maxaddr_volatile - vollog.log(constants.LOGLEVEL_VVVV, f"Setting hive {self.name} max address to {hex(self._maxaddr)}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Setting hive {self.name} max address to {hex(self._maxaddr)}", + ) except exceptions.InvalidAddressException: - self._hive_maxaddr_non_volatile = 0x7fffffff - self._hive_maxaddr_volatile = 0x7fffffff + self._hive_maxaddr_non_volatile = 0x7FFFFFFF + self._hive_maxaddr_volatile = 0x7FFFFFFF self._maxaddr = 0x80000000 | self._hive_maxaddr_volatile - vollog.log(constants.LOGLEVEL_VVVV, - f"Exception when setting hive {self.name} max address, using {hex(self._maxaddr)}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Exception when setting hive {self.name} max address, using {hex(self._maxaddr)}", + ) def _get_hive_maxaddr(self, volatile): - return self._hive_maxaddr_volatile if volatile else self._hive_maxaddr_non_volatile + return ( + self._hive_maxaddr_volatile if volatile else self._hive_maxaddr_non_volatile + ) def get_name(self) -> str: return self._cmhive_name or "[NONAME]" @@ -93,42 +117,54 @@ class RegistryHive(linear.LinearlyMappedLayer): def root_cell_offset(self) -> int: """Returns the offset for the root cell in this hive.""" with contextlib.suppress(InvalidAddressException): - if self._base_block.Signature.cast("string", max_length = 4, encoding = "latin-1") == 'regf': + if ( + self._base_block.Signature.cast( + "string", max_length=4, encoding="latin-1" + ) + == "regf" + ): return self._base_block.RootCell return 0x20 - def get_cell(self, cell_offset: int) -> 'objects.StructType': + def get_cell(self, cell_offset: int) -> "objects.StructType": """Returns the appropriate Cell value for a cell offset.""" # This would be an _HCELL containing CELL_DATA, but to save time we skip the size of the HCELL - cell = self._context.object(object_type = self._table_name + constants.BANG + "_CELL_DATA", - offset = cell_offset + 4, - layer_name = self.name) + cell = self._context.object( + object_type=self._table_name + constants.BANG + "_CELL_DATA", + offset=cell_offset + 4, + layer_name=self.name, + ) return cell - def get_node(self, cell_offset: int) -> 'objects.StructType': + def get_node(self, cell_offset: int) -> "objects.StructType": """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast('string', max_length = 2, encoding = 'latin-1') - if signature == 'nk': + signature = cell.cast("string", max_length=2, encoding="latin-1") + if signature == "nk": return cell.u.KeyNode - elif signature == 'sk': + elif signature == "sk": return cell.u.KeySecurity - elif signature == 'vk': + elif signature == "vk": return cell.u.KeyValue - elif signature == 'db': + elif signature == "db": # Big Data return cell.u.ValueData - elif signature == 'lf' or signature == 'lh' or signature == 'ri': + elif signature == "lf" or signature == "lh" or signature == "ri": # Fast Leaf, Hash Leaf, Index Root return cell.u.KeyIndex else: # It doesn't matter that we use KeyNode, we're just after the first two bytes - vollog.debug("Unknown Signature {} (0x{:x}) at offset {}".format(signature, cell.u.KeyNode.Signature, - cell_offset)) + vollog.debug( + "Unknown Signature {} (0x{:x}) at offset {}".format( + signature, cell.u.KeyNode.Signature, cell_offset + ) + ) return cell - def get_key(self, key: str, return_list: bool = False) -> Union[List[objects.StructType], objects.StructType]: + def get_key( + self, key: str, return_list: bool = False + ) -> Union[List[objects.StructType], objects.StructType]: """Gets a specific registry key by key path. return_list specifies whether the return result will be a single @@ -138,7 +174,7 @@ class RegistryHive(linear.LinearlyMappedLayer): node_key = [self.get_node(self.root_cell_offset)] if key.endswith("\\"): key = key[:-1] - key_array = key.split('\\') + key_array = key.split("\\") found_key: List[str] = [] while key_array and node_key: subkeys = node_key[-1].get_subkeys() @@ -152,14 +188,18 @@ class RegistryHive(linear.LinearlyMappedLayer): else: node_key = [] if not node_key: - raise KeyError("Key {} not found under {}".format(key_array[0], '\\'.join(found_key))) + raise KeyError( + "Key {} not found under {}".format(key_array[0], "\\".join(found_key)) + ) if return_list: return node_key return node_key[-1] - def visit_nodes(self, - visitor: Callable[[objects.StructType], None], - node: Optional[objects.StructType] = None) -> None: + def visit_nodes( + self, + visitor: Callable[[objects.StructType], None], + node: Optional[objects.StructType] = None, + ) -> None: """Applies a callable (visitor) to all nodes within the registry tree from a given node.""" if not node: @@ -172,22 +212,28 @@ class RegistryHive(linear.LinearlyMappedLayer): def _mask(value: int, high_bit: int, low_bit: int) -> int: """Returns the bits of a value between highbit and lowbit inclusive.""" high_mask = (2 ** (high_bit + 1)) - 1 - low_mask = (2 ** low_bit) - 1 - mask = (high_mask ^ low_mask) + low_mask = (2**low_bit) - 1 + mask = high_mask ^ low_mask # print(high_bit, low_bit, bin(mask), bin(value)) return value & mask @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - IntRequirement(name = 'hive_offset', - description = 'Offset within the base layer at which the hive lives', - default = 0, - optional = False), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), - TranslationLayerRequirement(name = 'base_layer', - description = 'Layer in which the registry hive lives', - optional = False) + IntRequirement( + name="hive_offset", + description="Offset within the base layer at which the hive lives", + default=0, + optional=False, + ), + requirements.SymbolTableRequirement( + name="nt_symbols", description="Windows kernel symbols" + ), + TranslationLayerRequirement( + name="base_layer", + description="Layer in which the registry hive lives", + optional=False, + ), ] def _translate(self, offset: int) -> int: @@ -196,15 +242,20 @@ class RegistryHive(linear.LinearlyMappedLayer): # Ignore the volatile bit when determining maxaddr validity volatile = self._mask(offset, 31, 31) >> 31 - if offset & 0x7fffffff > self._get_hive_maxaddr(volatile): - vollog.log(constants.LOGLEVEL_VVV, - "Layer {} couldn't translate offset {}, greater than {} in {} store of {}".format( - self.name, - hex(offset & 0x7fffffff), - hex(self._get_hive_maxaddr(volatile)), - "volative" if volatile else "non-volatile", - self.get_name())) - raise RegistryInvalidIndex(self.name, "Mapping request for value greater than maxaddr") + if offset & 0x7FFFFFFF > self._get_hive_maxaddr(volatile): + vollog.log( + constants.LOGLEVEL_VVV, + "Layer {} couldn't translate offset {}, greater than {} in {} store of {}".format( + self.name, + hex(offset & 0x7FFFFFFF), + hex(self._get_hive_maxaddr(volatile)), + "volative" if volatile else "non-volatile", + self.get_name(), + ), + ) + raise RegistryInvalidIndex( + self.name, "Mapping request for value greater than maxaddr" + ) storage = self.hive.Storage[volatile] dir_index = self._mask(offset, 30, 21) >> 21 @@ -215,10 +266,9 @@ class RegistryHive(linear.LinearlyMappedLayer): entry = table.Table[table_index] return entry.get_block_offset() + suboffset - def mapping(self, - offset: int, - length: int, - ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + def mapping( + self, offset: int, length: int, ignore_errors: bool = False + ) -> Iterable[Tuple[int, int, int, int, str]]: if length < 0: raise ValueError("Mapping length of RegistryHive must be positive or zero") @@ -234,7 +284,15 @@ class RegistryHive(linear.LinearlyMappedLayer): chunk_size = min(chunk_size, remaining_length, self._page_size) try: translated_offset = self._translate(current_offset) - response.append((current_offset, chunk_size, translated_offset, chunk_size, self._base_layer)) + response.append( + ( + current_offset, + chunk_size, + translated_offset, + chunk_size, + self._base_layer, + ) + ) except exceptions.LayerException: if not ignore_errors: raise @@ -246,16 +304,18 @@ class RegistryHive(linear.LinearlyMappedLayer): @property def dependencies(self) -> List[str]: """Returns a list of layer names that this layer translates onto.""" - return [self.config['base_layer']] + return [self.config["base_layer"]] def is_valid(self, offset: int, length: int = 1) -> bool: """Returns a boolean based on whether the offset is valid or not.""" with contextlib.suppress(exceptions.InvalidAddressException): # Pass this to the lower layers for now - return all([ - self.context.layers[layer].is_valid(offset, length) - for (_, _, offset, length, layer) in self.mapping(offset, length) - ]) + return all( + [ + self.context.layers[layer].is_valid(offset, length) + for (_, _, offset, length, layer) in self.mapping(offset, length) + ] + ) return False @property diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 73f59bdbd..6083e795e 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -63,10 +63,12 @@ class ResourceAccessor(object): list_handlers = True - def __init__(self, - progress_callback: Optional[constants.ProgressCallback] = None, - context: Optional[ssl.SSLContext] = None, - enable_cache: bool = True) -> None: + def __init__( + self, + progress_callback: Optional[constants.ProgressCallback] = None, + context: Optional[ssl.SSLContext] = None, + enable_cache: bool = True, + ) -> None: """Creates a resource accessor. Note: context is an SSL context, not a volatility context @@ -76,20 +78,24 @@ class ResourceAccessor(object): self._handlers = list(framework.class_subclasses(urllib.request.BaseHandler)) self._enable_cache = enable_cache if self.list_handlers: - vollog.log(constants.LOGLEVEL_VVV, - f"Available URL handlers: {', '.join([x.__name__ for x in self._handlers])}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Available URL handlers: {', '.join([x.__name__ for x in self._handlers])}", + ) self.__class__.list_handlers = False def uses_cache(self, url: str) -> bool: """Determines whether a URLs contents should be cached""" parsed_url = urllib.parse.urlparse(url) - return self._enable_cache and parsed_url.scheme not in self._non_cached_schemes() + return ( + self._enable_cache and parsed_url.scheme not in self._non_cached_schemes() + ) @staticmethod def _non_cached_schemes() -> List[str]: """Returns the list of schemes not to be cached""" - result = ['file'] + result = ["file"] for clazz in framework.class_subclasses(VolatilityHandler): result += clazz.non_cached_schemes() return result @@ -103,34 +109,52 @@ class ResourceAccessor(object): urllib.request.install_opener(urllib.request.build_opener(*self._handlers)) # Python bug 46654 - if sys.platform == 'win32': + if sys.platform == "win32": # We only need to worry about UNC paths on windows, on linux they'd be smb:// and need pysmb or similar - parsed_url = urllib.parse.urlparse(url, scheme = 'file') + parsed_url = urllib.parse.urlparse(url, scheme="file") # Only worry about file scheme URLs, make sure that there's either a host or # the unparsing left an extra slash at the start (which will get lost with urlunparse) - if parsed_url.scheme == 'file' and (parsed_url.netloc or parsed_url.path.startswith('//')): + if parsed_url.scheme == "file" and ( + parsed_url.netloc or parsed_url.path.startswith("//") + ): # Change the netloc to '/' and then prepend the netloc to the path # Urlunparse will remove extra initial slashes from path, hence setting netloc - new_url = urllib.parse.urlunparse((parsed_url.scheme, '/', - '/' + parsed_url.netloc + parsed_url.path, parsed_url.params, - parsed_url.query, parsed_url.fragment)) - vollog.log(constants.LOGLEVEL_VVVV, f'UNC path detected, converted path {url} to {new_url}') + new_url = urllib.parse.urlunparse( + ( + parsed_url.scheme, + "/", + "/" + parsed_url.netloc + parsed_url.path, + parsed_url.params, + parsed_url.query, + parsed_url.fragment, + ) + ) + vollog.log( + constants.LOGLEVEL_VVVV, + f"UNC path detected, converted path {url} to {new_url}", + ) url = new_url try: - fp = urllib.request.urlopen(url, context = self._context) + fp = urllib.request.urlopen(url, context=self._context) except error.URLError as excp: if excp.args: # TODO: As of python3.7 this can be removed - unverified_retrieval = (hasattr(ssl, "SSLCertVerificationError") and isinstance( - excp.args[0], ssl.SSLCertVerificationError)) or (isinstance(excp.args[0], ssl.SSLError) and - excp.args[0].reason == "CERTIFICATE_VERIFY_FAILED") + unverified_retrieval = ( + hasattr(ssl, "SSLCertVerificationError") + and isinstance(excp.args[0], ssl.SSLCertVerificationError) + ) or ( + isinstance(excp.args[0], ssl.SSLError) + and excp.args[0].reason == "CERTIFICATE_VERIFY_FAILED" + ) if unverified_retrieval: - vollog.warning("SSL certificate verification failed: attempting UNVERIFIED retrieval") + vollog.warning( + "SSL certificate verification failed: attempting UNVERIFIED retrieval" + ) non_verifying_ctx = ssl.SSLContext() non_verifying_ctx.check_hostname = False non_verifying_ctx.verify_mode = ssl.CERT_NONE - fp = urllib.request.urlopen(url, context = non_verifying_ctx) + fp = urllib.request.urlopen(url, context=non_verifying_ctx) else: raise excp else: @@ -144,19 +168,22 @@ class ResourceAccessor(object): if not self.uses_cache(url): # ZipExtFiles (files in zips) cannot seek, so must be cached in order to use and/or decompress - curfile = urllib.request.urlopen(url, context = self._context) + curfile = urllib.request.urlopen(url, context=self._context) else: # TODO: find a way to check if we already have this file (look at http headers?) block_size = 1028 * 8 temp_filename = os.path.join( constants.CACHE_PATH, - "data_" + hashlib.sha512(bytes(url, 'raw_unicode_escape')).hexdigest() + ".cache") + "data_" + + hashlib.sha512(bytes(url, "raw_unicode_escape")).hexdigest() + + ".cache", + ) if not os.path.exists(temp_filename): vollog.debug(f"Caching file at: {temp_filename}") try: - content_length = fp.info().get('Content-Length', -1) + content_length = fp.info().get("Content-Length", -1) except AttributeError: # If our fp doesn't have an info member, carry on gracefully content_length = -1 @@ -167,8 +194,10 @@ class ResourceAccessor(object): while block: count += len(block) if self._progress_callback: - self._progress_callback(count * 100 / max(count, int(content_length)), - f"Reading file {url}") + self._progress_callback( + count * 100 / max(count, int(content_length)), + f"Reading file {url}", + ) cache_file.write(block) block = fp.read(block_size) cache_file.close() @@ -177,7 +206,7 @@ class ResourceAccessor(object): # Re-open the cache with a different mode # Since we don't want people thinking they're able to save to the cache file, # open it in read mode only and allow breakages to happen if they wanted to write - curfile = open(temp_filename, mode = "rb") + curfile = open(temp_filename, mode="rb") # Determine whether the file is a particular type of file, and if so, open it as such IMPORTED_MAGIC = False @@ -193,13 +222,21 @@ class ResourceAccessor(object): # Only file's python has magic.detect_from_fobj if detected: - if detected.mime_type == 'application/x-xz': - curfile = cascadeCloseFile(lzma.LZMAFile(curfile, mode), curfile) - elif detected.mime_type == 'application/x-bzip2': + if detected.mime_type == "application/x-xz": + curfile = cascadeCloseFile( + lzma.LZMAFile(curfile, mode), curfile + ) + elif detected.mime_type == "application/x-bzip2": curfile = cascadeCloseFile(bz2.BZ2File(curfile, mode), curfile) - elif detected.mime_type == 'application/x-gzip': - curfile = cascadeCloseFile(gzip.GzipFile(fileobj = curfile, mode = mode), curfile) - if detected.mime_type in ['application/x-xz', 'application/x-bzip2', 'application/x-gzip']: + elif detected.mime_type == "application/x-gzip": + curfile = cascadeCloseFile( + gzip.GzipFile(fileobj=curfile, mode=mode), curfile + ) + if detected.mime_type in [ + "application/x-xz", + "application/x-bzip2", + "application/x-gzip", + ]: # Read and rewind to ensure we're inside any compressed file layers curfile.read(1) curfile.seek(0) @@ -222,7 +259,9 @@ class ResourceAccessor(object): elif extension == "bz2": curfile = cascadeCloseFile(bz2.BZ2File(curfile, mode), curfile) elif extension == "gz": - curfile = cascadeCloseFile(gzip.GzipFile(fileobj = curfile, mode = mode), curfile) + curfile = cascadeCloseFile( + gzip.GzipFile(fileobj=curfile, mode=mode), curfile + ) else: stop = True @@ -233,7 +272,6 @@ class ResourceAccessor(object): class VolatilityHandler(urllib.request.BaseHandler): - @classmethod def non_cached_schemes(cls) -> List[str]: return [] @@ -251,21 +289,27 @@ class JarHandler(VolatilityHandler): @classmethod def non_cached_schemes(cls) -> List[str]: - return ['jar'] + return ["jar"] @staticmethod def default_open(req: urllib.request.Request) -> Optional[Any]: """Handles the request if it's the jar scheme.""" - if req.type == 'jar': - subscheme, remainder = req.full_url.split(":")[1], ":".join(req.full_url.split(":")[2:]) - if subscheme != 'file': - vollog.log(constants.LOGLEVEL_VVV, f"Unsupported jar subscheme {subscheme}") + if req.type == "jar": + subscheme, remainder = req.full_url.split(":")[1], ":".join( + req.full_url.split(":")[2:] + ) + if subscheme != "file": + vollog.log( + constants.LOGLEVEL_VVV, f"Unsupported jar subscheme {subscheme}" + ) return None zipsplit = remainder.split("!") if len(zipsplit) != 2: - vollog.log(constants.LOGLEVEL_VVV, - f"Path did not contain exactly one fragment indicator: {remainder}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Path did not contain exactly one fragment indicator: {remainder}", + ) return None zippath, filepath = zipsplit @@ -276,6 +320,6 @@ class JarHandler(VolatilityHandler): class OfflineHandler(VolatilityHandler): @staticmethod def default_open(req: urllib.request.Request) -> Optional[Any]: - if constants.OFFLINE and req.type in ['http', 'https']: + if constants.OFFLINE and req.type in ["http", "https"]: raise exceptions.OfflineException(req.full_url) return None diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index ec66f2708..dd8dc46be 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -35,6 +35,7 @@ class RegExScanner(layers.ScannerInterface): The default flags include DOTALL, since the searches are through binary data and the newline character should have no specific significance in such searches""" + thread_safe = True _required_framework_version = (2, 0, 0) @@ -80,7 +81,7 @@ class MultiStringScanner(layers.ScannerInterface): def _process_trie(self, trie: Optional[Dict[int, Optional[Dict]]]) -> bytes: if trie is None or len(trie) == 1 and -1 in trie: # We've reached the end of this path, return the empty byte string - return b'' + return b"" choices = [] suffixes = [] @@ -101,16 +102,16 @@ class MultiStringScanner(layers.ScannerInterface): if len(suffixes) == 1: choices.append(suffixes[0]) elif len(suffixes) > 1: - choices.append(b'[' + b''.join(suffixes) + b']') + choices.append(b"[" + b"".join(suffixes) + b"]") if len(choices) == 0: # If there's none, return the empty byte string - response = b'' + response = b"" elif len(choices) == 1: # If there's only one return it response = choices[0] else: - response = b'(?:' + b'|'.join(choices) + b')' + response = b"(?:" + b"|".join(choices) + b")" if finished: # We finished one string, so everything after this is optional @@ -118,7 +119,9 @@ class MultiStringScanner(layers.ScannerInterface): return response - def __call__(self, data: bytes, data_offset: int) -> Generator[Tuple[int, bytes], None, None]: + def __call__( + self, data: bytes, data_offset: int + ) -> Generator[Tuple[int, bytes], None, None]: """Runs through the data looking for the needles.""" for offset, pattern in self.search(data): if offset < self.chunk_size: @@ -128,6 +131,8 @@ class MultiStringScanner(layers.ScannerInterface): if not isinstance(haystack, bytes): raise TypeError("Search haystack must be a byte string") if not self._regex: - raise ValueError("MultiRegexp cannot be used with an empty set of search strings") + raise ValueError( + "MultiRegexp cannot be used with an empty set of search strings" + ) for match in re.finditer(self._regex, haystack): yield match.start(0), match.group() diff --git a/volatility3/framework/layers/scanners/multiregexp.py b/volatility3/framework/layers/scanners/multiregexp.py index 45feb51d1..be3581f05 100644 --- a/volatility3/framework/layers/scanners/multiregexp.py +++ b/volatility3/framework/layers/scanners/multiregexp.py @@ -11,7 +11,7 @@ class MultiRegexp(object): def __init__(self) -> None: self._pattern_strings: List[bytes] = [] - self._regex = re.compile(b'') + self._regex = re.compile(b"") def add_pattern(self, pattern: bytes) -> None: self._pattern_strings.append(pattern) @@ -19,12 +19,14 @@ class MultiRegexp(object): def preprocess(self) -> None: if not self._pattern_strings: raise ValueError("No strings to compile into a regular expression") - self._regex = re.compile(b'|'.join(map(re.escape, self._pattern_strings))) + self._regex = re.compile(b"|".join(map(re.escape, self._pattern_strings))) def search(self, haystack: bytes) -> Generator[Tuple[int, bytes], None, None]: if not isinstance(haystack, bytes): raise TypeError("Search haystack must be a byte string") if not self._regex.pattern: - raise ValueError("MultiRegexp cannot be used with an empty set of search strings") + raise ValueError( + "MultiRegexp cannot be used with an empty set of search strings" + ) for match in re.finditer(self._regex, haystack): yield (match.start(0), match.group()) diff --git a/volatility3/framework/layers/segmented.py b/volatility3/framework/layers/segmented.py index 05fc01b97..beb667436 100644 --- a/volatility3/framework/layers/segmented.py +++ b/volatility3/framework/layers/segmented.py @@ -10,19 +10,25 @@ from volatility3.framework.configuration import requirements from volatility3.framework.layers import linear -class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, metaclass = ABCMeta): +class NonLinearlySegmentedLayer( + interfaces.layers.TranslationLayerInterface, metaclass=ABCMeta +): """A class to handle a single run-based layer-to-layer mapping. In the documentation "mapped address" or "mapped offset" refers to an offset once it has been mapped to the underlying layer """ - def __init__(self, - context: interfaces.context.ContextInterface, - config_path: str, - name: str, - metadata: Optional[Dict[str, Any]] = None) -> None: - super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__( + context=context, config_path=config_path, name=name, metadata=metadata + ) self._base_layer = self.config["base_layer"] self._segments: List[Tuple[int, int, int, int]] = [] @@ -45,11 +51,17 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met try: base_layer = self._context.layers[self._base_layer] return all( - [base_layer.is_valid(mapped_offset) for _i, _i, mapped_offset, _i, _s in self.mapping(offset, length)]) + [ + base_layer.is_valid(mapped_offset) + for _i, _i, mapped_offset, _i, _s in self.mapping(offset, length) + ] + ) except exceptions.InvalidAddressException: return False - def _find_segment(self, offset: int, next: bool = False) -> Tuple[int, int, int, int]: + def _find_segment( + self, offset: int, next: bool = False + ) -> Tuple[int, int, int, int]: """Finds the segment containing a given offset. Returns the segment tuple (offset, mapped_offset, length, mapped_length) @@ -59,7 +71,10 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met self._load_segments() # Find rightmost value less than or equal to x - i = bisect_right(self._segments, (offset, self.context.layers[self._base_layer].maximum_address)) + i = bisect_right( + self._segments, + (offset, self.context.layers[self._base_layer].maximum_address), + ) if i and not next: segment = self._segments[i - 1] if segment[0] <= offset < segment[0] + segment[2]: @@ -67,16 +82,17 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met if next: if i < len(self._segments): return self._segments[i] - raise exceptions.InvalidAddressException(self.name, offset, f"Invalid address at {offset:0x}") + raise exceptions.InvalidAddressException( + self.name, offset, f"Invalid address at {offset:0x}" + ) # Determines whether larger segments are in use and the offsets within them should be tracked linearly # When no decoding of the data occurs, this should be set to true _track_offset = False - def mapping(self, - offset: int, - length: int, - ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + def mapping( + self, offset: int, length: int, ignore_errors: bool = False + ) -> Iterable[Tuple[int, int, int, int, str]]: """Returns a sorted iterable of (offset, length, mapped_offset, mapped_length, layer) mappings.""" done = False @@ -84,7 +100,9 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met while not done: try: # Search for the appropriate segment that contains the current_offset - logical_offset, mapped_offset, size, mapped_size = self._find_segment(current_offset) + logical_offset, mapped_offset, size, mapped_size = self._find_segment( + current_offset + ) # If it starts before the current_offset, bring the lower edge up to the right place if current_offset > logical_offset: difference = current_offset - logical_offset @@ -98,7 +116,12 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met raise try: # Find the next valid segment after our current_offset - logical_offset, mapped_offset, size, mapped_size = self._find_segment(current_offset, next = True) + ( + logical_offset, + mapped_offset, + size, + mapped_size, + ) = self._find_segment(current_offset, next=True) # We know that the logical_offset must be greater than current_offset so skip to that value current_offset = logical_offset # If it starts too late then we're done @@ -140,16 +163,21 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [requirements.TranslationLayerRequirement(name = 'base_layer', optional = False)] + return [ + requirements.TranslationLayerRequirement(name="base_layer", optional=False) + ] -class SegmentedLayer(NonLinearlySegmentedLayer, linear.LinearlyMappedLayer, metaclass = ABCMeta): +class SegmentedLayer( + NonLinearlySegmentedLayer, linear.LinearlyMappedLayer, metaclass=ABCMeta +): _track_offset = True - def mapping(self, - offset: int, - length: int, - ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + def mapping( + self, offset: int, length: int, ignore_errors: bool = False + ) -> Iterable[Tuple[int, int, int, int, str]]: # Linear mappings must return the same length of segment as that requested - for offset, length, mapped_offset, mapped_length, layer in super().mapping(offset, length, ignore_errors): + for offset, length, mapped_offset, mapped_length, layer in super().mapping( + offset, length, ignore_errors + ): yield offset, length, mapped_offset, length, layer diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 61b13eb88..0bc1a350b 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -22,18 +22,23 @@ class VmwareLayer(segmented.SegmentedLayer): header_structure = "<4sII" group_structure = "64sQQ" - def __init__(self, - context: interfaces.context.ContextInterface, - config_path: str, - name: str, - metadata: Optional[Dict[str, Any]] = None) -> None: + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: # Construct these so we can use self.config self._context = context self._config_path = config_path self._page_size = 0x1000 - self._base_layer, self._meta_layer = self.config["base_layer"], self.config["meta_layer"] + self._base_layer, self._meta_layer = ( + self.config["base_layer"], + self.config["meta_layer"], + ) # Then call the super, which will call load_segments (which needs the base_layer before it'll work) - super().__init__(context, config_path = config_path, name = name, metadata = metadata) + super().__init__(context, config_path=config_path, name=name, metadata=metadata) def _load_segments(self) -> None: """Loads up the segments from the meta_layer.""" @@ -46,22 +51,33 @@ class VmwareLayer(segmented.SegmentedLayer): def _read_header(self) -> None: """Checks the vmware header to make sure it's valid.""" if "vmware" not in self._context.symbol_space: - self._context.symbol_space.append(native.NativeTable("vmware", native.std_ctypes)) + self._context.symbol_space.append( + native.NativeTable("vmware", native.std_ctypes) + ) meta_layer = self.context.layers.get(self._meta_layer, None) header_size = struct.calcsize(self.header_structure) data = meta_layer.read(0, header_size) magic, unknown, groupCount = struct.unpack(self.header_structure, data) - if magic not in [b"\xD0\xBE\xD2\xBE", b"\xD1\xBA\xD1\xBA", b"\xD2\xBE\xD2\xBE", b"\xD3\xBE\xD3\xBE"]: - raise VmwareFormatException(self.name, f"Wrong magic bytes for Vmware layer: {repr(magic)}") + if magic not in [ + b"\xD0\xBE\xD2\xBE", + b"\xD1\xBA\xD1\xBA", + b"\xD2\xBE\xD2\xBE", + b"\xD3\xBE\xD3\xBE", + ]: + raise VmwareFormatException( + self.name, f"Wrong magic bytes for Vmware layer: {repr(magic)}" + ) - version = magic[0] & 0xf + version = magic[0] & 0xF group_size = struct.calcsize(self.group_structure) groups = {} for group in range(groupCount): name, tag_location, _unknown = struct.unpack( - self.group_structure, meta_layer.read(header_size + (group * group_size), group_size)) + self.group_structure, + meta_layer.read(header_size + (group * group_size), group_size), + ) name = name.rstrip(b"\x00") groups[name] = tag_location memory = groups[b"memory"] @@ -75,43 +91,70 @@ class VmwareLayer(segmented.SegmentedLayer): name_len = ord(meta_layer.read(offset + 1, 1)) tags_read = (flags == 0) and (name_len == 0) if not tags_read: - name = self._context.object("vmware!string", - layer_name = self._meta_layer, - offset = offset + 2, - max_length = name_len) + name = self._context.object( + "vmware!string", + layer_name=self._meta_layer, + offset=offset + 2, + max_length=name_len, + ) indices_len = (flags >> 6) & 3 indices = [] for index in range(indices_len): indices.append( - self._context.object("vmware!unsigned int", - offset = offset + name_len + 2 + (index * index_len), - layer_name = self._meta_layer)) - data_len = flags & 0x3f + self._context.object( + "vmware!unsigned int", + offset=offset + name_len + 2 + (index * index_len), + layer_name=self._meta_layer, + ) + ) + data_len = flags & 0x3F - if data_len in [62, 63]: # Handle special data sizes that indicate a longer data stream + if data_len in [ + 62, + 63, + ]: # Handle special data sizes that indicate a longer data stream data_len = 4 if version == 0 else 8 # Read the size of the data - data_size = self._context.object(self._choose_type(data_len), - layer_name = self._meta_layer, - offset = offset + 2 + name_len + (indices_len * index_len)) + data_size = self._context.object( + self._choose_type(data_len), + layer_name=self._meta_layer, + offset=offset + 2 + name_len + (indices_len * index_len), + ) # Skip two bytes of padding (as it seems?) # Read the actual data - data = self._context.object("vmware!bytes", - layer_name = self._meta_layer, - offset = offset + 2 + name_len + (indices_len * index_len) + - 2 * data_len + 2, - length = data_size) - offset += 2 + name_len + (indices_len * index_len) + 2 * data_len + 2 + data_size + data = self._context.object( + "vmware!bytes", + layer_name=self._meta_layer, + offset=offset + + 2 + + name_len + + (indices_len * index_len) + + 2 * data_len + + 2, + length=data_size, + ) + offset += ( + 2 + + name_len + + (indices_len * index_len) + + 2 * data_len + + 2 + + data_size + ) else: # Handle regular cases - data = self._context.object(self._choose_type(data_len), - layer_name = self._meta_layer, - offset = offset + 2 + name_len + (indices_len * index_len)) + data = self._context.object( + self._choose_type(data_len), + layer_name=self._meta_layer, + offset=offset + 2 + name_len + (indices_len * index_len), + ) offset += 2 + name_len + (indices_len * index_len) + data_len tags[(name, tuple(indices))] = (flags, data) if tags[("regionsCount", ())][1] == 0: - raise VmwareFormatException(self.name, "VMware VMEM is not split into regions") + raise VmwareFormatException( + self.name, "VMware VMEM is not split into regions" + ) for region in range(tags[("regionsCount", ())][1]): offset = tags[("regionPPN", (region,))][1] * self._page_size mapped_offset = tags[("regionPageNum", (region,))][1] * self._page_size @@ -127,8 +170,8 @@ class VmwareLayer(segmented.SegmentedLayer): """This vmware translation layer always requires a separate metadata layer.""" return [ - requirements.TranslationLayerRequirement(name = 'base_layer', optional = False), - requirements.TranslationLayerRequirement(name = 'meta_layer', optional = False) + requirements.TranslationLayerRequirement(name="base_layer", optional=False), + requirements.TranslationLayerRequirement(name="meta_layer", optional=False), ] @@ -136,10 +179,12 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): stack_order = 20 @classmethod - def stack(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempt to stack this based on the starting information.""" memlayer = context.layers[layer_name] if not isinstance(memlayer, physical.FileLayer): @@ -149,32 +194,52 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): vmss = location[:-5] + ".vmss" vmsn = location[:-5] + ".vmsn" current_layer_name = context.layers.free_layer_name("VmwareMetaLayer") - current_config_path = interfaces.configuration.path_join("automagic", "layer_stacker", "stack", - current_layer_name) + current_config_path = interfaces.configuration.path_join( + "automagic", "layer_stacker", "stack", current_layer_name + ) vmss_success = False with contextlib.suppress(IOError): with resources.ResourceAccessor().open(vmss) as fp: _ = fp.read(10) - context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss - context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) + context.config[ + interfaces.configuration.path_join(current_config_path, "location") + ] = vmss + context.layers.add_layer( + physical.FileLayer(context, current_config_path, current_layer_name) + ) vmss_success = True vmsn_success = False if not vmss_success: with contextlib.suppress(IOError): _ = resources.ResourceAccessor().open(vmsn).read(10) - context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmsn - context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) + context.config[ + interfaces.configuration.path_join( + current_config_path, "location" + ) + ] = vmsn + context.layers.add_layer( + physical.FileLayer( + context, current_config_path, current_layer_name + ) + ) vmsn_success = True - vollog.log(constants.LOGLEVEL_VVVV, f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})", + ) if not vmss_success and not vmsn_success: return None new_layer_name = context.layers.free_layer_name("VmwareLayer") - context.config[interfaces.configuration.path_join(current_config_path, "base_layer")] = layer_name - context.config[interfaces.configuration.path_join(current_config_path, "meta_layer")] = current_layer_name + context.config[ + interfaces.configuration.path_join(current_config_path, "base_layer") + ] = layer_name + context.config[ + interfaces.configuration.path_join(current_config_path, "meta_layer") + ] = current_layer_name new_layer = VmwareLayer(context, current_config_path, new_layer_name) return new_layer return None diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 2b026ccd1..a04eedd87 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -6,28 +6,51 @@ import collections import collections.abc import logging import struct -from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload +from typing import ( + Any, + ClassVar, + Dict, + Iterable, + List, + Optional, + Tuple, + Type, + Union as TUnion, + overload, +) from volatility3.framework import constants, interfaces from volatility3.framework.objects import templates vollog = logging.getLogger(__name__) -DataFormatInfo = collections.namedtuple('DataFormatInfo', ['length', 'byteorder', 'signed']) +DataFormatInfo = collections.namedtuple( + "DataFormatInfo", ["length", "byteorder", "signed"] +) -def convert_data_to_value(data: bytes, struct_type: Type[TUnion[int, float, bytes, str, bool]], - data_format: DataFormatInfo) -> TUnion[int, float, bytes, str, bool]: +def convert_data_to_value( + data: bytes, + struct_type: Type[TUnion[int, float, bytes, str, bool]], + data_format: DataFormatInfo, +) -> TUnion[int, float, bytes, str, bool]: """Converts a series of bytes to a particular type of value.""" if struct_type == int: - return int.from_bytes(data, byteorder = data_format.byteorder, signed = data_format.signed) + return int.from_bytes( + data, byteorder=data_format.byteorder, signed=data_format.signed + ) if struct_type == bool: struct_format = "?" elif struct_type == float: float_vals = "zzezfzzzd" - if data_format.length > len(float_vals) or float_vals[data_format.length] not in "efd": + if ( + data_format.length > len(float_vals) + or float_vals[data_format.length] not in "efd" + ): raise ValueError("Invalid float size") - struct_format = ("<" if data_format.byteorder == 'little' else ">") + float_vals[data_format.length] + struct_format = ( + "<" if data_format.byteorder == "little" else ">" + ) + float_vals[data_format.length] elif struct_type in [bytes, str]: struct_format = str(data_format.length) + "s" else: @@ -36,29 +59,40 @@ def convert_data_to_value(data: bytes, struct_type: Type[TUnion[int, float, byte return struct.unpack(struct_format, data)[0] -def convert_value_to_data(value: TUnion[int, float, bytes, str, bool], struct_type: Type[TUnion[int, float, bytes, str, - bool]], - data_format: DataFormatInfo) -> bytes: +def convert_value_to_data( + value: TUnion[int, float, bytes, str, bool], + struct_type: Type[TUnion[int, float, bytes, str, bool]], + data_format: DataFormatInfo, +) -> bytes: """Converts a particular value to a series of bytes.""" if not isinstance(value, struct_type): - raise TypeError(f"Written value is not of the correct type for {struct_type.__name__}") + raise TypeError( + f"Written value is not of the correct type for {struct_type.__name__}" + ) if struct_type == int and isinstance(value, int): # Doubling up on the isinstance is for mypy - return int.to_bytes(value, - length = data_format.length, - byteorder = data_format.byteorder, - signed = data_format.signed) + return int.to_bytes( + value, + length=data_format.length, + byteorder=data_format.byteorder, + signed=data_format.signed, + ) if struct_type == bool: struct_format = "?" elif struct_type == float: float_vals = "zzezfzzzd" - if data_format.length > len(float_vals) or float_vals[data_format.length] not in "efd": + if ( + data_format.length > len(float_vals) + or float_vals[data_format.length] not in "efd" + ): raise ValueError("Invalid float size") - struct_format = ("<" if data_format.byteorder == 'little' else ">") + float_vals[data_format.length] + struct_format = ( + "<" if data_format.byteorder == "little" else ">" + ) + float_vals[data_format.length] elif struct_type in [bytes, str]: if isinstance(value, str): - value = bytes(value, 'latin-1') + value = bytes(value, "latin-1") struct_format = str(data_format.length) + "s" else: raise TypeError(f"Cannot construct struct format for type {type(struct_type)}") @@ -70,7 +104,6 @@ class Void(interfaces.objects.ObjectInterface): """Returns an object to represent void/unknown types.""" class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): - @classmethod def size(cls, template: interfaces.objects.Template) -> int: """Dummy size for Void objects. @@ -95,20 +128,33 @@ class Function(interfaces.objects.ObjectInterface): class PrimitiveObject(interfaces.objects.ObjectInterface): """PrimitiveObject is an interface for any objects that should simulate a Python primitive.""" + _struct_type: ClassVar[Type] = int - def __init__(self, context: interfaces.context.ContextInterface, type_name: str, - object_info: interfaces.objects.ObjectInformation, data_format: DataFormatInfo) -> None: - super().__init__(context = context, type_name = type_name, object_info = object_info, data_format = data_format) + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + data_format: DataFormatInfo, + ) -> None: + super().__init__( + context=context, + type_name=type_name, + object_info=object_info, + data_format=data_format, + ) self._data_format = data_format - def __new__(cls: Type, - context: interfaces.context.ContextInterface, - type_name: str, - object_info: interfaces.objects.ObjectInformation, - data_format: DataFormatInfo, - new_value: TUnion[int, float, bool, bytes, str] = None, - **kwargs) -> 'PrimitiveObject': + def __new__( + cls: Type, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + data_format: DataFormatInfo, + new_value: TUnion[int, float, bool, bytes, str] = None, + **kwargs, + ) -> "PrimitiveObject": """Creates the appropriate class and returns it so that the native type is inherited. @@ -135,26 +181,38 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): for k, v in self._vol.maps[-1].items(): if k not in ["context", "data_format", "object_info", "type_name"]: kwargs[k] = v - kwargs['new_value'] = self.__new_value - return (self._context, self._vol.maps[-3]['type_name'], self._vol.maps[-2], self._data_format), kwargs + kwargs["new_value"] = self.__new_value + return ( + self._context, + self._vol.maps[-3]["type_name"], + self._vol.maps[-2], + self._data_format, + ), kwargs @classmethod - def _unmarshall(cls, context: interfaces.context.ContextInterface, data_format: DataFormatInfo, - object_info: interfaces.objects.ObjectInformation) -> TUnion[int, float, bool, bytes, str]: + def _unmarshall( + cls, + context: interfaces.context.ContextInterface, + data_format: DataFormatInfo, + object_info: interfaces.objects.ObjectInformation, + ) -> TUnion[int, float, bool, bytes, str]: # Don't try to lookup a 0 length data format, incase it's at an invalid offset. Length 0 means b'' - data = b'' + data = b"" if data_format.length > 0: - data = context.layers.read(object_info.layer_name, object_info.offset, data_format.length) + data = context.layers.read( + object_info.layer_name, object_info.offset, data_format.length + ) return convert_data_to_value(data, cls._struct_type, data_format) class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): - @classmethod def size(cls, template: interfaces.objects.Template) -> int: """Returns the size of the templated object.""" return template.vol.data_format.length - def write(self, value: TUnion[int, float, bool, bytes, str]) -> interfaces.objects.ObjectInterface: + def write( + self, value: TUnion[int, float, bool, bytes, str] + ) -> interfaces.objects.ObjectInterface: """Writes the object into the layer of the context at the current offset.""" data = convert_value_to_data(value, self._struct_type, self._data_format) @@ -167,6 +225,7 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): # https://mail.python.org/pipermail/python-dev/2004-February/042537.html class Boolean(PrimitiveObject, int): """Primitive Object that handles boolean types.""" + _struct_type: ClassVar[Type] = int @@ -176,35 +235,44 @@ class Integer(PrimitiveObject, int): class Float(PrimitiveObject, float): """Primitive Object that handles double or floating point numbers.""" + _struct_type: ClassVar[Type] = float class Char(PrimitiveObject, int): """Primitive Object that handles characters.""" + _struct_type: ClassVar[Type] = int class Bytes(PrimitiveObject, bytes): """Primitive Object that handles specific series of bytes.""" + _struct_type: ClassVar[Type] = bytes - def __init__(self, - context: interfaces.context.ContextInterface, - type_name: str, - object_info: interfaces.objects.ObjectInformation, - length: int = 1) -> None: - super().__init__(context = context, - type_name = type_name, - object_info = object_info, - data_format = DataFormatInfo(length, "big", False)) - self._vol['length'] = length + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + length: int = 1, + ) -> None: + super().__init__( + context=context, + type_name=type_name, + object_info=object_info, + data_format=DataFormatInfo(length, "big", False), + ) + self._vol["length"] = length - def __new__(cls: Type, - context: interfaces.context.ContextInterface, - type_name: str, - object_info: interfaces.objects.ObjectInformation, - length: int = 1, - **kwargs) -> 'Bytes': + def __new__( + cls: Type, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + length: int = 1, + **kwargs, + ) -> "Bytes": """Creates the appropriate class and returns it so that the native type is inherited. @@ -213,11 +281,15 @@ class Bytes(PrimitiveObject, bytes): override __new__ """ return cls._struct_type.__new__( - cls, cls._unmarshall(context, data_format = DataFormatInfo(length, "big", False), - object_info = object_info)) + cls, + cls._unmarshall( + context, + data_format=DataFormatInfo(length, "big", False), + object_info=object_info, + ), + ) class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): - @classmethod def size(cls, template: interfaces.objects.Template) -> int: return template.vol.length @@ -230,31 +302,38 @@ class String(PrimitiveObject, str): max_length: specifies the maximum possible length that the string could hold within memory (for multibyte characters, this will not be the maximum length of the string) """ + _struct_type: ClassVar[Type] = str - def __init__(self, - context: interfaces.context.ContextInterface, - type_name: str, - object_info: interfaces.objects.ObjectInformation, - max_length: int = 1, - encoding: str = "utf-8", - errors: str = "strict") -> None: - super().__init__(context = context, - type_name = type_name, - object_info = object_info, - data_format = DataFormatInfo(max_length, "big", False)) + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + max_length: int = 1, + encoding: str = "utf-8", + errors: str = "strict", + ) -> None: + super().__init__( + context=context, + type_name=type_name, + object_info=object_info, + data_format=DataFormatInfo(max_length, "big", False), + ) self._vol["max_length"] = max_length - self._vol['encoding'] = encoding - self._vol['errors'] = errors + self._vol["encoding"] = encoding + self._vol["errors"] = errors - def __new__(cls: Type, - context: interfaces.context.ContextInterface, - type_name: str, - object_info: interfaces.objects.ObjectInformation, - max_length: int = 1, - encoding: str = "utf-8", - errors: str = "strict", - **kwargs) -> 'String': + def __new__( + cls: Type, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + max_length: int = 1, + encoding: str = "utf-8", + errors: str = "strict", + **kwargs, + ) -> "String": """Creates the appropriate class and returns it so that the native type is inherited. @@ -264,20 +343,24 @@ class String(PrimitiveObject, str): """ params = {} if encoding: - params['encoding'] = encoding + params["encoding"] = encoding if errors: - params['errors'] = errors + params["errors"] = errors # Pass the encoding and error parameters to the string constructor to appropriately encode the string value = cls._struct_type.__new__( cls, - cls._unmarshall(context, data_format = DataFormatInfo(max_length, "big", False), object_info = object_info), - **params) - if value.find('\x00') >= 0: - value = value[:value.find('\x00')] + cls._unmarshall( + context, + data_format=DataFormatInfo(max_length, "big", False), + object_info=object_info, + ), + **params, + ) + if value.find("\x00") >= 0: + value = value[: value.find("\x00")] return value class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): - @classmethod def size(cls, template: interfaces.objects.Template) -> int: """Returns the size of the templated object.""" @@ -287,19 +370,30 @@ class String(PrimitiveObject, str): class Pointer(Integer): """Pointer which points to another object.""" - def __init__(self, - context: interfaces.context.ContextInterface, - type_name: str, - object_info: interfaces.objects.ObjectInformation, - data_format: DataFormatInfo, - subtype: Optional[templates.ObjectTemplate] = None) -> None: - super().__init__(context = context, object_info = object_info, type_name = type_name, data_format = data_format) - self._vol['subtype'] = subtype + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + data_format: DataFormatInfo, + subtype: Optional[templates.ObjectTemplate] = None, + ) -> None: + super().__init__( + context=context, + object_info=object_info, + type_name=type_name, + data_format=data_format, + ) + self._vol["subtype"] = subtype self._cache: Dict[str, interfaces.objects.ObjectInterface] = {} @classmethod - def _unmarshall(cls, context: interfaces.context.ContextInterface, data_format: DataFormatInfo, - object_info: interfaces.objects.ObjectInformation) -> Any: + def _unmarshall( + cls, + context: interfaces.context.ContextInterface, + data_format: DataFormatInfo, + object_info: interfaces.objects.ObjectInformation, + ) -> Any: """Ensure that pointer values always fall within the domain of the layer they're constructed on. @@ -312,10 +406,12 @@ class Pointer(Integer): raise ValueError("Pointers cannot have signed values") mask = context.layers[object_info.native_layer_name].address_mask data = context.layers.read(object_info.layer_name, object_info.offset, length) - value = int.from_bytes(data, byteorder = endian, signed = signed) + value = int.from_bytes(data, byteorder=endian, signed=signed) return value & mask - def dereference(self, layer_name: Optional[str] = None) -> interfaces.objects.ObjectInterface: + def dereference( + self, layer_name: Optional[str] = None + ) -> interfaces.objects.ObjectInterface: """Dereferences the pointer. Layer_name is identifies the appropriate layer within the @@ -332,12 +428,15 @@ class Pointer(Integer): layer_name = layer_name or self.vol.native_layer_name mask = self._context.layers[layer_name].address_mask offset = self & mask - self._cache[layer_name] = self.vol.subtype(context = self._context, - object_info = interfaces.objects.ObjectInformation( - layer_name = layer_name, - offset = offset, - parent = self, - size = self.vol.subtype.size)) + self._cache[layer_name] = self.vol.subtype( + context=self._context, + object_info=interfaces.objects.ObjectInformation( + layer_name=layer_name, + offset=offset, + parent=self, + size=self.vol.subtype.size, + ), + ) return self._cache[layer_name] def is_readable(self, layer_name: Optional[str] = None) -> bool: @@ -349,109 +448,136 @@ class Pointer(Integer): def __getattr__(self, attr: str) -> Any: """Convenience function to access unknown attributes by getting them from the subtype object.""" - if attr in ['vol', '_vol', '_cache']: + if attr in ["vol", "_vol", "_cache"]: raise AttributeError("Pointer not initialized before use") return getattr(self.dereference(), attr) def has_member(self, member_name: str) -> bool: """Returns whether the dereferenced type has this member.""" - return self._vol['subtype'].has_member(member_name) + return self._vol["subtype"].has_member(member_name) class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): - @classmethod def size(cls, template: interfaces.objects.Template) -> int: return Integer.VolTemplateProxy.size(template) @classmethod - def children(cls, template: interfaces.objects.Template) -> List[interfaces.objects.Template]: + def children( + cls, template: interfaces.objects.Template + ) -> List[interfaces.objects.Template]: """Returns the children of the template.""" - if 'subtype' in template.vol: + if "subtype" in template.vol: return [template.vol.subtype] return [] @classmethod - def replace_child(cls, template: interfaces.objects.Template, old_child: interfaces.objects.Template, - new_child: interfaces.objects.Template) -> None: + def replace_child( + cls, + template: interfaces.objects.Template, + old_child: interfaces.objects.Template, + new_child: interfaces.objects.Template, + ) -> None: """Substitutes the old_child for the new_child.""" - if 'subtype' in template.vol: + if "subtype" in template.vol: if template.vol.subtype == old_child: - template.update_vol(subtype = new_child) + template.update_vol(subtype=new_child) @classmethod - def has_member(cls, template: interfaces.objects.Template, member_name: str) -> bool: - return template.vol['subtype'].has_member(member_name) + def has_member( + cls, template: interfaces.objects.Template, member_name: str + ) -> bool: + return template.vol["subtype"].has_member(member_name) class BitField(interfaces.objects.ObjectInterface, int): """Object containing a field which is made up of bits rather than whole bytes.""" - def __init__(self, - context: interfaces.context.ContextInterface, - type_name: str, - object_info: interfaces.objects.ObjectInformation, - base_type: interfaces.objects.Template, - start_bit: int = 0, - end_bit: int = 0) -> None: + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + base_type: interfaces.objects.Template, + start_bit: int = 0, + end_bit: int = 0, + ) -> None: super().__init__(context, type_name, object_info) - self._vol['base_type'] = base_type - self._vol['start_bit'] = start_bit - self._vol['end_bit'] = end_bit + self._vol["base_type"] = base_type + self._vol["start_bit"] = start_bit + self._vol["end_bit"] = end_bit - def __new__(cls, - context: interfaces.context.ContextInterface, - type_name: str, - object_info: interfaces.objects.ObjectInformation, - base_type: interfaces.objects.Template, - start_bit: int = 0, - end_bit: int = 0, - **kwargs) -> 'BitField': - value = base_type(context = context, object_info = object_info) + def __new__( + cls, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + base_type: interfaces.objects.Template, + start_bit: int = 0, + end_bit: int = 0, + **kwargs, + ) -> "BitField": + value = base_type(context=context, object_info=object_info) return int.__new__(cls, ((value & ((1 << end_bit) - 1)) >> start_bit)) # type: ignore def write(self, value): raise NotImplementedError("Writing to BitFields is not yet implemented") class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): - @classmethod def size(cls, template: interfaces.objects.Template) -> int: return template.vol.base_type.size @classmethod - def children(cls, template: interfaces.objects.Template) -> List[interfaces.objects.Template]: + def children( + cls, template: interfaces.objects.Template + ) -> List[interfaces.objects.Template]: """Returns the children of the template.""" - if 'base_type' in template.vol: + if "base_type" in template.vol: return [template.vol.base_type] return [] @classmethod - def replace_child(cls, template: interfaces.objects.Template, old_child: interfaces.objects.Template, - new_child: interfaces.objects.Template) -> None: + def replace_child( + cls, + template: interfaces.objects.Template, + old_child: interfaces.objects.Template, + new_child: interfaces.objects.Template, + ) -> None: """Substitutes the old_child for the new_child.""" - if 'base_type' in template.vol: + if "base_type" in template.vol: if template.vol.base_type == old_child: - template.update_vol(base_type = new_child) + template.update_vol(base_type=new_child) class Enumeration(interfaces.objects.ObjectInterface, int): """Returns an object made up of choices.""" - def __new__(cls, context: interfaces.context.ContextInterface, type_name: str, - object_info: interfaces.objects.ObjectInformation, base_type: interfaces.objects.Template, - choices: Dict[str, int], **kwargs) -> 'Enumeration': - value = base_type(context = context, object_info = object_info) + def __new__( + cls, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + base_type: interfaces.objects.Template, + choices: Dict[str, int], + **kwargs, + ) -> "Enumeration": + value = base_type(context=context, object_info=object_info) return int.__new__(cls, value) # type: ignore - def __init__(self, context: interfaces.context.ContextInterface, type_name: str, - object_info: interfaces.objects.ObjectInformation, base_type: Integer, choices: Dict[str, - int]) -> None: + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + base_type: Integer, + choices: Dict[str, int], + ) -> None: super().__init__(context, type_name, object_info) self._inverse_choices = self._generate_inverse_choices(choices) - self._vol['choices'] = choices + self._vol["choices"] = choices - self._vol['base_type'] = base_type + self._vol["base_type"] = base_type def __eq__(self, other): """An enumeration must be equivalent to its value, even if the other value is not an enumeration""" @@ -470,7 +596,9 @@ class Enumeration(interfaces.objects.ObjectInterface, int): # Technically this shouldn't be a problem, but since we inverse cache # and can't map one value to two possibilities we throw an exception during build # We can remove/work around this if it proves a common issue - raise ValueError(f"Enumeration value {v} duplicated as {k} and {inverse_choices[v]}") + raise ValueError( + f"Enumeration value {v} duplicated as {k} and {inverse_choices[v]}" + ) inverse_choices[v] = k return inverse_choices @@ -489,7 +617,7 @@ class Enumeration(interfaces.objects.ObjectInterface, int): @property def choices(self) -> Dict[str, int]: - return self._vol['choices'] + return self._vol["choices"] @property def is_valid_choice(self) -> bool: @@ -498,59 +626,73 @@ class Enumeration(interfaces.objects.ObjectInterface, int): def __getattr__(self, attr: str) -> str: """Returns the value for a specific name.""" - if attr in self._vol['choices']: - return self._vol['choices'][attr] - raise AttributeError(f"Unknown attribute {attr} for Enumeration {self._vol['type_name']}") + if attr in self._vol["choices"]: + return self._vol["choices"][attr] + raise AttributeError( + f"Unknown attribute {attr} for Enumeration {self._vol['type_name']}" + ) def write(self, value: bytes): raise NotImplementedError("Writing to Enumerations is not yet implemented") class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): - _methods = ['lookup'] + _methods = ["lookup"] @classmethod def lookup(cls, template: interfaces.objects.Template, value: int) -> str: """Looks up an individual value and returns the associated name.""" - _inverse_choices = Enumeration._generate_inverse_choices(template.vol['choices']) + _inverse_choices = Enumeration._generate_inverse_choices( + template.vol["choices"] + ) if value in _inverse_choices: return _inverse_choices[value] - raise ValueError("The value of the enumeration is outside the possible choices") + raise ValueError( + "The value of the enumeration is outside the possible choices" + ) @classmethod def size(cls, template: interfaces.objects.Template) -> int: - return template.vol['base_type'].size + return template.vol["base_type"].size @classmethod - def children(cls, template: interfaces.objects.Template) -> List[interfaces.objects.Template]: + def children( + cls, template: interfaces.objects.Template + ) -> List[interfaces.objects.Template]: """Returns the children of the template.""" - if 'base_type' in template.vol: + if "base_type" in template.vol: return [template.vol.base_type] return [] @classmethod - def replace_child(cls, template: interfaces.objects.Template, old_child: interfaces.objects.Template, - new_child: interfaces.objects.Template) -> None: + def replace_child( + cls, + template: interfaces.objects.Template, + old_child: interfaces.objects.Template, + new_child: interfaces.objects.Template, + ) -> None: """Substitutes the old_child for the new_child.""" - if 'base_type' in template.vol: + if "base_type" in template.vol: if template.vol.base_type == old_child: - template.update_vol(base_type = new_child) + template.update_vol(base_type=new_child) class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): """Object which can contain a fixed number of an object type.""" - def __init__(self, - context: interfaces.context.ContextInterface, - type_name: str, - object_info: interfaces.objects.ObjectInformation, - count: int = 0, - subtype: templates.ObjectTemplate = None) -> None: - super().__init__(context = context, type_name = type_name, object_info = object_info) - self._vol['count'] = count - self._vol['subtype'] = subtype - self._vol['size'] = 0 + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + count: int = 0, + subtype: templates.ObjectTemplate = None, + ) -> None: + super().__init__(context=context, type_name=type_name, object_info=object_info) + self._vol["count"] = count + self._vol["subtype"] = subtype + self._vol["size"] = 0 if subtype is not None: - self._vol['size'] = count * subtype.size + self._vol["size"] = count * subtype.size # This overrides the little known Sequence.count(val) that returns the number of items in the list that match val # Changing the name would be confusing (since we use count of an array everywhere else), so this is more important @@ -562,59 +704,71 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): @count.setter def count(self, value: int) -> None: """Sets the count to a specific value.""" - self._vol['count'] = value - self._vol['size'] = value * self._vol['subtype'].size + self._vol["count"] = value + self._vol["size"] = value * self._vol["subtype"].size def __repr__(self) -> str: """Describes the object appropriately""" return AggregateType.__repr__(self) class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): - @classmethod def size(cls, template: interfaces.objects.Template) -> int: """Returns the size of the array, based on the count and the subtype.""" - if 'subtype' not in template.vol and 'count' not in template.vol: - raise ValueError("Array ObjectTemplate must be provided a count and subtype") - return template.vol.get('subtype', None).size * template.vol.get('count', 0) + if "subtype" not in template.vol and "count" not in template.vol: + raise ValueError( + "Array ObjectTemplate must be provided a count and subtype" + ) + return template.vol.get("subtype", None).size * template.vol.get("count", 0) @classmethod - def children(cls, template: interfaces.objects.Template) -> List[interfaces.objects.Template]: + def children( + cls, template: interfaces.objects.Template + ) -> List[interfaces.objects.Template]: """Returns the children of the template.""" - if 'subtype' in template.vol: + if "subtype" in template.vol: return [template.vol.subtype] return [] @classmethod - def replace_child(cls, template: interfaces.objects.Template, old_child: interfaces.objects.Template, - new_child: interfaces.objects.Template) -> None: + def replace_child( + cls, + template: interfaces.objects.Template, + old_child: interfaces.objects.Template, + new_child: interfaces.objects.Template, + ) -> None: """Substitutes the old_child for the new_child.""" - if 'subtype' in template.vol: - if template.vol['subtype'] == old_child: - template.update_vol(subtype = new_child) + if "subtype" in template.vol: + if template.vol["subtype"] == old_child: + template.update_vol(subtype=new_child) @classmethod - def relative_child_offset(cls, template: interfaces.objects.Template, child: str) -> int: + def relative_child_offset( + cls, template: interfaces.objects.Template, child: str + ) -> int: """Returns the relative offset from the head of the parent data to the child member.""" - if 'subtype' in template.vol and child == 'subtype': + if "subtype" in template.vol and child == "subtype": return 0 raise IndexError(f"Member not present in array template: {child}") @classmethod - def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: + def child_template( + cls, template: interfaces.objects.Template, child: str + ) -> interfaces.objects.Template: """Returns the template of the child member.""" - if 'subtype' in template.vol and child == 'subtype': + if "subtype" in template.vol and child == "subtype": return template.vol.subtype raise IndexError(f"Member not present in array template: {child}") + @overload + def __getitem__(self, i: int) -> interfaces.objects.Template: + ... @overload - def __getitem__(self, i: int) -> interfaces.objects.Template: ... - - @overload - def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: ... + def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: + ... def __getitem__(self, i): """Returns the i-th item from the array.""" @@ -628,12 +782,13 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): series = [series] for index in series: object_info = interfaces.objects.ObjectInformation( - layer_name = self.vol.layer_name, - offset = mask & (self.vol.offset + (self.vol.subtype.size * index)), - parent = self, - native_layer_name = self.vol.native_layer_name, - size = self.vol.subtype.size) - result += [self.vol.subtype(context = self._context, object_info = object_info)] + layer_name=self.vol.layer_name, + offset=mask & (self.vol.offset + (self.vol.subtype.size * index)), + parent=self, + native_layer_name=self.vol.native_layer_name, + size=self.vol.subtype.size, + ) + result += [self.vol.subtype(context=self._context, object_info=object_info)] if not return_list: return result[0] return result @@ -657,14 +812,21 @@ class AggregateType(interfaces.objects.ObjectInterface): each one could overload a valid member. """ - def __init__(self, context: interfaces.context.ContextInterface, type_name: str, - object_info: interfaces.objects.ObjectInformation, size: int, - members: Dict[str, Tuple[int, interfaces.objects.Template]]) -> None: - super().__init__(context = context, - type_name = type_name, - object_info = object_info, - size = size, - members = members) + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + size: int, + members: Dict[str, Tuple[int, interfaces.objects.Template]], + ) -> None: + super().__init__( + context=context, + type_name=type_name, + object_info=object_info, + size=size, + members=members, + ) # self._check_members(members) self._concrete_members: Dict[str, Dict] = {} @@ -675,7 +837,7 @@ class AggregateType(interfaces.objects.ObjectInterface): def __repr__(self) -> str: """Describes the object appropriately""" - extras = member_name = '' + extras = member_name = "" if self.vol.native_layer_name != self.vol.layer_name: extras += f" (Native: {self.vol.native_layer_name})" if self.vol.member_name: @@ -683,25 +845,30 @@ class AggregateType(interfaces.objects.ObjectInterface): return f"<{self.__class__.__name__} {self.vol.type_name}{member_name}: {self.vol.layer_name} @ 0x{self.vol.offset:x} #{self.vol.size}{extras}>" class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): - @classmethod def size(cls, template: interfaces.objects.Template) -> int: """Method to return the size of this type.""" - if template.vol.get('size', None) is None: + if template.vol.get("size", None) is None: raise ValueError("ObjectTemplate not provided with a size") return template.vol.size @classmethod - def children(cls, template: interfaces.objects.Template) -> List[interfaces.objects.Template]: + def children( + cls, template: interfaces.objects.Template + ) -> List[interfaces.objects.Template]: """Method to list children of a template.""" return [member for _, member in template.vol.members.values()] @classmethod - def replace_child(cls, template: interfaces.objects.Template, old_child: interfaces.objects.Template, - new_child: interfaces.objects.Template) -> None: + def replace_child( + cls, + template: interfaces.objects.Template, + old_child: interfaces.objects.Template, + new_child: interfaces.objects.Template, + ) -> None: """Replace a child elements within the arguments handed to the template.""" - for member in template.vol.members.get('members', {}): + for member in template.vol.members.get("members", {}): relative_offset, member_template = template.vol.members[member] if member_template == old_child: # Members will give access to the mutable members list, @@ -711,10 +878,12 @@ class AggregateType(interfaces.objects.ObjectInterface): # If there's trouble with mutability, consider making update_vol return a clone with the changes # (there will be a few other places that will be necessary) and/or making these part of the # permanent dictionaries rather than the non-cloneable ones - template.update_vol(members = tmp_list) + template.update_vol(members=tmp_list) @classmethod - def relative_child_offset(cls, template: interfaces.objects.Template, child: str) -> int: + def relative_child_offset( + cls, template: interfaces.objects.Template, child: str + ) -> int: """Returns the relative offset of a child to its parent.""" retlist = template.vol.members.get(child, None) if retlist is None: @@ -722,67 +891,83 @@ class AggregateType(interfaces.objects.ObjectInterface): return retlist[0] @classmethod - def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: + def child_template( + cls, template: interfaces.objects.Template, child: str + ) -> interfaces.objects.Template: """Returns the template of a child to its parent.""" retlist = template.vol.members.get(child, None) if retlist is None: raise IndexError(f"Member not present in template: {child}") return retlist[1] - @classmethod - def has_member(cls, template: interfaces.objects.Template, member_name: str) -> bool: + def has_member( + cls, template: interfaces.objects.Template, member_name: str + ) -> bool: """Returns whether the object would contain a member called member_name.""" return member_name in template.vol.members @classmethod - def _check_members(cls, members: Dict[str, Tuple[int, interfaces.objects.Template]]) -> None: + def _check_members( + cls, members: Dict[str, Tuple[int, interfaces.objects.Template]] + ) -> None: # Members should be an iterable mapping of symbol names to tuples of (relative_offset, ObjectTemplate) # An object template is a callable that when called with a context, offset, layer_name and type_name # We duplicate this code to avoid polluting the methodspace - agg_name = 'AggregateType' + agg_name = "AggregateType" for agg_type in AggregateTypes: if isinstance(cls, agg_type): agg_name = agg_type.__name__ - assert isinstance(members, collections.abc.Mapping), f"{agg_name} members parameter must be a mapping: {type(members)}" - assert all([(isinstance(member, tuple) and len(member) == 2) for member in members.values()]), f"{agg_name} members must be a tuple of relative_offsets and templates" + assert isinstance( + members, collections.abc.Mapping + ), f"{agg_name} members parameter must be a mapping: {type(members)}" + assert all( + [ + (isinstance(member, tuple) and len(member) == 2) + for member in members.values() + ] + ), f"{agg_name} members must be a tuple of relative_offsets and templates" - def member(self, attr: str = 'member') -> object: + def member(self, attr: str = "member") -> object: """Specifically named method for retrieving members.""" return self.__getattr__(attr) def __getattr__(self, attr: str) -> Any: """Method for accessing members of the type.""" - if attr in ['_concrete_members', 'vol']: + if attr in ["_concrete_members", "vol"]: raise AttributeError("Object has not been properly initialized") if attr in self._concrete_members: return self._concrete_members[attr] if attr.startswith("_") and not attr.startswith("__") and "__" in attr: - attr = attr[attr.find("__", 1):] # See issue #522 + attr = attr[attr.find("__", 1) :] # See issue #522 if attr in self.vol.members: mask = self._context.layers[self.vol.layer_name].address_mask relative_offset, template = self.vol.members[attr] if isinstance(template, templates.ReferenceTemplate): template = self._context.symbol_space.get_type(template.vol.type_name) - object_info = interfaces.objects.ObjectInformation(layer_name = self.vol.layer_name, - offset = mask & (self.vol.offset + relative_offset), - member_name = attr, - parent = self, - native_layer_name = self.vol.native_layer_name, - size = template.size) - member = template(context = self._context, object_info = object_info) + object_info = interfaces.objects.ObjectInformation( + layer_name=self.vol.layer_name, + offset=mask & (self.vol.offset + relative_offset), + member_name=attr, + parent=self, + native_layer_name=self.vol.native_layer_name, + size=template.size, + ) + member = template(context=self._context, object_info=object_info) self._concrete_members[attr] = member return member # We duplicate this code to avoid polluting the methodspace - agg_name = 'AggregateType' + agg_name = "AggregateType" for agg_type in AggregateTypes: if isinstance(self, agg_type): agg_name = agg_type.__name__ - raise AttributeError(f"{agg_name} has no attribute: {self.vol.type_name}.{attr}") + raise AttributeError( + f"{agg_name} has no attribute: {self.vol.type_name}.{attr}" + ) # Disable messing around with setattr until the consequences have been considered properly # For example pdbutil constructs objects and then sets values for them @@ -801,12 +986,13 @@ class AggregateType(interfaces.objects.ObjectInterface): def write(self, value): # We duplicate this code to avoid polluting the methodspace - agg_name = 'AggregateType' + agg_name = "AggregateType" for agg_type in AggregateTypes: if isinstance(self, agg_type): agg_name = agg_type.__name__ raise TypeError( - f"{agg_name}s cannot be written to directly, individual members must be written instead") + f"{agg_name}s cannot be written to directly, individual members must be written instead" + ) class StructType(AggregateType): @@ -821,4 +1007,4 @@ class ClassType(AggregateType): pass -AggregateTypes = {StructType: 'struct', UnionType: 'union', ClassType: 'class'} +AggregateTypes = {StructType: "struct", UnionType: "union", ClassType: "class"} diff --git a/volatility3/framework/objects/templates.py b/volatility3/framework/objects/templates.py index e8b523373..7782029bc 100644 --- a/volatility3/framework/objects/templates.py +++ b/volatility3/framework/objects/templates.py @@ -22,13 +22,22 @@ class ObjectTemplate(interfaces.objects.Template): * etc """ - def __init__(self, object_class: Type[interfaces.objects.ObjectInterface], type_name: str, **arguments) -> None: - arguments['object_class'] = object_class - super().__init__(type_name = type_name, **arguments) + def __init__( + self, + object_class: Type[interfaces.objects.ObjectInterface], + type_name: str, + **arguments, + ) -> None: + arguments["object_class"] = object_class + super().__init__(type_name=type_name, **arguments) proxy_cls = self.vol.object_class.VolTemplateProxy for method_name in proxy_cls._methods: - setattr(self, method_name, functools.partial(getattr(proxy_cls, method_name), self)) + setattr( + self, + method_name, + functools.partial(getattr(proxy_cls, method_name), self), + ) @property def size(self) -> int: @@ -54,28 +63,39 @@ class ObjectTemplate(interfaces.objects.Template): plateProxy`)""" return self.vol.object_class.VolTemplateProxy.child_template(self, child) - def replace_child(self, old_child: interfaces.objects.Template, new_child: interfaces.objects.Template) -> None: + def replace_child( + self, + old_child: interfaces.objects.Template, + new_child: interfaces.objects.Template, + ) -> None: """Replaces `old_child` for `new_child` in the templated object's child list (see :class:`~volatility3.framework.interfaces.objects.ObjectInterf ace.VolTemplateProxy`)""" - return self.vol.object_class.VolTemplateProxy.replace_child(self, old_child, new_child) + return self.vol.object_class.VolTemplateProxy.replace_child( + self, old_child, new_child + ) def has_member(self, member_name: str) -> bool: """Returns whether the object would contain a member called member_name.""" return self.vol.object_class.VolTemplateProxy.has_member(self, member_name) - def __call__(self, context: interfaces.context.ContextInterface, - object_info: interfaces.objects.ObjectInformation) -> interfaces.objects.ObjectInterface: + def __call__( + self, + context: interfaces.context.ContextInterface, + object_info: interfaces.objects.ObjectInformation, + ) -> interfaces.objects.ObjectInterface: """Constructs the object. Returns: an object adhering to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface` """ arguments: Dict[str, Any] = {} for arg in self.vol: - if arg != 'object_class': + if arg != "object_class": arguments[arg] = self.vol[arg] - return self.vol.object_class(context = context, object_info = object_info, **arguments) + return self.vol.object_class( + context=context, object_info=object_info, **arguments + ) class ReferenceTemplate(interfaces.objects.Template): @@ -99,8 +119,10 @@ class ReferenceTemplate(interfaces.objects.Template): table_name = type_name[0] symbol_name = type_name[-1] raise exceptions.SymbolError( - symbol_name, table_name, - f"Template contains no information about its structure: {self.vol.type_name}") + symbol_name, + table_name, + f"Template contains no information about its structure: {self.vol.type_name}", + ) size: ClassVar[Any] = property(_unresolved) replace_child: ClassVar[Any] = _unresolved @@ -108,6 +130,10 @@ class ReferenceTemplate(interfaces.objects.Template): child_template: ClassVar[Any] = _unresolved has_member: ClassVar[Any] = _unresolved - def __call__(self, context: interfaces.context.ContextInterface, object_info: interfaces.objects.ObjectInformation): + def __call__( + self, + context: interfaces.context.ContextInterface, + object_info: interfaces.objects.ObjectInformation, + ): template = context.symbol_space.get_type(self.vol.type_name) - return template(context = context, object_info = object_info) + return template(context=context, object_info=object_info) diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index 9c4b59575..177074cd7 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -7,9 +7,9 @@ from typing import Optional, Union from volatility3.framework import interfaces, objects, constants -def array_to_string(array: 'objects.Array', - count: Optional[int] = None, - errors: str = 'replace') -> interfaces.objects.ObjectInterface: +def array_to_string( + array: "objects.Array", count: Optional[int] = None, errors: str = "replace" +) -> interfaces.objects.ObjectInterface: """Takes a volatility Array of characters and returns a string.""" # TODO: Consider checking the Array's target is a native char if count is None: @@ -17,28 +17,35 @@ def array_to_string(array: 'objects.Array', if not isinstance(array, objects.Array): raise TypeError("Array_to_string takes an Array of char") - return array.cast("string", max_length = count, errors = errors) + return array.cast("string", max_length=count, errors=errors) -def pointer_to_string(pointer: 'objects.Pointer', count: int, errors: str = 'replace'): +def pointer_to_string(pointer: "objects.Pointer", count: int, errors: str = "replace"): """Takes a volatility Pointer to characters and returns a string.""" if not isinstance(pointer, objects.Pointer): raise TypeError("pointer_to_string takes a Pointer") if count < 1: raise ValueError("pointer_to_string requires a positive count") char = pointer.dereference() - return char.cast("string", max_length = count, errors = errors) + return char.cast("string", max_length=count, errors=errors) -def array_of_pointers(array: interfaces.objects.ObjectInterface, count: int, - subtype: Union[str, interfaces.objects.Template], - context: interfaces.context.ContextInterface) -> interfaces.objects.ObjectInterface: +def array_of_pointers( + array: interfaces.objects.ObjectInterface, + count: int, + subtype: Union[str, interfaces.objects.Template], + context: interfaces.context.ContextInterface, +) -> interfaces.objects.ObjectInterface: """Takes an object, and recasts it as an array of pointers to subtype.""" symbol_table = array.vol.type_name.split(constants.BANG)[0] if isinstance(subtype, str) and context is not None: subtype = context.symbol_space.get_type(subtype) if not isinstance(subtype, interfaces.objects.Template) or subtype is None: - raise TypeError("Subtype must be a valid template (or string name of an object template)") - subtype_pointer = context.symbol_space.get_type(symbol_table + constants.BANG + "pointer") - subtype_pointer.update_vol(subtype = subtype) - return array.cast("array", count = count, subtype = subtype_pointer) + raise TypeError( + "Subtype must be a valid template (or string name of an object template)" + ) + subtype_pointer = context.symbol_space.get_type( + symbol_table + constants.BANG + "pointer" + ) + subtype_pointer.update_vol(subtype=subtype) + return array.cast("array", count=count, subtype=subtype_pointer) diff --git a/volatility3/framework/plugins/__init__.py b/volatility3/framework/plugins/__init__.py index 7dbce5208..5c3a03152 100644 --- a/volatility3/framework/plugins/__init__.py +++ b/volatility3/framework/plugins/__init__.py @@ -15,11 +15,14 @@ from volatility3.framework import interfaces, automagic, exceptions, constants vollog = logging.getLogger(__name__) -def construct_plugin(context: interfaces.context.ContextInterface, - automagics: List[interfaces.automagic.AutomagicInterface], - plugin: Type[interfaces.plugins.PluginInterface], base_config_path: str, - progress_callback: constants.ProgressCallback, - open_method: Type[interfaces.plugins.FileHandlerInterface]) -> interfaces.plugins.PluginInterface: +def construct_plugin( + context: interfaces.context.ContextInterface, + automagics: List[interfaces.automagic.AutomagicInterface], + plugin: Type[interfaces.plugins.PluginInterface], + base_config_path: str, + progress_callback: constants.ProgressCallback, + open_method: Type[interfaces.plugins.FileHandlerInterface], +) -> interfaces.plugins.PluginInterface: """Constructs a plugin object based on the parameters. Clever magic figures out how to fulfill each requirement that might not be fulfilled @@ -35,9 +38,17 @@ def construct_plugin(context: interfaces.context.ContextInterface, Returns: The constructed plugin object """ - errors = automagic.run(automagics, context, plugin, base_config_path, progress_callback = progress_callback) + errors = automagic.run( + automagics, + context, + plugin, + base_config_path, + progress_callback=progress_callback, + ) # Plugins always get their configuration stored under their plugin name - plugin_config_path = interfaces.configuration.path_join(base_config_path, plugin.__name__) + plugin_config_path = interfaces.configuration.path_join( + base_config_path, plugin.__name__ + ) # Check all the requirements and/or go back to the automagic step unsatisfied = plugin.unsatisfied(context, plugin_config_path) @@ -45,10 +56,12 @@ def construct_plugin(context: interfaces.context.ContextInterface, for error in errors: error_string = [x for x in error.format_exception_only()][-1] vollog.warning(f"Automagic exception occurred: {error_string[:-1]}") - vollog.log(constants.LOGLEVEL_V, "".join(error.format(chain = True))) + vollog.log(constants.LOGLEVEL_V, "".join(error.format(chain=True))) raise exceptions.UnsatisfiedException(unsatisfied) - constructed = plugin(context, plugin_config_path, progress_callback = progress_callback) + constructed = plugin( + context, plugin_config_path, progress_callback=progress_callback + ) if open_method: constructed.set_open_method(open_method) return constructed diff --git a/volatility3/framework/plugins/banners.py b/volatility3/framework/plugins/banners.py index ac2006207..b3c2fd3a5 100644 --- a/volatility3/framework/plugins/banners.py +++ b/volatility3/framework/plugins/banners.py @@ -19,32 +19,47 @@ class Banners(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [requirements.TranslationLayerRequirement(name = 'primary', description = 'Memory layer to scan')] + return [ + requirements.TranslationLayerRequirement( + name="primary", description="Memory layer to scan" + ) + ] def _generator(self): - layer = self.context.layers[self.config['primary']] + layer = self.context.layers[self.config["primary"]] if isinstance(layer, layers.intel.Intel): - layer = self.context.layers[layer.config['memory_layer']] + layer = self.context.layers[layer.config["memory_layer"]] for offset, banner in self.locate_banners(self.context, layer.name): yield 0, (offset, banner) @classmethod - def locate_banners(cls, context: interfaces.context.ContextInterface, layer_name: str): + def locate_banners( + cls, context: interfaces.context.ContextInterface, layer_name: str + ): """Identifies banners from a memory image""" layer = context.layers[layer_name] for offset in layer.scan( - context = context, - scanner = scanners.RegExScanner(rb"(Linux version|Darwin Kernel Version) [0-9]+\.[0-9]+\.[0-9]+")): - data = layer.read(offset, 0xfff) - data_index = data.find(b'\x00') + context=context, + scanner=scanners.RegExScanner( + rb"(Linux version|Darwin Kernel Version) [0-9]+\.[0-9]+\.[0-9]+" + ), + ): + data = layer.read(offset, 0xFFF) + data_index = data.find(b"\x00") if data_index > 0: data = data[:data_index].strip() failed = [ - char for char in data - if char not in b' #()+,;/-.0123456789:@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~' + char + for char in data + if char + not in b" #()+,;/-.0123456789:@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~" ] if not failed: - yield format_hints.Hex(offset), str(data, encoding = 'latin-1', errors = '?') + yield format_hints.Hex(offset), str( + data, encoding="latin-1", errors="?" + ) def run(self): - return renderers.TreeGrid([("Offset", format_hints.Hex), ("Banner", str)], self._generator()) + return renderers.TreeGrid( + [("Offset", format_hints.Hex), ("Banner", str)], self._generator() + ) diff --git a/volatility3/framework/plugins/configwriter.py b/volatility3/framework/plugins/configwriter.py index f0979eb4d..eca01a84a 100644 --- a/volatility3/framework/plugins/configwriter.py +++ b/volatility3/framework/plugins/configwriter.py @@ -22,25 +22,36 @@ class ConfigWriter(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.BooleanRequirement(name = 'extra', - description = 'Outputs whole configuration tree', - default = False, - optional = True) + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.BooleanRequirement( + name="extra", + description="Outputs whole configuration tree", + default=False, + optional=True, + ), ] def _generator(self): filename = "config.json" config = dict(self.build_configuration()) - if self.config.get('extra', False): - vollog.debug("Outputting additional information, this will NOT work with the -c option") + if self.config.get("extra", False): + vollog.debug( + "Outputting additional information, this will NOT work with the -c option" + ) config = dict(self.context.config) filename = "config.extra" try: with self.open(filename) as file_data: - file_data.write(bytes(json.dumps(config, sort_keys = True, indent = 2), 'raw_unicode_escape')) + file_data.write( + bytes( + json.dumps(config, sort_keys=True, indent=2), + "raw_unicode_escape", + ) + ) except Exception as excp: vollog.warning(f"Unable to JSON encode configuration: {excp}") diff --git a/volatility3/framework/plugins/frameworkinfo.py b/volatility3/framework/plugins/frameworkinfo.py index 63ba24d09..39b7f3bdc 100644 --- a/volatility3/framework/plugins/frameworkinfo.py +++ b/volatility3/framework/plugins/frameworkinfo.py @@ -20,19 +20,19 @@ class FrameworkInfo(plugins.PluginInterface): def _generator(self): categories = { - 'Automagic': interfaces.automagic.AutomagicInterface, - 'Requirement': interfaces.configuration.RequirementInterface, - 'Layer': interfaces.layers.DataLayerInterface, - 'LayerStacker': interfaces.automagic.StackerLayerInterface, - 'Object': interfaces.objects.ObjectInterface, - 'Plugin': interfaces.plugins.PluginInterface, - 'Renderer': interfaces.renderers.Renderer + "Automagic": interfaces.automagic.AutomagicInterface, + "Requirement": interfaces.configuration.RequirementInterface, + "Layer": interfaces.layers.DataLayerInterface, + "LayerStacker": interfaces.automagic.StackerLayerInterface, + "Object": interfaces.objects.ObjectInterface, + "Plugin": interfaces.plugins.PluginInterface, + "Renderer": interfaces.renderers.Renderer, } for category, module_interface in categories.items(): - yield (0, (category, )) + yield (0, (category,)) for clazz in framework.class_subclasses(module_interface): - yield (1, (clazz.__name__, )) + yield (1, (clazz.__name__,)) def run(self): return renderers.TreeGrid([("Data", str)], self._generator()) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index efffa9b87..4f07bd5a8 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -27,41 +27,53 @@ class IsfInfo(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ListRequirement(name = 'filter', - description = 'String that must be present in the file URI to display the ISF', - optional = True, - default = []), - requirements.URIRequirement(name = 'isf', - description = "Specific ISF file to process", - default = None, - optional = True), - requirements.BooleanRequirement(name = 'validate', - description = 'Validate against schema if possible', - default = False, - optional = True), - requirements.VersionRequirement(name = 'SQLiteCache', - component = symbol_cache.SqliteCache, - version = (1, 0, 0)), - requirements.BooleanRequirement(name = 'live', - description = 'Traverse all files, rather than use the cache', - default = False, - optional = True) + requirements.ListRequirement( + name="filter", + description="String that must be present in the file URI to display the ISF", + optional=True, + default=[], + ), + requirements.URIRequirement( + name="isf", + description="Specific ISF file to process", + default=None, + optional=True, + ), + requirements.BooleanRequirement( + name="validate", + description="Validate against schema if possible", + default=False, + optional=True, + ), + requirements.VersionRequirement( + name="SQLiteCache", + component=symbol_cache.SqliteCache, + version=(1, 0, 0), + ), + requirements.BooleanRequirement( + name="live", + description="Traverse all files, rather than use the cache", + default=False, + optional=True, + ), ] @classmethod def list_all_isf_files(cls) -> Generator[str, None, None]: """Lists all the ISF files that can be found""" for symbol_path in symbols.__path__: - for root, dirs, files in os.walk(symbol_path, followlinks = True): + for root, dirs, files in os.walk(symbol_path, followlinks=True): for filename in files: base_name = os.path.join(root, filename) - if filename.endswith('zip'): - with zipfile.ZipFile(base_name, 'r') as zfile: + if filename.endswith("zip"): + with zipfile.ZipFile(base_name, "r") as zfile: for name in zfile.namelist(): for extension in constants.ISF_EXTENSIONS: # By ending with an extension (and therefore, not /), we should not return any directories if name.endswith(extension): - yield "jar:file:" + str(pathlib.Path(base_name)) + "!" + name + yield "jar:file:" + str( + pathlib.Path(base_name) + ) + "!" + name else: for extension in constants.ISF_EXTENSIONS: @@ -69,81 +81,123 @@ class IsfInfo(plugins.PluginInterface): yield pathlib.Path(base_name).as_uri() def _generator(self): - if self.config.get('isf', None) is not None: - file_list = [self.config['isf']] + if self.config.get("isf", None) is not None: + file_list = [self.config["isf"]] else: file_list = list(self.list_all_isf_files()) # Filter the files filtered_list = [] - if not len(self.config['filter']): + if not len(self.config["filter"]): filtered_list = file_list else: for isf_file in file_list: - for filter_item in self.config['filter']: + for filter_item in self.config["filter"]: if filter_item in isf_file: filtered_list.append(isf_file) try: import jsonschema - if not self.config['validate']: + + if not self.config["validate"]: raise ImportError # Act as if we couldn't import if validation is turned off def check_valid(data): return "True" if schemas.validate(data, True) else "False" + except ImportError: def check_valid(data): return "Unknown" - if self.config['live']: + if self.config["live"]: # Process the filtered list for entry in filtered_list: num_types = num_enums = num_bases = num_symbols = 0 valid = "Unknown" - with resources.ResourceAccessor().open(url = entry) as fp: + with resources.ResourceAccessor().open(url=entry) as fp: try: data = json.load(fp) - num_symbols = len(data.get('symbols', [])) - num_types = len(data.get('user_types', [])) - num_enums = len(data.get('enums', [])) - num_bases = len(data.get('base_types', [])) + num_symbols = len(data.get("symbols", [])) + num_types = len(data.get("user_types", [])) + num_enums = len(data.get("enums", [])) + num_bases = len(data.get("base_types", [])) - identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + identifiers_path = os.path.join( + constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + ) identifier_cache = symbol_cache.SqliteCache(identifiers_path) - identifier = identifier_cache.get_identifier(location = entry) + identifier = identifier_cache.get_identifier(location=entry) if identifier: - identifier = identifier.decode('utf-8', errors = 'replace') + identifier = identifier.decode("utf-8", errors="replace") else: identifier = renderers.NotAvailableValue() valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {entry}") - yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) + yield ( + 0, + ( + entry, + valid, + num_bases, + num_types, + num_symbols, + num_enums, + identifier, + ), + ) else: - identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + identifiers_path = os.path.join( + constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + ) cache = symbol_cache.SqliteCache(identifiers_path) - valid = 'Unknown' + valid = "Unknown" for identifier, location in cache.get_identifier_dictionary().items(): - num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) + ( + num_bases, + num_types, + num_enums, + num_symbols, + ) = cache.get_location_statistics(location) if identifier: json_hash = cache.get_hash(location) if json_hash and json_hash in schemas.cached_validations: - valid = 'True (cached)' - if self.config['validate']: + valid = "True (cached)" + if self.config["validate"]: # Even if we're not live, if we've been explicitly asked to validate, then do-so - with resources.ResourceAccessor().open(url = location) as fp: + with resources.ResourceAccessor().open(url=location) as fp: try: data = json.load(fp) valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {location}") - yield (0, (location, valid, num_bases, num_types, num_symbols, num_enums, str(identifier))) + yield ( + 0, + ( + location, + valid, + num_bases, + num_types, + num_symbols, + num_enums, + str(identifier), + ), + ) # Try to open the file, load it as JSON, read the data from it def run(self): - return renderers.TreeGrid([("URI", str), ("Valid", str), - ("Number of base_types", int), ("Number of types", int), ("Number of symbols", int), - ("Number of enums", int), ("Identifying information", str)], self._generator()) + return renderers.TreeGrid( + [ + ("URI", str), + ("Valid", str), + ("Number of base_types", int), + ("Number of types", int), + ("Number of symbols", int), + ("Number of enums", int), + ("Identifying information", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 0068ec224..1bee5f20d 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -23,32 +23,40 @@ class LayerWriter(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', description = 'Memory layer for the kernel'), - requirements.IntRequirement(name = 'block_size', - description = "Size of blocks to copy over", - default = cls.default_block_size, - optional = True), - requirements.BooleanRequirement(name = 'list', - description = 'List available layers', - default = False, - optional = True), + requirements.TranslationLayerRequirement( + name="primary", description="Memory layer for the kernel" + ), + requirements.IntRequirement( + name="block_size", + description="Size of blocks to copy over", + default=cls.default_block_size, + optional=True, + ), + requirements.BooleanRequirement( + name="list", + description="List available layers", + default=False, + optional=True, + ), requirements.ListRequirement( - name = 'layers', - element_type = str, - description = 'Names of layers to write (defaults to the highest non-mapped layer)', - default = None, - optional = True) + name="layers", + element_type=str, + description="Names of layers to write (defaults to the highest non-mapped layer)", + default=None, + optional=True, + ), ] @classmethod def write_layer( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - preferred_name: str, - open_method: Type[plugins.FileHandlerInterface], - chunk_size: Optional[int] = None, - progress_callback: Optional[constants.ProgressCallback] = None) -> Optional[plugins.FileHandlerInterface]: + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + preferred_name: str, + open_method: Type[plugins.FileHandlerInterface], + chunk_size: Optional[int] = None, + progress_callback: Optional[constants.ProgressCallback] = None, + ) -> Optional[plugins.FileHandlerInterface]: """Produces a FileHandler from the named layer in the provided context or None on failure Args: @@ -70,42 +78,48 @@ class LayerWriter(plugins.PluginInterface): file_handle = open_method(preferred_name) for i in range(0, layer.maximum_address, chunk_size): current_chunk_size = min(chunk_size, layer.maximum_address - i) - data = layer.read(i, current_chunk_size, pad = True) + data = layer.read(i, current_chunk_size, pad=True) file_handle.write(data) if progress_callback: - progress_callback((i / layer.maximum_address) * 100, f'Writing layer {layer_name}') + progress_callback( + (i / layer.maximum_address) * 100, f"Writing layer {layer_name}" + ) return file_handle def _generator(self): - if self.config['list']: + if self.config["list"]: for name in self.context.layers: - yield 0, (name, ) + yield 0, (name,) else: # Choose the most recently added layer that isn't virtual - if not self.config['layers']: - self.config['layers'] = [] + if not self.config["layers"]: + self.config["layers"] = [] for name in self.context.layers: - if not self.context.layers[name].metadata.get('mapped', False): - self.config['layers'] = [name] + if not self.context.layers[name].metadata.get("mapped", False): + self.config["layers"] = [name] - for name in self.config['layers']: + for name in self.config["layers"]: # Check the layer exists and validate the output file if name not in self.context.layers: - yield 0, (f'Layer Name {name} does not exist', ) + yield 0, (f"Layer Name {name} does not exist",) else: - output_name = self.config.get('output', ".".join([name, "raw"])) + output_name = self.config.get("output", ".".join([name, "raw"])) try: - file_handle = self.write_layer(self.context, - name, - output_name, - self.open, - self.config.get('block_size', self.default_block_size), - progress_callback = self._progress_callback) + file_handle = self.write_layer( + self.context, + name, + output_name, + self.open, + self.config.get("block_size", self.default_block_size), + progress_callback=self._progress_callback, + ) file_handle.close() except IOError as excp: - yield 0, (f"Layer cannot be written to {self.config['output_name']}: {excp}", ) + yield 0, ( + f"Layer cannot be written to {self.config['output_name']}: {excp}", + ) - yield 0, (f'Layer has been written to {output_name}', ) + yield 0, (f"Layer has been written to {output_name}",) def _generate_layers(self): """List layer names from this run""" @@ -113,6 +127,8 @@ class LayerWriter(plugins.PluginInterface): yield (0, (name, self.context.layers[name].__class__.__name__)) def run(self): - if self.config['list']: - return renderers.TreeGrid([("Layer name", str), ('Layer type', str)], self._generate_layers()) + if self.config["list"]: + return renderers.TreeGrid( + [("Layer name", str), ("Layer type", str)], self._generate_layers() + ) return renderers.TreeGrid([("Status", str)], self._generator()) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index 7f606115c..b7dc2c16b 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -26,18 +26,27 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), ] def _generator(self, tasks): vmlinux = self.context.modules[self.config["kernel"]] - is_32bit = not symbols.symbol_table_is_64bit(self.context, vmlinux.symbol_table_name) + is_32bit = not symbols.symbol_table_is_64bit( + self.context, vmlinux.symbol_table_name + ) if is_32bit: pack_format = "I" bash_json_file = "bash32" @@ -45,10 +54,13 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): pack_format = "Q" bash_json_file = "bash64" - bash_table_name = BashIntermedSymbols.create(self.context, self.config_path, "linux", bash_json_file) + bash_table_name = BashIntermedSymbols.create( + self.context, self.config_path, "linux", bash_json_file + ) - ts_offset = self.context.symbol_space.get_type(bash_table_name + constants.BANG + - "hist_entry").relative_child_offset("timestamp") + ts_offset = self.context.symbol_space.get_type( + bash_table_name + constants.BANG + "hist_entry" + ).relative_child_offset("timestamp") for task in tasks: task_name = utility.array_to_string(task.comm) @@ -64,44 +76,61 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): bang_addrs = [] # find '#' values on the heap - for address in proc_layer.scan(self.context, - scanners.BytesScanner(b"#"), - sections = task.get_process_memory_sections(heap_only = True)): + for address in proc_layer.scan( + self.context, + scanners.BytesScanner(b"#"), + sections=task.get_process_memory_sections(heap_only=True), + ): bang_addrs.append(struct.pack(pack_format, address)) history_entries = [] if bang_addrs: - for address, _ in proc_layer.scan(self.context, - scanners.MultiStringScanner(bang_addrs), - sections = task.get_process_memory_sections(heap_only = True)): - hist = self.context.object(bash_table_name + constants.BANG + "hist_entry", - offset = address - ts_offset, - layer_name = proc_layer_name) + for address, _ in proc_layer.scan( + self.context, + scanners.MultiStringScanner(bang_addrs), + sections=task.get_process_memory_sections(heap_only=True), + ): + hist = self.context.object( + bash_table_name + constants.BANG + "hist_entry", + offset=address - ts_offset, + layer_name=proc_layer_name, + ) if hist.is_valid(): history_entries.append(hist) - for hist in sorted(history_entries, key = lambda x: x.get_time_as_integer()): - yield (0, (task.pid, task_name, hist.get_time_object(), hist.get_command())) + for hist in sorted(history_entries, key=lambda x: x.get_time_as_integer()): + yield ( + 0, + (task.pid, task_name, hist.get_time_object(), hist.get_command()), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - return renderers.TreeGrid([("PID", int), ("Process", str), ("CommandTime", datetime.datetime), - ("Command", str)], - self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("CommandTime", datetime.datetime), + ("Command", str), + ], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), + ) def generate_timeline(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for row in self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func)): + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ): _depth, row_data = row - description = f"{row_data[0]} ({row_data[1]}): \"{row_data[3]}\"" + description = f'{row_data[0]} ({row_data[1]}): "{row_data[3]}"' yield (description, timeliner.TimeLinerType.CREATED, row_data[2]) diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index 4cd065c7e..c177ee642 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -23,8 +23,11 @@ class Check_afinfo(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), ] # returns whether the symbol is found within the kernel (system.map) or not @@ -40,7 +43,7 @@ class Check_afinfo(plugins.PluginInterface): continue if check == "write": - addr = var_ops.member(attr = 'write') + addr = var_ops.member(attr="write") else: addr = getattr(var_ops, check) @@ -48,12 +51,16 @@ class Check_afinfo(plugins.PluginInterface): yield check, addr def _check_afinfo(self, var_name, var, op_members, seq_members): - for hooked_member, hook_address in self._check_members(var.seq_fops, var_name, op_members): + for hooked_member, hook_address in self._check_members( + var.seq_fops, var_name, op_members + ): yield var_name, hooked_member, hook_address # newer kernels if var.has_member("seq_ops"): - for hooked_member, hook_address in self._check_members(var.seq_ops, var_name, seq_members): + for hooked_member, hook_address in self._check_members( + var.seq_ops, var_name, seq_members + ): yield var_name, hooked_member, hook_address # this is the most commonly hooked member by rootkits, so a force a check on it @@ -62,13 +69,21 @@ class Check_afinfo(plugins.PluginInterface): def _generator(self): - vmlinux = self.context.modules[self.config['kernel']] + vmlinux = self.context.modules[self.config["kernel"]] - op_members = vmlinux.get_type('file_operations').members - seq_members = vmlinux.get_type('seq_operations').members + op_members = vmlinux.get_type("file_operations").members + seq_members = vmlinux.get_type("seq_operations").members tcp = ("tcp_seq_afinfo", ["tcp6_seq_afinfo", "tcp4_seq_afinfo"]) - udp = ("udp_seq_afinfo", ["udplite6_seq_afinfo", "udp6_seq_afinfo", "udplite4_seq_afinfo", "udp4_seq_afinfo"]) + udp = ( + "udp_seq_afinfo", + [ + "udplite6_seq_afinfo", + "udp6_seq_afinfo", + "udplite4_seq_afinfo", + "udp4_seq_afinfo", + ], + ) protocols = [tcp, udp] for (struct_type, global_vars) in protocols: @@ -79,12 +94,22 @@ class Check_afinfo(plugins.PluginInterface): except exceptions.SymbolError: continue - global_var = vmlinux.object(object_type = struct_type, offset = global_var.address) + global_var = vmlinux.object( + object_type=struct_type, offset=global_var.address + ) - for name, member, address in self._check_afinfo(global_var_name, global_var, op_members, seq_members): + for name, member, address in self._check_afinfo( + global_var_name, global_var, op_members, seq_members + ): yield 0, (name, member, format_hints.Hex(address)) def run(self): - return renderers.TreeGrid([("Symbol Name", str), ("Member", str), ("Handler Address", format_hints.Hex)], - self._generator()) + return renderers.TreeGrid( + [ + ("Symbol Name", str), + ("Member", str), + ("Handler Address", format_hints.Hex), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 9bc1a067d..6d4e2bc8a 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -19,13 +19,18 @@ class Check_creds(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), ] def _generator(self): - vmlinux = self.context.modules[self.config['kernel']] + vmlinux = self.context.modules[self.config["kernel"]] type_task = vmlinux.get_type("task_struct") diff --git a/volatility3/framework/plugins/linux/check_idt.py b/volatility3/framework/plugins/linux/check_idt.py index 1764b6364..cc3a08933 100644 --- a/volatility3/framework/plugins/linux/check_idt.py +++ b/volatility3/framework/plugins/linux/check_idt.py @@ -15,27 +15,38 @@ vollog = logging.getLogger(__name__) class Check_idt(interfaces.plugins.PluginInterface): - """ Checks if the IDT has been altered """ + """Checks if the IDT has been altered""" _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (2, 0, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), ] def _generator(self): - vmlinux = self.context.modules[self.config['kernel']] + vmlinux = self.context.modules[self.config["kernel"]] modules = lsmod.Lsmod.list_modules(self.context, vmlinux.name) - handlers = linux.LinuxUtilities.generate_kernel_handler_info(self.context, vmlinux.name, modules) + handlers = linux.LinuxUtilities.generate_kernel_handler_info( + self.context, vmlinux.name, modules + ) - is_32bit = not symbols.symbol_table_is_64bit(self.context, vmlinux.symbol_table_name) + is_32bit = not symbols.symbol_table_is_64bit( + self.context, vmlinux.symbol_table_name + ) idt_table_size = 256 @@ -59,11 +70,13 @@ class Check_idt(interfaces.plugins.PluginInterface): addrs = vmlinux.object_from_symbol("idt_table") - table = vmlinux.object(object_type = 'array', - offset = addrs.vol.offset, - subtype = vmlinux.get_type(idt_type), - count = idt_table_size, - absolute = True) + table = vmlinux.object( + object_type="array", + offset=addrs.vol.offset, + subtype=vmlinux.get_type(idt_type), + count=idt_table_size, + absolute=True, + ) for i in check_idxs: ent = table[i] @@ -86,10 +99,27 @@ class Check_idt(interfaces.plugins.PluginInterface): idt_addr = idt_addr & address_mask - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address(vmlinux, handlers, idt_addr) + module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( + vmlinux, handlers, idt_addr + ) - yield (0, [format_hints.Hex(i), format_hints.Hex(idt_addr), module_name, symbol_name]) + yield ( + 0, + [ + format_hints.Hex(i), + format_hints.Hex(idt_addr), + module_name, + symbol_name, + ], + ) def run(self): - return renderers.TreeGrid([("Index", format_hints.Hex), ("Address", format_hints.Hex), ("Module", str), - ("Symbol", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Index", format_hints.Hex), + ("Address", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/linux/check_modules.py b/volatility3/framework/plugins/linux/check_modules.py index 2c478cebf..766858888 100644 --- a/volatility3/framework/plugins/linux/check_modules.py +++ b/volatility3/framework/plugins/linux/check_modules.py @@ -23,13 +23,20 @@ class Check_modules(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), ] @classmethod - def get_kset_modules(cls, context: interfaces.context.ContextInterface, vmlinux_name: str): + def get_kset_modules( + cls, context: interfaces.context.ContextInterface, vmlinux_name: str + ): vmlinux = context.modules[vmlinux_name] @@ -45,12 +52,17 @@ class Check_modules(plugins.PluginInterface): ret = {} - kobj_off = vmlinux.get_type('module_kobject').relative_child_offset('kobj') + kobj_off = vmlinux.get_type("module_kobject").relative_child_offset("kobj") - for kobj in module_kset.list.to_list(vmlinux.symbol_table_name + constants.BANG + "kobject", "entry"): + for kobj in module_kset.list.to_list( + vmlinux.symbol_table_name + constants.BANG + "kobject", "entry" + ): - mod_kobj = vmlinux.object(object_type = "module_kobject", offset = kobj.vol.offset - kobj_off, - absolute = True) + mod_kobj = vmlinux.object( + object_type="module_kobject", + offset=kobj.vol.offset - kobj_off, + absolute=True, + ) mod = mod_kobj.mod @@ -61,14 +73,18 @@ class Check_modules(plugins.PluginInterface): return ret def _generator(self): - kset_modules = self.get_kset_modules(self.context, self.config['kernel']) + kset_modules = self.get_kset_modules(self.context, self.config["kernel"]) lsmod_modules = set( str(utility.array_to_string(modules.name)) - for modules in lsmod.Lsmod.list_modules(self.context, self.config['kernel'])) + for modules in lsmod.Lsmod.list_modules(self.context, self.config["kernel"]) + ) for mod_name in set(kset_modules.keys()).difference(lsmod_modules): yield (0, (format_hints.Hex(kset_modules[mod_name]), str(mod_name))) def run(self): - return renderers.TreeGrid([("Module Address", format_hints.Hex), ("Module Name", str)], self._generator()) + return renderers.TreeGrid( + [("Module Address", format_hints.Hex), ("Module Name", str)], + self._generator(), + ) diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 6ec5fd354..6b11038ec 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -30,8 +30,11 @@ class Check_syscall(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), ] def _get_table_size_next_symbol(self, table_addr, ptr_sz, vmlinux): @@ -63,8 +66,12 @@ class Check_syscall(plugins.PluginInterface): accurate.""" return len( - [sym for sym in self.context.symbol_space[vmlinux.symbol_table_name].symbols if - sym.startswith("__syscall_meta__")]) + [ + sym + for sym in self.context.symbol_space[vmlinux.symbol_table_name].symbols + if sym.startswith("__syscall_meta__") + ] + ) def _get_table_info_other(self, table_addr, ptr_sz, vmlinux): table_size_meta = self._get_table_size_meta(vmlinux) @@ -100,12 +107,12 @@ class Check_syscall(plugins.PluginInterface): # if we can't find the disassemble function then bail and rely on a different method return 0 - vmlinux = self.context.modules[self.config['kernel']] + vmlinux = self.context.modules[self.config["kernel"]] data = self.context.layers.read(vmlinux.layer_name, func_addr, 6) for (address, size, mnemonic, op_str) in md.disasm_lite(data, func_addr): - if mnemonic == 'CMP': - table_size = int(op_str.split(",")[1].strip()) & 0xffff + if mnemonic == "CMP": + table_size = int(op_str.split(",")[1].strip()) & 0xFFFF break return table_size @@ -126,7 +133,7 @@ class Check_syscall(plugins.PluginInterface): # TODO - add finding and parsing unistd.h once cached file enumeration is added def _generator(self): - vmlinux = self.context.modules[self.config['kernel']] + vmlinux = self.context.modules[self.config["kernel"]] ptr_sz = vmlinux.get_type("pointer").size if ptr_sz == 4: @@ -155,10 +162,12 @@ class Check_syscall(plugins.PluginInterface): tables.append(("32bit", ia32_info)) for (table_name, (tableaddr, tblsz)) in tables: - table = vmlinux.object(object_type = "array", - subtype = vmlinux.get_type("pointer"), - offset = tableaddr, - count = tblsz) + table = vmlinux.object( + object_type="array", + subtype=vmlinux.get_type("pointer"), + offset=tableaddr, + count=tblsz, + ) for (i, call_addr) in enumerate(table): if not call_addr: @@ -167,14 +176,34 @@ class Check_syscall(plugins.PluginInterface): symbols = list(vmlinux.get_symbols_by_absolute_location(call_addr)) if len(symbols) > 0: - sym_name = str(symbols[0].split(constants.BANG)[1]) if constants.BANG in symbols[0] else \ - str(symbols[0]) + sym_name = ( + str(symbols[0].split(constants.BANG)[1]) + if constants.BANG in symbols[0] + else str(symbols[0]) + ) else: sym_name = "UNKNOWN" - yield (0, (format_hints.Hex(tableaddr), table_name, i, format_hints.Hex(call_addr), sym_name)) + yield ( + 0, + ( + format_hints.Hex(tableaddr), + table_name, + i, + format_hints.Hex(call_addr), + sym_name, + ), + ) def run(self): - return renderers.TreeGrid([("Table Address", format_hints.Hex), ("Table Name", str), ("Index", int), - ("Handler Address", format_hints.Hex), ("Handler Symbol", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Table Address", format_hints.Hex), + ("Table Name", str), + ("Index", int), + ("Handler Address", format_hints.Hex), + ("Handler Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index d2380817d..822a69dd6 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -22,13 +22,20 @@ class Elfs(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] def _generator(self, tasks): @@ -42,20 +49,42 @@ class Elfs(plugins.PluginInterface): name = utility.array_to_string(task.comm) for vma in task.mm.get_mmap_iter(): - hdr = proc_layer.read(vma.vm_start, 4, pad = True) - if not (hdr[0] == 0x7f and hdr[1] == 0x45 and hdr[2] == 0x4c and hdr[3] == 0x46): + hdr = proc_layer.read(vma.vm_start, 4, pad=True) + if not ( + hdr[0] == 0x7F + and hdr[1] == 0x45 + and hdr[2] == 0x4C + and hdr[3] == 0x46 + ): continue path = vma.get_name(self.context, task) - yield (0, (task.pid, name, format_hints.Hex(vma.vm_start), format_hints.Hex(vma.vm_end), path)) + yield ( + 0, + ( + task.pid, + name, + format_hints.Hex(vma.vm_start), + format_hints.Hex(vma.vm_end), + path, + ), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - return renderers.TreeGrid([("PID", int), ("Process", str), ("Start", format_hints.Hex), - ("End", format_hints.Hex), ("File Path", str)], - self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Start", format_hints.Hex), + ("End", format_hints.Hex), + ("File Path", str), + ], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), + ) diff --git a/volatility3/framework/plugins/linux/keyboard_notifiers.py b/volatility3/framework/plugins/linux/keyboard_notifiers.py index 51e684e6f..72273a77b 100644 --- a/volatility3/framework/plugins/linux/keyboard_notifiers.py +++ b/volatility3/framework/plugins/linux/keyboard_notifiers.py @@ -21,18 +21,27 @@ class Keyboard_notifiers(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) + ), ] def _generator(self): - vmlinux = self.context.modules[self.config['kernel']] + vmlinux = self.context.modules[self.config["kernel"]] modules = lsmod.Lsmod.list_modules(self.context, vmlinux.name) - handlers = linux.LinuxUtilities.generate_kernel_handler_info(self.context, vmlinux.name, modules) + handlers = linux.LinuxUtilities.generate_kernel_handler_info( + self.context, vmlinux.name, modules + ) try: knl_addr = vmlinux.object_from_symbol("keyboard_notifier_list") @@ -46,14 +55,25 @@ class Keyboard_notifiers(interfaces.plugins.PluginInterface): "This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." ) - knl = vmlinux.object(object_type = "atomic_notifier_head", offset = knl_addr.vol.offset, absolute = True) + knl = vmlinux.object( + object_type="atomic_notifier_head", + offset=knl_addr.vol.offset, + absolute=True, + ) - for call_back in linux.LinuxUtilities.walk_internal_list(vmlinux, "notifier_block", "next", knl.head): + for call_back in linux.LinuxUtilities.walk_internal_list( + vmlinux, "notifier_block", "next", knl.head + ): call_addr = call_back.notifier_call - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address(vmlinux, handlers, call_addr) + module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( + vmlinux, handlers, call_addr + ) yield (0, [format_hints.Hex(call_addr), module_name, symbol_name]) def run(self): - return renderers.TreeGrid([("Address", format_hints.Hex), ("Module", str), ("Symbol", str)], self._generator()) + return renderers.TreeGrid( + [("Address", format_hints.Hex), ("Module", str), ("Symbol", str)], + self._generator(), + ) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 8f4540766..3f7345bdc 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -6,7 +6,13 @@ from abc import ABC, abstractmethod from enum import Enum from typing import Generator, Iterator, List, Tuple -from volatility3.framework import class_subclasses, constants, contexts, interfaces, renderers +from volatility3.framework import ( + class_subclasses, + constants, + contexts, + interfaces, + renderers, +) from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -24,6 +30,7 @@ class DescStateEnum(Enum): class ABCKmsg(ABC): """Kernel log buffer reader""" + LEVELS = ( "emerg", # system is unusable "alert", # action must be taken immediately @@ -47,27 +54,27 @@ class ABCKmsg(ABC): "uucp", # UUCP subsystem "cron", # clock daemon "authpriv", # security/authorization messages (private) - "ftp" # FTP daemon + "ftp", # FTP daemon ) def __init__( - self, - context: interfaces.context.ContextInterface, - config: interfaces.configuration.HierarchicalDict + self, + context: interfaces.context.ContextInterface, + config: interfaces.configuration.HierarchicalDict, ): self._context = context self._config = config - vmlinux = context.modules[self._config['kernel']] + vmlinux = context.modules[self._config["kernel"]] self.layer_name = vmlinux.layer_name # type: ignore symbol_table_name = vmlinux.symbol_table_name # type: ignore self.vmlinux = contexts.Module.create(context, symbol_table_name, self.layer_name, 0) # type: ignore - self.long_unsigned_int_size = self.vmlinux.get_type('long unsigned int').size + self.long_unsigned_int_size = self.vmlinux.get_type("long unsigned int").size @classmethod def run_all( - cls, - context: interfaces.context.ContextInterface, - config: interfaces.configuration.HierarchicalDict + cls, + context: interfaces.context.ContextInterface, + config: interfaces.configuration.HierarchicalDict, ) -> Iterator[Tuple[str, str, str, str, str]]: """It calls each subclass symtab_checks() to test the required conditions to that specific kernel implementation. @@ -79,17 +86,24 @@ class ABCKmsg(ABC): Yields: kmsg records """ - vmlinux = context.modules[config['kernel']] + vmlinux = context.modules[config["kernel"]] kmsg_inst = None # type: ignore for subclass in class_subclasses(cls): - if not subclass.symtab_checks(vmlinux = vmlinux): - vollog.log(constants.LOGLEVEL_VVVV, - "Kmsg implementation '%s' doesn't match this memory dump", subclass.__name__) + if not subclass.symtab_checks(vmlinux=vmlinux): + vollog.log( + constants.LOGLEVEL_VVVV, + "Kmsg implementation '%s' doesn't match this memory dump", + subclass.__name__, + ) continue - vollog.log(constants.LOGLEVEL_VVVV, "Kmsg implementation '%s' matches!", subclass.__name__) - kmsg_inst = subclass(context = context, config = config) + vollog.log( + constants.LOGLEVEL_VVVV, + "Kmsg implementation '%s' matches!", + subclass.__name__, + ) + kmsg_inst = subclass(context=context, config=config) # More than one class could be executed for an specific kernel # version i.e. Netfilter Ingress hooks # We expect just one implementation to be executed for an specific kernel @@ -116,7 +130,7 @@ class ABCKmsg(ABC): def get_string(self, addr: int, length: int) -> str: txt = self._context.layers[self.layer_name].read(addr, length) # type: ignore - return txt.decode(encoding = 'utf8', errors = 'replace') + return txt.decode(encoding="utf8", errors="replace") def nsec_to_sec_str(self, nsec: int) -> str: # See kernel/printk/printk.c:print_time() @@ -138,19 +152,24 @@ class ABCKmsg(ABC): # In some kernel versions, it's only available if CONFIG_PRINTK_CALLER is defined. # caller_id is a member of printk_log struct from 5.1 to the latest 5.9 # From kernels 5.10 on, it's a member of printk_info struct - if obj.has_member('caller_id'): + if obj.has_member("caller_id"): return self.get_caller_text(obj.caller_id) else: return "" def get_caller_text(self, caller_id): - caller_name = 'CPU' if caller_id & 0x80000000 else 'Task' + caller_name = "CPU" if caller_id & 0x80000000 else "Task" caller = "%s(%u)" % (caller_name, caller_id & ~0x80000000) return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: # obj could be printk_log or printk_info - return obj.facility, obj.level, self.get_timestamp_in_sec_str(obj), self.get_caller(obj) + return ( + obj.facility, + obj.level, + self.get_timestamp_in_sec_str(obj), + self.get_caller(obj), + ) @classmethod def get_level_text(cls, level: int) -> str: @@ -185,10 +204,10 @@ class KmsgLegacy(ABCKmsg): @classmethod def symtab_checks(cls, vmlinux) -> bool: - return vmlinux.has_type('printk_log') + return vmlinux.has_type("printk_log") def get_text_from_printk_log(self, msg) -> str: - msg_offset = msg.vol.offset + self.vmlinux.get_type('printk_log').size + msg_offset = msg.vol.offset + self.vmlinux.get_type("printk_log").size return self.get_string(msg_offset, msg.text_len) def get_log_lines(self, msg) -> Generator[str, None, None]: @@ -199,26 +218,34 @@ class KmsgLegacy(ABCKmsg): def get_dict_lines(self, msg) -> Generator[str, None, None]: if msg.dict_len == 0: return None - dict_offset = msg.vol.offset + self.vmlinux.get_type('printk_log').size + msg.text_len - dict_data = self._context.layers[self.layer_name].read(dict_offset, msg.dict_len) - for chunk in dict_data.split(b'\x00'): + dict_offset = ( + msg.vol.offset + self.vmlinux.get_type("printk_log").size + msg.text_len + ) + dict_data = self._context.layers[self.layer_name].read( + dict_offset, msg.dict_len + ) + for chunk in dict_data.split(b"\x00"): yield " " + chunk.decode() def run(self) -> Iterator[Tuple[str, str, str, str, str]]: - log_buf_ptr = self.vmlinux.object_from_symbol(symbol_name = 'log_buf') + log_buf_ptr = self.vmlinux.object_from_symbol(symbol_name="log_buf") if log_buf_ptr == 0: # This is weird, let's fallback to check the static ringbuffer. - log_buf_ptr = self.vmlinux.object_from_symbol(symbol_name = '__log_buf').vol.offset + log_buf_ptr = self.vmlinux.object_from_symbol( + symbol_name="__log_buf" + ).vol.offset if log_buf_ptr == 0: raise ValueError("Log buffer is not available") - log_first_idx = int(self.vmlinux.object_from_symbol(symbol_name = 'log_first_idx')) + log_first_idx = int( + self.vmlinux.object_from_symbol(symbol_name="log_first_idx") + ) cur_idx = log_first_idx end_idx = None # We don't need log_next_idx here. See below msg.len == 0 while cur_idx != end_idx: end_idx = log_first_idx msg_offset = log_buf_ptr + cur_idx # type: ignore - msg = self.vmlinux.object(object_type = 'printk_log', offset = msg_offset) + msg = self.vmlinux.object(object_type="printk_log", offset=msg_offset) if msg.len == 0: # As per kernel/printk/printk.c: # A length == 0 for the next message indicates a wrap-around to @@ -284,7 +311,7 @@ class KmsgFiveTen(ABCKmsg): @classmethod def symtab_checks(cls, vmlinux) -> bool: - return vmlinux.has_symbol('prb') + return vmlinux.has_symbol("prb") def get_text_from_data_ring(self, text_data_ring, desc, info) -> str: text_data_sz = text_data_ring.size_bits @@ -327,20 +354,24 @@ class KmsgFiveTen(ABCKmsg): def run(self) -> Iterator[Tuple[str, str, str, str, str]]: # static struct printk_ringbuffer *prb = &printk_rb_static; - ringbuffers = self.vmlinux.object_from_symbol(symbol_name = 'prb').dereference() + ringbuffers = self.vmlinux.object_from_symbol(symbol_name="prb").dereference() desc_ring = ringbuffers.desc_ring text_data_ring = ringbuffers.text_data_ring desc_count = 1 << desc_ring.count_bits - desc_arr = self.vmlinux.object(object_type = "array", - offset = desc_ring.descs, - subtype = self.vmlinux.get_type("prb_desc"), - count = desc_count) - info_arr = self.vmlinux.object(object_type = "array", - offset = desc_ring.infos, - subtype = self.vmlinux.get_type("printk_info"), - count = desc_count) + desc_arr = self.vmlinux.object( + object_type="array", + offset=desc_ring.descs, + subtype=self.vmlinux.get_type("prb_desc"), + count=desc_count, + ) + info_arr = self.vmlinux.object( + object_type="array", + offset=desc_ring.infos, + subtype=self.vmlinux.get_type("printk_info"), + count=desc_count, + ) # See kernel/printk/printk_ringbuffer.h desc_state_var_bytes_sz = self.long_unsigned_int_size @@ -356,7 +387,10 @@ class KmsgFiveTen(ABCKmsg): desc = desc_arr[cur_id % desc_count] # type: ignore info = info_arr[cur_id % desc_count] # type: ignore desc_state = DescStateEnum((desc.state_var.counter >> desc_flags_shift) & 3) - if desc_state in (DescStateEnum.desc_committed, DescStateEnum.desc_finalized): + if desc_state in ( + DescStateEnum.desc_committed, + DescStateEnum.desc_finalized, + ): facility, level, timestamp, caller = self.get_prefix(info) level_txt = self.get_level_text(level) facility_txt = self.get_facility_text(facility) @@ -380,18 +414,25 @@ class Kmsg(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ['Intel32', 'Intel64']), + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), ] def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str, str]]]: - for values in ABCKmsg.run_all(context = self.context, config = self.config): + for values in ABCKmsg.run_all(context=self.context, config=self.config): yield (0, values) def run(self): - return renderers.TreeGrid([("facility", str), - ("level", str), - ("timestamp", str), - ("caller", str), - ("line", str)], - self._generator()) # type: ignore + return renderers.TreeGrid( + [ + ("facility", str), + ("level", str), + ("timestamp", str), + ("caller", str), + ("line", str), + ], + self._generator(), + ) # type: ignore diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index ecb262d00..1c1e094c3 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -25,13 +25,17 @@ class Lsmod(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), ] @classmethod - def list_modules(cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str) -> Iterable[ - interfaces.objects.ObjectInterface]: + def list_modules( + cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the modules in the primary layer. Args: @@ -46,7 +50,7 @@ class Lsmod(plugins.PluginInterface): """ vmlinux = context.modules[vmlinux_module_name] - modules = vmlinux.object_from_symbol(symbol_name = "modules").cast("list_head") + modules = vmlinux.object_from_symbol(symbol_name="modules").cast("list_head") table_name = modules.vol.type_name.split(constants.BANG)[0] @@ -55,7 +59,7 @@ class Lsmod(plugins.PluginInterface): def _generator(self): try: - for module in self.list_modules(self.context, self.config['kernel']): + for module in self.list_modules(self.context, self.config["kernel"]): mod_size = module.get_init_size() + module.get_core_size() @@ -69,4 +73,7 @@ class Lsmod(plugins.PluginInterface): ) def run(self): - return renderers.TreeGrid([("Offset", format_hints.Hex), ("Name", str), ("Size", int)], self._generator()) + return renderers.TreeGrid( + [("Offset", format_hints.Hex), ("Name", str), ("Size", int)], + self._generator(), + ) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index a074f5744..81541c57d 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -24,14 +24,23 @@ class Lsof(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] def _generator(self, tasks): @@ -45,15 +54,23 @@ class Lsof(plugins.PluginInterface): name = utility.array_to_string(task.comm) pid = int(task.pid) - for fd_num, _, full_path in linux.LinuxUtilities.files_descriptors_for_process( - self.context, symbol_table, task): + for ( + fd_num, + _, + full_path, + ) in linux.LinuxUtilities.files_descriptors_for_process( + self.context, symbol_table, task + ): yield (0, (pid, name, fd_num, full_path)) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - return renderers.TreeGrid([("PID", int), ("Process", str), ("FD", int), ("Path", str)], - self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [("PID", int), ("Process", str), ("FD", int), ("Path", str)], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), + ) diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index abc2cf7d2..18237b80c 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -20,13 +20,20 @@ class Malfind(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] def _list_injections(self, task): @@ -41,13 +48,18 @@ class Malfind(interfaces.plugins.PluginInterface): for vma in task.mm.get_mmap_iter(): if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]": - data = proc_layer.read(vma.vm_start, 64, pad = True) + data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data def _generator(self, tasks): # determine if we're on a 32 or 64 bit kernel - vmlinux = self.context.modules[self.config['kernel']] - if self.context.symbol_space.get_type(vmlinux.symbol_table_name + constants.BANG + "pointer").size == 4: + vmlinux = self.context.modules[self.config["kernel"]] + if ( + self.context.symbol_space.get_type( + vmlinux.symbol_table_name + constants.BANG + "pointer" + ).size + == 4 + ): is_32bit_arch = True else: is_32bit_arch = False @@ -61,18 +73,39 @@ class Malfind(interfaces.plugins.PluginInterface): else: architecture = "intel64" - disasm = interfaces.renderers.Disassembly(data, vma.vm_start, architecture) + disasm = interfaces.renderers.Disassembly( + data, vma.vm_start, architecture + ) - yield (0, (task.pid, process_name, format_hints.Hex(vma.vm_start), format_hints.Hex(vma.vm_end), - vma.get_protection(), format_hints.HexBytes(data), disasm)) + yield ( + 0, + ( + task.pid, + process_name, + format_hints.Hex(vma.vm_start), + format_hints.Hex(vma.vm_end), + vma.get_protection(), + format_hints.HexBytes(data), + disasm, + ), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - return renderers.TreeGrid([("PID", int), ("Process", str), ("Start", format_hints.Hex), - ("End", format_hints.Hex), ("Protection", str), ("Hexdump", format_hints.HexBytes), - ("Disasm", interfaces.renderers.Disassembly)], - self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Start", format_hints.Hex), + ("End", format_hints.Hex), + ("Protection", str), + ("Hexdump", format_hints.HexBytes), + ("Disasm", interfaces.renderers.Disassembly), + ], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), + ) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 551d128ad..c849d51c6 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -13,8 +13,22 @@ from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) -MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "mnt_root_path", "path_root", - "mnt_opts", "fields", "mnt_type", "devname", "sb_opts")) +MountInfoData = namedtuple( + "MountInfoData", + ( + "mnt_id", + "parent_id", + "st_dev", + "mnt_root_path", + "path_root", + "mnt_opts", + "fields", + "mnt_type", + "devname", + "sb_opts", + ), +) + class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" @@ -26,25 +40,35 @@ class MountInfo(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name="kernel", description="Linux kernel", - architectures=["Intel32", "Intel64"]), - requirements.PluginRequirement(name="pslist", - plugin=pslist.PsList, version=(2, 0, 0)), - requirements.ListRequirement(name="pids", - description="Filter on specific process IDs.", - element_type=int, - optional=True), - requirements.ListRequirement(name="mntns", - description="Filter results by mount namespace. " - "Otherwise, all of them are shown.", - element_type=int, - optional=True), - requirements.BooleanRequirement(name="mount-format", - description="Shows a brief summary of the mount points information " - "with similar output format to the older /proc/[pid]/mounts or the " - "user-land command 'mount -l'.", - optional=True, - default=False), + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pids", + description="Filter on specific process IDs.", + element_type=int, + optional=True, + ), + requirements.ListRequirement( + name="mntns", + description="Filter results by mount namespace. " + "Otherwise, all of them are shown.", + element_type=int, + optional=True, + ), + requirements.BooleanRequirement( + name="mount-format", + description="Shows a brief summary of the mount points information " + "with similar output format to the older /proc/[pid]/mounts or the " + "user-land command 'mount -l'.", + optional=True, + default=False, + ), ] @classmethod @@ -79,8 +103,11 @@ class MountInfo(plugins.PluginInterface): return path @classmethod - def get_mountinfo(cls, mnt, task) -> Union[None, Tuple[int, int, str, str, str, List[str], - List[str], str, str, List[str]]]: + def get_mountinfo( + cls, mnt, task + ) -> Union[ + None, Tuple[int, int, str, str, str, List[str], List[str], str, str, List[str]] + ]: """Extract various information about a mount point. It mimics the Linux kernel show_mountinfo function. """ @@ -129,13 +156,31 @@ class MountInfo(plugins.PluginInterface): sb_opts.append(superblock.get_flags_access()) sb_opts.extend(superblock.get_flags_opts()) - return MountInfoData(mnt_id, parent_id, st_dev, mnt_root_path, path_root, mnt_opts, fields, - mnt_type, devname, sb_opts) + return MountInfoData( + mnt_id, + parent_id, + st_dev, + mnt_root_path, + path_root, + mnt_opts, + fields, + mnt_type, + devname, + sb_opts, + ) - def _get_tasks_mountpoints(self, tasks: Iterable[interfaces.objects.ObjectInterface], per_namespace: bool): + def _get_tasks_mountpoints( + self, tasks: Iterable[interfaces.objects.ObjectInterface], per_namespace: bool + ): seen_namespaces = set() for task in tasks: - if not (task and task.fs and task.fs.root and task.nsproxy and task.nsproxy.mnt_ns): + if not ( + task + and task.fs + and task.fs.root + and task.nsproxy + and task.nsproxy.mnt_ns + ): # This task doesn't have all the information required continue @@ -152,11 +197,12 @@ class MountInfo(plugins.PluginInterface): yield task, mount, mnt_ns_id def _generator( - self, - tasks: Iterable[interfaces.objects.ObjectInterface], - mnt_ns_ids: List[int], - mount_format: bool, - per_namespace: bool) -> Iterable[Tuple[int, Tuple]]: + self, + tasks: Iterable[interfaces.objects.ObjectInterface], + mnt_ns_ids: List[int], + mount_format: bool, + per_namespace: bool, + ) -> Iterable[Tuple[int, Tuple]]: for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, per_namespace): if mnt_ns_ids and mnt_ns_id not in mnt_ns_ids: @@ -172,15 +218,29 @@ class MountInfo(plugins.PluginInterface): all_opts.update(mnt_info.sb_opts) all_opts_str = ",".join(all_opts) - extra_fields_values = [mnt_info.devname, mnt_info.path_root, mnt_info.mnt_type, all_opts_str] + extra_fields_values = [ + mnt_info.devname, + mnt_info.path_root, + mnt_info.mnt_type, + all_opts_str, + ] else: mnt_opts_str = ",".join(mnt_info.mnt_opts) fields_str = " ".join(mnt_info.fields) sb_opts_str = ",".join(mnt_info.sb_opts) - extra_fields_values = [mnt_info.mnt_id, mnt_info.parent_id, mnt_info.st_dev, mnt_info.mnt_root_path, - mnt_info.path_root, mnt_opts_str, fields_str, mnt_info.mnt_type, - mnt_info.devname, sb_opts_str] + extra_fields_values = [ + mnt_info.mnt_id, + mnt_info.parent_id, + mnt_info.st_dev, + mnt_info.mnt_root_path, + mnt_info.path_root, + mnt_opts_str, + fields_str, + mnt_info.mnt_type, + mnt_info.devname, + sb_opts_str, + ] fields_values = [mnt_ns_id] if not per_namespace: @@ -190,12 +250,14 @@ class MountInfo(plugins.PluginInterface): yield (0, fields_values) def run(self): - pids = self.config.get('pids') - mount_ns_ids = self.config.get('mntns') - mount_format = self.config.get('mount-format') + pids = self.config.get("pids") + mount_ns_ids = self.config.get("mntns") + mount_format = self.config.get("mount-format") pid_filter = pslist.PsList.create_pid_filter(pids) - tasks = pslist.PsList.list_tasks(self.context, self.config['kernel'], filter_func=pid_filter) + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=pid_filter + ) columns = [("MNT_NS_ID", int)] # The PID column does not make sense when a PID filter is not specified. In that case, the default behavior is @@ -206,14 +268,30 @@ class MountInfo(plugins.PluginInterface): else: per_namespace = True - if self.config.get('mount-format'): - extra_columns = [("DEVNAME", str), ("PATH", str), ("FSTYPE", str), ("MNT_OPTS", str)] + if self.config.get("mount-format"): + extra_columns = [ + ("DEVNAME", str), + ("PATH", str), + ("FSTYPE", str), + ("MNT_OPTS", str), + ] else: # /proc/[pid]/mountinfo output format - extra_columns = [("MOUNT ID", int), ("PARENT_ID", int), ("MAJOR:MINOR", str), ("ROOT", str), - ("MOUNT_POINT", str), ("MOUNT_OPTIONS", str), ("FIELDS", str), ("FSTYPE", str), - ("MOUNT_SRC", str), ("SB_OPTIONS", str)] + extra_columns = [ + ("MOUNT ID", int), + ("PARENT_ID", int), + ("MAJOR:MINOR", str), + ("ROOT", str), + ("MOUNT_POINT", str), + ("MOUNT_OPTIONS", str), + ("FIELDS", str), + ("FSTYPE", str), + ("MOUNT_SRC", str), + ("SB_OPTIONS", str), + ] columns.extend(extra_columns) - return renderers.TreeGrid(columns, self._generator(tasks, mount_ns_ids, mount_format, per_namespace)) + return renderers.TreeGrid( + columns, self._generator(tasks, mount_ns_ids, mount_format, per_namespace) + ) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 13fd87f53..9d8af482e 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,13 +21,20 @@ class Maps(plugins.PluginInterface): def get_requirements(cls): # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] def _generator(self, tasks): @@ -54,17 +61,41 @@ class Maps(plugins.PluginInterface): path = vma.get_name(self.context, task) - yield (0, (task.pid, name, format_hints.Hex(vma.vm_start), format_hints.Hex(vma.vm_end), flags, - format_hints.Hex(page_offset), major, minor, inode, path)) + yield ( + 0, + ( + task.pid, + name, + format_hints.Hex(vma.vm_start), + format_hints.Hex(vma.vm_end), + flags, + format_hints.Hex(page_offset), + major, + minor, + inode, + path, + ), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - return renderers.TreeGrid([("PID", int), ("Process", str), - ("Start", format_hints.Hex), ("End", format_hints.Hex), ("Flags", str), - ("PgOff", format_hints.Hex), ("Major", int), ("Minor", int), ("Inode", int), - ("File Path", str)], - self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Start", format_hints.Hex), + ("End", format_hints.Hex), + ("Flags", str), + ("PgOff", format_hints.Hex), + ("Major", int), + ("Minor", int), + ("Inode", int), + ("File Path", str), + ], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), + ) diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index d8b844ca4..a4a23498f 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -12,7 +12,7 @@ from volatility3.plugins.linux import pslist class PsAux(plugins.PluginInterface): - """ Lists processes with their command line arguments """ + """Lists processes with their command line arguments""" _required_framework_version = (2, 0, 0) @@ -20,17 +20,25 @@ class PsAux(plugins.PluginInterface): def get_requirements(cls): # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] - def _get_command_line_args(self, task: interfaces.objects.ObjectInterface, - name: str) -> Optional[str]: + def _get_command_line_args( + self, task: interfaces.objects.ObjectInterface, name: str + ) -> Optional[str]: """ Reads the command line arguments of a process These are stored on the userland stack @@ -69,7 +77,7 @@ class PsAux(plugins.PluginInterface): return renderers.UnreadableValue() # the arguments are null byte terminated, replace the nulls with spaces - s = argv.decode().split('\x00') + s = argv.decode().split("\x00") args = " ".join(s) else: # kernel thread @@ -84,7 +92,7 @@ class PsAux(plugins.PluginInterface): return args def _generator(self, tasks): - """ Generates a listing of processes along with command line arguments """ + """Generates a listing of processes along with command line arguments""" # walk the process list and report the arguments for task in tasks: @@ -102,10 +110,13 @@ class PsAux(plugins.PluginInterface): yield (0, (pid, ppid, name, args)) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], - self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), + ) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index f9a1fe2a0..af260a772 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -19,20 +19,29 @@ class PsList(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True), - requirements.BooleanRequirement(name="threads", - description="Include user threads", - optional=True, - default=False), - requirements.BooleanRequirement(name="decorate_comm", - description="Show `user threads` comm in curly brackets, and `kernel threads` comm in square brackets", - optional=True, - default=False), + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.BooleanRequirement( + name="threads", + description="Include user threads", + optional=True, + default=False, + ), + requirements.BooleanRequirement( + name="decorate_comm", + description="Show `user threads` comm in curly brackets, and `kernel threads` comm in square brackets", + optional=True, + default=False, + ), ] @classmethod @@ -58,9 +67,8 @@ class PsList(interfaces.plugins.PluginInterface): return lambda _: False def _get_task_fields( - self, - task: interfaces.objects.ObjectInterface, - decorate_comm: bool = False) -> Tuple[int, int, int, str]: + self, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False + ) -> Tuple[int, int, int, str]: """Extract the fields needed for the final output Args: @@ -86,10 +94,11 @@ class PsList(interfaces.plugins.PluginInterface): return task_fields def _generator( - self, - pid_filter: Callable[[Any], bool], - include_threads: bool = False, - decorate_comm: bool = False): + self, + pid_filter: Callable[[Any], bool], + include_threads: bool = False, + decorate_comm: bool = False, + ): """Generates the tasks list. Args: @@ -104,20 +113,20 @@ class PsList(interfaces.plugins.PluginInterface): Yields: Each rows """ - for task in self.list_tasks(self.context, - self.config['kernel'], - pid_filter, - include_threads): + for task in self.list_tasks( + self.context, self.config["kernel"], pid_filter, include_threads + ): row = self._get_task_fields(task, decorate_comm) yield (0, row) @classmethod def list_tasks( - cls, - context: interfaces.context.ContextInterface, - vmlinux_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False, - include_threads: bool = False) -> Iterable[interfaces.objects.ObjectInterface]: + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + filter_func: Callable[[int], bool] = lambda _: False, + include_threads: bool = False, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer. Args: @@ -130,7 +139,7 @@ class PsList(interfaces.plugins.PluginInterface): """ vmlinux = context.modules[vmlinux_module_name] - init_task = vmlinux.object_from_symbol(symbol_name = "init_task") + init_task = vmlinux.object_from_symbol(symbol_name="init_task") # Note that the init_task itself is not yielded, since "ps" also never shows it. for task in init_task.tasks: @@ -143,10 +152,18 @@ class PsList(interfaces.plugins.PluginInterface): yield from task.get_threads() def run(self): - pids = self.config.get('pid') - include_threads = self.config.get('threads') - decorate_comm = self.config.get('decorate_comm') + pids = self.config.get("pid") + include_threads = self.config.get("threads") + decorate_comm = self.config.get("decorate_comm") filter_func = self.create_pid_filter(pids) - columns = [("OFFSET (V)", format_hints.Hex), ("PID", int), ("TID", int), ("PPID", int), ("COMM", str)] - return renderers.TreeGrid(columns, self._generator(filter_func, include_threads, decorate_comm)) + columns = [ + ("OFFSET (V)", format_hints.Hex), + ("PID", int), + ("TID", int), + ("PPID", int), + ("COMM", str), + ] + return renderers.TreeGrid( + columns, self._generator(filter_func, include_threads, decorate_comm) + ) diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 3ad5f3e19..e07a8aced 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -39,10 +39,8 @@ class PsTree(pslist.PsList): self._levels[pid] = level def _generator( - self, - pid_filter, - include_threads: bool = False, - decorate_com: bool = False): + self, pid_filter, include_threads: bool = False, decorate_com: bool = False + ): """Generates the tasks hierarchy tree. Args: @@ -57,11 +55,13 @@ class PsTree(pslist.PsList): Yields: Each rows """ - vmlinux = self.context.modules[self.config['kernel']] - for proc in self.list_tasks(self.context, - vmlinux.name, - filter_func=pid_filter, - include_threads=include_threads): + vmlinux = self.context.modules[self.config["kernel"]] + for proc in self.list_tasks( + self.context, + vmlinux.name, + filter_func=pid_filter, + include_threads=include_threads, + ): self._tasks[proc.pid] = proc # Build the child/level maps diff --git a/volatility3/framework/plugins/linux/tty_check.py b/volatility3/framework/plugins/linux/tty_check.py index e1d8339e8..dcc9f3e06 100644 --- a/volatility3/framework/plugins/linux/tty_check.py +++ b/volatility3/framework/plugins/linux/tty_check.py @@ -24,18 +24,27 @@ class tty_check(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) + ), ] def _generator(self): - vmlinux = self.context.modules[self.config['kernel']] + vmlinux = self.context.modules[self.config["kernel"]] modules = lsmod.Lsmod.list_modules(self.context, vmlinux.name) - handlers = linux.LinuxUtilities.generate_kernel_handler_info(self.context, vmlinux.name, modules) + handlers = linux.LinuxUtilities.generate_kernel_handler_info( + self.context, vmlinux.name, modules + ) try: tty_drivers = vmlinux.object_from_symbol("tty_drivers").cast("list_head") @@ -49,13 +58,17 @@ class tty_check(plugins.PluginInterface): "This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." ) - for tty in tty_drivers.to_list(vmlinux.symbol_table_name + constants.BANG + "tty_driver", "tty_drivers"): + for tty in tty_drivers.to_list( + vmlinux.symbol_table_name + constants.BANG + "tty_driver", "tty_drivers" + ): try: - ttys = utility.array_of_pointers(tty.ttys.dereference(), - count = tty.num, - subtype = vmlinux.symbol_table_name + constants.BANG + "tty_struct", - context = self.context) + ttys = utility.array_of_pointers( + tty.ttys.dereference(), + count=tty.num, + subtype=vmlinux.symbol_table_name + constants.BANG + "tty_struct", + context=self.context, + ) except exceptions.PagedInvalidAddressException: continue @@ -68,10 +81,19 @@ class tty_check(plugins.PluginInterface): recv_buf = tty_dev.ldisc.ops.receive_buf - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address(vmlinux, handlers, recv_buf) + module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( + vmlinux, handlers, recv_buf + ) yield (0, (name, format_hints.Hex(recv_buf), module_name, symbol_name)) def run(self): - return renderers.TreeGrid([("Name", str), ("Address", format_hints.Hex), ("Module", str), ("Symbol", str)], - self._generator()) + return renderers.TreeGrid( + [ + ("Name", str), + ("Address", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/bash.py b/volatility3/framework/plugins/mac/bash.py index e16a0d79f..a52ae616a 100644 --- a/volatility3/framework/plugins/mac/bash.py +++ b/volatility3/framework/plugins/mac/bash.py @@ -25,18 +25,27 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] def _generator(self, tasks): - darwin = self.context.modules[self.config['kernel']] - is_32bit = not symbols.symbol_table_is_64bit(self.context, darwin.symbol_table_name) + darwin = self.context.modules[self.config["kernel"]] + is_32bit = not symbols.symbol_table_is_64bit( + self.context, darwin.symbol_table_name + ) if is_32bit: pack_format = "I" bash_json_file = "bash32" @@ -44,10 +53,13 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): pack_format = "Q" bash_json_file = "bash64" - bash_table_name = BashIntermedSymbols.create(self.context, self.config_path, "linux", bash_json_file) + bash_table_name = BashIntermedSymbols.create( + self.context, self.config_path, "linux", bash_json_file + ) - ts_offset = self.context.symbol_space.get_type(bash_table_name + constants.BANG + - "hist_entry").relative_child_offset("timestamp") + ts_offset = self.context.symbol_space.get_type( + bash_table_name + constants.BANG + "hist_entry" + ).relative_child_offset("timestamp") for task in tasks: task_name = utility.array_to_string(task.p_comm) @@ -63,49 +75,71 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): bang_addrs = [] # find '#' values on the heap - for address in proc_layer.scan(self.context, - scanners.BytesScanner(b"#"), - sections = task.get_process_memory_sections(self.context, - self.config['kernel'], - rw_no_file = True)): + for address in proc_layer.scan( + self.context, + scanners.BytesScanner(b"#"), + sections=task.get_process_memory_sections( + self.context, self.config["kernel"], rw_no_file=True + ), + ): bang_addrs.append(struct.pack(pack_format, address)) history_entries = [] - for address, _ in proc_layer.scan(self.context, - scanners.MultiStringScanner(bang_addrs), - sections = task.get_process_memory_sections(self.context, - self.config['kernel'], - rw_no_file = True)): - hist = self.context.object(bash_table_name + constants.BANG + "hist_entry", - offset = address - ts_offset, - layer_name = proc_layer_name) + for address, _ in proc_layer.scan( + self.context, + scanners.MultiStringScanner(bang_addrs), + sections=task.get_process_memory_sections( + self.context, self.config["kernel"], rw_no_file=True + ), + ): + hist = self.context.object( + bash_table_name + constants.BANG + "hist_entry", + offset=address - ts_offset, + layer_name=proc_layer_name, + ) if hist.is_valid(): history_entries.append(hist) - for hist in sorted(history_entries, key = lambda x: x.get_time_as_integer()): - yield (0, (int(task.p_pid), task_name, hist.get_time_object(), hist.get_command())) + for hist in sorted(history_entries, key=lambda x: x.get_time_as_integer()): + yield ( + 0, + ( + int(task.p_pid), + task_name, + hist.get_time_object(), + hist.get_command(), + ), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - list_tasks = pslist.PsList.get_list_tasks(self.config.get('pslist_method', pslist.PsList.pslist_methods[0])) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + list_tasks = pslist.PsList.get_list_tasks( + self.config.get("pslist_method", pslist.PsList.pslist_methods[0]) + ) - return renderers.TreeGrid([("PID", int), ("Process", str), ("CommandTime", datetime.datetime), - ("Command", str)], - self._generator( - list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("CommandTime", datetime.datetime), + ("Command", str), + ], + self._generator( + list_tasks(self.context, self.config["kernel"], filter_func=filter_func) + ), + ) def generate_timeline(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - list_tasks = pslist.PsList.get_list_tasks(self.config.get('pslist_method', pslist.PsList.pslist_methods[0])) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + list_tasks = pslist.PsList.get_list_tasks( + self.config.get("pslist_method", pslist.PsList.pslist_methods[0]) + ) for row in self._generator( - list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func)): + list_tasks(self.context, self.config["kernel"], filter_func=filter_func) + ): _depth, row_data = row - description = f"{row_data[0]} ({row_data[1]}): \"{row_data[3]}\"" + description = f'{row_data[0]} ({row_data[1]}): "{row_data[3]}"' yield (description, timeliner.TimeLinerType.CREATED, row_data[2]) diff --git a/volatility3/framework/plugins/mac/check_syscall.py b/volatility3/framework/plugins/mac/check_syscall.py index 9072e76d1..a7a32e9ab 100644 --- a/volatility3/framework/plugins/mac/check_syscall.py +++ b/volatility3/framework/plugins/mac/check_syscall.py @@ -23,20 +23,29 @@ class Check_syscall(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) + mods = lsmod.Lsmod.list_modules(self.context, self.config["kernel"]) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info( + self.context, kernel.layer_name, kernel, mods + ) - table = kernel.object_from_symbol(symbol_name = "sysent") + table = kernel.object_from_symbol(symbol_name="sysent") for (i, ent) in enumerate(table): try: @@ -47,13 +56,31 @@ class Check_syscall(plugins.PluginInterface): if not call_addr or call_addr == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, - call_addr, self.config['kernel']) + module_name, symbol_name = mac.MacUtilities.lookup_module_address( + self.context, handlers, call_addr, self.config["kernel"] + ) - yield (0, (format_hints.Hex(table.vol.offset), "SysCall", i, format_hints.Hex(call_addr), module_name, - symbol_name)) + yield ( + 0, + ( + format_hints.Hex(table.vol.offset), + "SysCall", + i, + format_hints.Hex(call_addr), + module_name, + symbol_name, + ), + ) def run(self): - return renderers.TreeGrid([("Table Address", format_hints.Hex), ("Table Name", str), ("Index", int), - ("Handler Address", format_hints.Hex), ("Handler Module", str), - ("Handler Symbol", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Table Address", format_hints.Hex), + ("Table Name", str), + ("Index", int), + ("Handler Address", format_hints.Hex), + ("Handler Module", str), + ("Handler Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/check_sysctl.py b/volatility3/framework/plugins/mac/check_sysctl.py index fc6ab5c37..165aad436 100644 --- a/volatility3/framework/plugins/mac/check_sysctl.py +++ b/volatility3/framework/plugins/mac/check_sysctl.py @@ -25,10 +25,17 @@ class Check_sysctl(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), ] def _parse_global_variable_sysctls(self, kernel, name): @@ -43,7 +50,7 @@ class Check_sysctl(plugins.PluginInterface): var_name = known_sysctls[name] try: - var_array = kernel.object_from_symbol(symbol_name = var_name) + var_array = kernel.object_from_symbol(symbol_name=var_name) except exceptions.SymbolError: var_array = None @@ -52,7 +59,7 @@ class Check_sysctl(plugins.PluginInterface): return var_str - def _process_sysctl_list(self, kernel, sysctl_list, recursive = 0): + def _process_sysctl_list(self, kernel, sysctl_list, recursive=0): if type(sysctl_list) == volatility3.framework.objects.Pointer: sysctl_list = sysctl_list.dereference().cast("sysctl_oid_list") @@ -84,20 +91,22 @@ class Check_sysctl(plugins.PluginInterface): if arg1 == 0 or arg1_ptr == 0: val = self._parse_global_variable_sysctls(kernel, name) - elif ctltype == 'CTLTYPE_NODE': + elif ctltype == "CTLTYPE_NODE": if sysctl.oid_handler == 0: - for info in self._process_sysctl_list(kernel, sysctl.oid_arg1, recursive = 1): + for info in self._process_sysctl_list( + kernel, sysctl.oid_arg1, recursive=1 + ): yield info val = "Node" - elif ctltype in ['CTLTYPE_INT', 'CTLTYPE_QUAD', 'CTLTYPE_OPAQUE']: + elif ctltype in ["CTLTYPE_INT", "CTLTYPE_QUAD", "CTLTYPE_OPAQUE"]: try: val = str(arg1.dereference().cast("int")) except exceptions.InvalidAddressException: val = "-1" - elif ctltype == 'CTLTYPE_STRING': + elif ctltype == "CTLTYPE_STRING": try: val = utility.pointer_to_string(sysctl.oid_arg1, 64) except exceptions.InvalidAddressException: @@ -113,13 +122,15 @@ class Check_sysctl(plugins.PluginInterface): break def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) + mods = lsmod.Lsmod.list_modules(self.context, self.config["kernel"]) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info( + self.context, kernel.layer_name, kernel, mods + ) - sysctl_list = kernel.object_from_symbol(symbol_name = "sysctl__children") + sysctl_list = kernel.object_from_symbol(symbol_name="sysctl__children") for sysctl, name, val in self._process_sysctl_list(kernel, sysctl_list): try: @@ -127,13 +138,33 @@ class Check_sysctl(plugins.PluginInterface): except exceptions.InvalidAddressException: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, check_addr, - self.config['kernel']) + module_name, symbol_name = mac.MacUtilities.lookup_module_address( + self.context, handlers, check_addr, self.config["kernel"] + ) - yield (0, (name, sysctl.oid_number, sysctl.get_perms(), format_hints.Hex(check_addr), val, module_name, - symbol_name)) + yield ( + 0, + ( + name, + sysctl.oid_number, + sysctl.get_perms(), + format_hints.Hex(check_addr), + val, + module_name, + symbol_name, + ), + ) def run(self): - return renderers.TreeGrid([("Name", str), ("Number", int), ("Perms", str), - ("Handler Address", format_hints.Hex), ("Value", str), ("Handler Module", str), - ("Handler Symbol", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Name", str), + ("Number", int), + ("Perms", str), + ("Handler Address", format_hints.Hex), + ("Value", str), + ("Handler Module", str), + ("Handler Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/check_trap_table.py b/volatility3/framework/plugins/mac/check_trap_table.py index 47d0ed57d..60f237208 100644 --- a/volatility3/framework/plugins/mac/check_trap_table.py +++ b/volatility3/framework/plugins/mac/check_trap_table.py @@ -24,20 +24,29 @@ class Check_trap_table(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 0, 0) + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) + mods = lsmod.Lsmod.list_modules(self.context, self.config["kernel"]) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info( + self.context, kernel.layer_name, kernel, mods + ) - table = kernel.object_from_symbol(symbol_name = "mach_trap_table") + table = kernel.object_from_symbol(symbol_name="mach_trap_table") for i, ent in enumerate(table): try: @@ -48,13 +57,31 @@ class Check_trap_table(plugins.PluginInterface): if not call_addr or call_addr == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, call_addr, - self.config['kernel']) + module_name, symbol_name = mac.MacUtilities.lookup_module_address( + self.context, handlers, call_addr, self.config["kernel"] + ) - yield (0, (format_hints.Hex(table.vol.offset), "TrapTable", i, format_hints.Hex(call_addr), module_name, - symbol_name)) + yield ( + 0, + ( + format_hints.Hex(table.vol.offset), + "TrapTable", + i, + format_hints.Hex(call_addr), + module_name, + symbol_name, + ), + ) def run(self): - return renderers.TreeGrid([("Table Address", format_hints.Hex), ("Table Name", str), ("Index", int), - ("Handler Address", format_hints.Hex), ("Handler Module", str), - ("Handler Symbol", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Table Address", format_hints.Hex), + ("Table Name", str), + ("Index", int), + ("Handler Address", format_hints.Hex), + ("Handler Module", str), + ("Handler Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/ifconfig.py b/volatility3/framework/plugins/mac/ifconfig.py index 330c13f07..6634dda2e 100644 --- a/volatility3/framework/plugins/mac/ifconfig.py +++ b/volatility3/framework/plugins/mac/ifconfig.py @@ -16,18 +16,23 @@ class Ifconfig(plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 0, 0) + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] try: - list_head = kernel.object_from_symbol(symbol_name = "ifnet_head") + list_head = kernel.object_from_symbol(symbol_name="ifnet_head") except exceptions.SymbolError: - list_head = kernel.object_from_symbol(symbol_name = "dlil_ifnet_head") + list_head = kernel.object_from_symbol(symbol_name="dlil_ifnet_head") for ifnet in mac.MacUtilities.walk_tailq(list_head, "if_link"): name = utility.pointer_to_string(ifnet.if_name, 32) @@ -46,5 +51,12 @@ class Ifconfig(plugins.PluginInterface): yield (0, (f"{name}{unit}", ip, mac_addr, prom)) def run(self): - return renderers.TreeGrid([("Interface", str), ("IP Address", str), ("Mac Address", str), - ("Promiscuous", bool)], self._generator()) + return renderers.TreeGrid( + [ + ("Interface", str), + ("IP Address", str), + ("Mac Address", str), + ("Promiscuous", bool), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/kauth_listeners.py b/volatility3/framework/plugins/mac/kauth_listeners.py index fba6a8e0a..0b945a8fd 100644 --- a/volatility3/framework/plugins/mac/kauth_listeners.py +++ b/volatility3/framework/plugins/mac/kauth_listeners.py @@ -11,33 +11,44 @@ from volatility3.plugins.mac import lsmod, kauth_scopes class Kauth_listeners(interfaces.plugins.PluginInterface): - """ Lists kauth listeners and their status """ + """Lists kauth listeners and their status""" _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 1, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)), - requirements.PluginRequirement(name = 'kauth_scopes', - plugin = kauth_scopes.Kauth_scopes, - version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 1, 0) + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="kauth_scopes", plugin=kauth_scopes.Kauth_scopes, version=(2, 0, 0) + ), ] def _generator(self): """ Enumerates the listeners for each kauth scope """ - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) + mods = lsmod.Lsmod.list_modules(self.context, self.config["kernel"]) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info( + self.context, kernel.layer_name, kernel, mods + ) - for scope in kauth_scopes.Kauth_scopes.list_kauth_scopes(self.context, self.config['kernel']): + for scope in kauth_scopes.Kauth_scopes.list_kauth_scopes( + self.context, self.config["kernel"] + ): scope_name = utility.pointer_to_string(scope.ks_identifier, 128) @@ -46,12 +57,29 @@ class Kauth_listeners(interfaces.plugins.PluginInterface): if callback == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, callback, - self.config['kernel']) + module_name, symbol_name = mac.MacUtilities.lookup_module_address( + self.context, handlers, callback, self.config["kernel"] + ) - yield (0, (scope_name, format_hints.Hex(listener.kll_idata), format_hints.Hex(callback), module_name, - symbol_name)) + yield ( + 0, + ( + scope_name, + format_hints.Hex(listener.kll_idata), + format_hints.Hex(callback), + module_name, + symbol_name, + ), + ) def run(self): - return renderers.TreeGrid([("Name", str), ("IData", format_hints.Hex), ("Callback Address", format_hints.Hex), - ("Module", str), ("Symbol", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Name", str), + ("IData", format_hints.Hex), + ("Callback Address", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/kauth_scopes.py b/volatility3/framework/plugins/mac/kauth_scopes.py index f1a2ad345..bfd7216a8 100644 --- a/volatility3/framework/plugins/mac/kauth_scopes.py +++ b/volatility3/framework/plugins/mac/kauth_scopes.py @@ -15,7 +15,7 @@ vollog = logging.getLogger(__name__) class Kauth_scopes(interfaces.plugins.PluginInterface): - """ Lists kauth scopes and their status """ + """Lists kauth scopes and their status""" _version = (2, 0, 0) _required_framework_version = (2, 0, 0) @@ -23,18 +23,26 @@ class Kauth_scopes(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 1, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 1, 0) + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), ] @classmethod - def list_kauth_scopes(cls, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def list_kauth_scopes( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + filter_func: Callable[[int], bool] = lambda _: False, + ) -> Iterable[interfaces.objects.ObjectInterface]: """ Enumerates the registered kauth scopes and yields each object Uses smear-safe enumeration API @@ -48,27 +56,47 @@ class Kauth_scopes(interfaces.plugins.PluginInterface): yield scope def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) + mods = lsmod.Lsmod.list_modules(self.context, self.config["kernel"]) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info( + self.context, kernel.layer_name, kernel, mods + ) - for scope in self.list_kauth_scopes(self.context, self.config['kernel']): + for scope in self.list_kauth_scopes(self.context, self.config["kernel"]): callback = scope.ks_callback if callback == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, callback, - self.config['kernel']) + module_name, symbol_name = mac.MacUtilities.lookup_module_address( + self.context, handlers, callback, self.config["kernel"] + ) identifier = utility.pointer_to_string(scope.ks_identifier, 128) - yield (0, (identifier, format_hints.Hex(scope.ks_idata), len([l for l in scope.get_listeners()]), - format_hints.Hex(callback), module_name, symbol_name)) + yield ( + 0, + ( + identifier, + format_hints.Hex(scope.ks_idata), + len([l for l in scope.get_listeners()]), + format_hints.Hex(callback), + module_name, + symbol_name, + ), + ) def run(self): - return renderers.TreeGrid([("Name", str), ("IData", format_hints.Hex), ("Listeners", int), - ("Callback Address", format_hints.Hex), ("Module", str), ("Symbol", str)], - self._generator()) + return renderers.TreeGrid( + [ + ("Name", str), + ("IData", format_hints.Hex), + ("Listeners", int), + ("Callback Address", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 4a82d81cd..74c5f6037 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -12,7 +12,7 @@ from volatility3.plugins.mac import pslist class Kevents(interfaces.plugins.PluginInterface): - """ Lists event handlers registered by processes """ + """Lists event handlers registered by processes""" _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @@ -28,34 +28,61 @@ class Kevents(interfaces.plugins.PluginInterface): 8: "EVFILT_MACHPORT", 9: "EVFILT_FS", 10: "EVFILT_USER", - 12: "EVFILT_VM" + 12: "EVFILT_VM", } - vnode_filters = [("NOTE_DELETE", 1), ("NOTE_WRITE", 2), ("NOTE_EXTEND", 4), ("NOTE_ATTRIB", 8), ("NOTE_LINK", 0x10), - ("NOTE_RENAME", 0x20), ("NOTE_REVOKE", 0x40)] + vnode_filters = [ + ("NOTE_DELETE", 1), + ("NOTE_WRITE", 2), + ("NOTE_EXTEND", 4), + ("NOTE_ATTRIB", 8), + ("NOTE_LINK", 0x10), + ("NOTE_RENAME", 0x20), + ("NOTE_REVOKE", 0x40), + ] - proc_filters = [("NOTE_EXIT", 0x80000000), ("NOTE_EXITSTATUS", 0x04000000), ("NOTE_FORK", 0x40000000), - ("NOTE_EXEC", 0x20000000), ("NOTE_SIGNAL", 0x08000000), ("NOTE_REAP", 0x10000000)] + proc_filters = [ + ("NOTE_EXIT", 0x80000000), + ("NOTE_EXITSTATUS", 0x04000000), + ("NOTE_FORK", 0x40000000), + ("NOTE_EXEC", 0x20000000), + ("NOTE_SIGNAL", 0x08000000), + ("NOTE_REAP", 0x10000000), + ] - timer_filters = [("NOTE_SECONDS", 1), ("NOTE_USECONDS", 2), ("NOTE_NSECONDS", 4), ("NOTE_ABSOLUTE", 8)] + timer_filters = [ + ("NOTE_SECONDS", 1), + ("NOTE_USECONDS", 2), + ("NOTE_NSECONDS", 4), + ("NOTE_ABSOLUTE", 8), + ] all_filters = { 4: vnode_filters, # EVFILT_VNODE 5: proc_filters, # EVFILT_PROC - 7: timer_filters # EVFILT_TIMER + 7: timer_filters, # EVFILT_TIMER } @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 2, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 2, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] def _parse_flags(self, filter_index, filter_flags): @@ -81,10 +108,12 @@ class Kevents(interfaces.plugins.PluginInterface): klist_array_pointer = getattr(fdp, array_pointer_member) array_size = getattr(fdp, array_size_member) - klist_array = kernel.object(object_type = "array", - offset = klist_array_pointer, - count = array_size + 1, - subtype = kernel.get_type("klist")) + klist_array = kernel.object( + object_type="array", + offset=klist_array_pointer, + count=array_size + 1, + subtype=kernel.get_type("klist"), + ) except exceptions.InvalidAddressException: return @@ -117,13 +146,18 @@ class Kevents(interfaces.plugins.PluginInterface): yield kn @classmethod - def list_kernel_events(cls, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False) -> \ - Iterable[Tuple[interfaces.objects.ObjectInterface, - interfaces.objects.ObjectInterface, - interfaces.objects.ObjectInterface]]: + def list_kernel_events( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + filter_func: Callable[[int], bool] = lambda _: False, + ) -> Iterable[ + Tuple[ + interfaces.objects.ObjectInterface, + interfaces.objects.ObjectInterface, + interfaces.objects.ObjectInterface, + ] + ]: """ Returns the kernel event filters registered @@ -145,11 +179,11 @@ class Kevents(interfaces.plugins.PluginInterface): yield task_name, pid, kn def _generator(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - for task_name, pid, kn in self.list_kernel_events(self.context, - self.config['kernel'], - filter_func = filter_func): + for task_name, pid, kn in self.list_kernel_events( + self.context, self.config["kernel"], filter_func=filter_func + ): filter_index = kn.kn_kevent.filter * -1 if filter_index in self.event_types: @@ -167,5 +201,13 @@ class Kevents(interfaces.plugins.PluginInterface): yield (0, (pid, task_name, ident, filter_name, context)) def run(self): - return renderers.TreeGrid([("PID", int), ("Process", str), ("Ident", int), ("Filter", str), ("Context", str)], - self._generator()) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Ident", int), + ("Filter", str), + ("Context", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/list_files.py b/volatility3/framework/plugins/mac/list_files.py index 8bae986b7..ede0fa32b 100644 --- a/volatility3/framework/plugins/mac/list_files.py +++ b/volatility3/framework/plugins/mac/list_files.py @@ -23,9 +23,14 @@ class List_Files(plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'mount', plugin = mount.Mount, version = (2, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="mount", plugin=mount.Mount, version=(2, 0, 0) + ), ] @classmethod @@ -50,8 +55,9 @@ class List_Files(plugins.PluginInterface): except exceptions.InvalidAddressException: return None - if parent and not context.layers[vnode.vol.native_layer_name].is_valid(parent.vol.offset, - parent.vol.size): + if parent and not context.layers[vnode.vol.native_layer_name].is_valid( + parent.vol.offset, parent.vol.size + ): return None return parent @@ -65,8 +71,9 @@ class List_Files(plugins.PluginInterface): and holds its name, parent address, and object """ - if not context.layers[vnode.vol.native_layer_name].is_valid(vnode.vol.offset, - vnode.vol.size): + if not context.layers[vnode.vol.native_layer_name].is_valid( + vnode.vol.offset, vnode.vol.size + ): return False key = vnode.vol.offset @@ -104,7 +111,7 @@ class List_Files(plugins.PluginInterface): if not cls._add_vnode(context, vnode, loop_vnodes): break - + added = True parent = cls._get_parent(context, vnode) @@ -127,10 +134,9 @@ class List_Files(plugins.PluginInterface): cls._walk_vnode(context, vnode, loop_vnodes) @classmethod - def _walk_mounts(cls, - context: interfaces.context.ContextInterface, - kernel_module_name: str) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def _walk_mounts( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str + ) -> Iterable[interfaces.objects.ObjectInterface]: loop_vnodes = {} @@ -177,10 +183,9 @@ class List_Files(plugins.PluginInterface): return path @classmethod - def list_files(cls, - context: interfaces.context.ContextInterface, - kernel_module_name: str) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def list_files( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str + ) -> Iterable[interfaces.objects.ObjectInterface]: vnodes = cls._walk_mounts(context, kernel_module_name) @@ -190,9 +195,11 @@ class List_Files(plugins.PluginInterface): yield vnode, full_path def _generator(self): - for vnode, full_path in self.list_files(self.context, self.config['kernel']): + for vnode, full_path in self.list_files(self.context, self.config["kernel"]): yield (0, (format_hints.Hex(vnode.vol.offset), full_path)) def run(self): - return renderers.TreeGrid([("Address", format_hints.Hex), ("File Path", str)], self._generator()) + return renderers.TreeGrid( + [("Address", format_hints.Hex), ("File Path", str)], self._generator() + ) diff --git a/volatility3/framework/plugins/mac/lsmod.py b/volatility3/framework/plugins/mac/lsmod.py index 345267fea..2cdd5e3de 100644 --- a/volatility3/framework/plugins/mac/lsmod.py +++ b/volatility3/framework/plugins/mac/lsmod.py @@ -22,12 +22,17 @@ class Lsmod(plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), ] @classmethod - def list_modules(cls, context: interfaces.context.ContextInterface, darwin_module_name: str): + def list_modules( + cls, context: interfaces.context.ContextInterface, darwin_module_name: str + ): """Lists all the modules in the primary layer. Args: @@ -41,25 +46,23 @@ class Lsmod(plugins.PluginInterface): kernel = context.modules[darwin_module_name] kernel_layer = context.layers[kernel.layer_name] - kmod_ptr = kernel.object_from_symbol(symbol_name = "kmod") + kmod_ptr = kernel.object_from_symbol(symbol_name="kmod") try: kmod = kmod_ptr.dereference().cast("kmod_info") except exceptions.InvalidAddressException: - return # Generation finished + return # Generation finished yield kmod try: kmod = kmod.next except exceptions.InvalidAddressException: - return # Generation finished + return # Generation finished seen: Set = set() - while kmod != 0 and \ - kmod not in seen and \ - len(seen) < 1024: + while kmod != 0 and kmod not in seen and len(seen) < 1024: kmod_obj = kmod.dereference() @@ -74,10 +77,10 @@ class Lsmod(plugins.PluginInterface): kmod = kmod.next except exceptions.InvalidAddressException: return - return # Generation finished + return # Generation finished def _generator(self): - for module in self.list_modules(self.context, self.config['kernel']): + for module in self.list_modules(self.context, self.config["kernel"]): mod_name = utility.array_to_string(module.name) mod_size = module.size @@ -85,4 +88,7 @@ class Lsmod(plugins.PluginInterface): yield 0, (format_hints.Hex(module.vol.offset), mod_name, mod_size) def run(self): - return renderers.TreeGrid([("Offset", format_hints.Hex), ("Name", str), ("Size", int)], self._generator()) + return renderers.TreeGrid( + [("Offset", format_hints.Hex), ("Name", str), ("Size", int)], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/lsof.py b/volatility3/framework/plugins/mac/lsof.py index c3941ec27..6832b837f 100644 --- a/volatility3/framework/plugins/mac/lsof.py +++ b/volatility3/framework/plugins/mac/lsof.py @@ -21,33 +21,45 @@ class Lsof(plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] def _generator(self, tasks): - darwin = self.context.modules[self.config['kernel']] + darwin = self.context.modules[self.config["kernel"]] for task in tasks: pid = task.p_pid - for _, filepath, fd in mac.MacUtilities.files_descriptors_for_process(self.context, - darwin.symbol_table_name, - task): + for _, filepath, fd in mac.MacUtilities.files_descriptors_for_process( + self.context, darwin.symbol_table_name, task + ): if filepath and len(filepath) > 0: yield (0, (pid, fd, filepath)) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - list_tasks = pslist.PsList.get_list_tasks(self.config.get('pslist_method', pslist.PsList.pslist_methods[0])) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + list_tasks = pslist.PsList.get_list_tasks( + self.config.get("pslist_method", pslist.PsList.pslist_methods[0]) + ) - return renderers.TreeGrid([("PID", int), ("File Descriptor", int), ("File Path", str)], - self._generator( - list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [("PID", int), ("File Descriptor", int), ("File Path", str)], + self._generator( + list_tasks(self.context, self.config["kernel"], filter_func=filter_func) + ), + ) diff --git a/volatility3/framework/plugins/mac/malfind.py b/volatility3/framework/plugins/mac/malfind.py index 7a42a0c5f..98b282e24 100644 --- a/volatility3/framework/plugins/mac/malfind.py +++ b/volatility3/framework/plugins/mac/malfind.py @@ -18,13 +18,20 @@ class Malfind(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] def _list_injections(self, task): @@ -38,13 +45,16 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.get_map_iter(): - if not vma.is_suspicious(self.context, self.context.modules[self.config['kernel']].symbol_table_name): - data = proc_layer.read(vma.links.start, 64, pad = True) + if not vma.is_suspicious( + self.context, + self.context.modules[self.config["kernel"]].symbol_table_name, + ): + data = proc_layer.read(vma.links.start, 64, pad=True) yield vma, data def _generator(self, tasks): # determine if we're on a 32 or 64 bit kernel - if self.context.modules[self.config['kernel']].get_type("pointer").size == 4: + if self.context.modules[self.config["kernel"]].get_type("pointer").size == 4: is_32bit_arch = True else: is_32bit_arch = False @@ -58,19 +68,40 @@ class Malfind(interfaces.plugins.PluginInterface): else: architecture = "intel64" - disasm = interfaces.renderers.Disassembly(data, vma.links.start, architecture) + disasm = interfaces.renderers.Disassembly( + data, vma.links.start, architecture + ) - yield (0, (task.p_pid, process_name, format_hints.Hex(vma.links.start), format_hints.Hex(vma.links.end), - vma.get_perms(), format_hints.HexBytes(data), disasm)) + yield ( + 0, + ( + task.p_pid, + process_name, + format_hints.Hex(vma.links.start), + format_hints.Hex(vma.links.end), + vma.get_perms(), + format_hints.HexBytes(data), + disasm, + ), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - list_tasks = pslist.PsList.get_list_tasks(self.config.get('pslist_method', pslist.PsList.pslist_methods[0])) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + list_tasks = pslist.PsList.get_list_tasks( + self.config.get("pslist_method", pslist.PsList.pslist_methods[0]) + ) - return renderers.TreeGrid([("PID", int), ("Process", str), ("Start", format_hints.Hex), - ("End", format_hints.Hex), ("Protection", str), ("Hexdump", format_hints.HexBytes), - ("Disasm", interfaces.renderers.Disassembly)], - self._generator( - list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Start", format_hints.Hex), + ("End", format_hints.Hex), + ("Protection", str), + ("Hexdump", format_hints.HexBytes), + ("Disasm", interfaces.renderers.Disassembly), + ], + self._generator( + list_tasks(self.context, self.config["kernel"], filter_func=filter_func) + ), + ) diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index ba3ab83c8..ff654e1a7 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -21,13 +21,20 @@ class Mount(plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 0, 0) + ), ] @classmethod - def list_mounts(cls, context: interfaces.context.ContextInterface, kernel_module_name: str): + def list_mounts( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str + ): """Lists all the mount structures in the primary layer. Args: @@ -40,13 +47,13 @@ class Mount(plugins.PluginInterface): """ kernel = context.modules[kernel_module_name] - list_head = kernel.object_from_symbol(symbol_name = "mountlist") + list_head = kernel.object_from_symbol(symbol_name="mountlist") for mount in mac.MacUtilities.walk_tailq(list_head, "mnt_list"): yield mount def _generator(self): - for mount in self.list_mounts(self.context, self.config['kernel']): + for mount in self.list_mounts(self.context, self.config["kernel"]): vfs = mount.mnt_vfsstat device_name = utility.array_to_string(vfs.f_mntonname) mount_point = utility.array_to_string(vfs.f_mntfromname) @@ -55,4 +62,6 @@ class Mount(plugins.PluginInterface): yield 0, (device_name, mount_point, mount_type) def run(self): - return renderers.TreeGrid([("Device", str), ("Mount Point", str), ("Type", str)], self._generator()) + return renderers.TreeGrid( + [("Device", str), ("Mount Point", str), ("Type", str)], self._generator() + ) diff --git a/volatility3/framework/plugins/mac/netstat.py b/volatility3/framework/plugins/mac/netstat.py index e231b8082..581a9c67f 100644 --- a/volatility3/framework/plugins/mac/netstat.py +++ b/volatility3/framework/plugins/mac/netstat.py @@ -24,24 +24,38 @@ class Netstat(plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] @classmethod - def list_sockets(cls, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False) -> \ - Iterable[Tuple[interfaces.objects.ObjectInterface, - interfaces.objects.ObjectInterface, - interfaces.objects.ObjectInterface]]: + def list_sockets( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + filter_func: Callable[[int], bool] = lambda _: False, + ) -> Iterable[ + Tuple[ + interfaces.objects.ObjectInterface, + interfaces.objects.ObjectInterface, + interfaces.objects.ObjectInterface, + ] + ]: """ Returns the open socket descriptors of a process @@ -58,14 +72,15 @@ class Netstat(plugins.PluginInterface): task_name = utility.array_to_string(task.p_comm) pid = task.p_pid - for filp, _, _ in mac.MacUtilities.files_descriptors_for_process(context, context.modules[ - kernel_module_name].symbol_table_name, task): + for filp, _, _ in mac.MacUtilities.files_descriptors_for_process( + context, context.modules[kernel_module_name].symbol_table_name, task + ): try: ftype = filp.f_fglob.get_fg_type() except exceptions.InvalidAddressException: continue - if ftype != 'SOCKET': + if ftype != "SOCKET": continue try: @@ -73,18 +88,19 @@ class Netstat(plugins.PluginInterface): except exceptions.InvalidAddressException: continue - if not context.layers[task.vol.native_layer_name].is_valid(socket.vol.offset, - socket.vol.size): + if not context.layers[task.vol.native_layer_name].is_valid( + socket.vol.offset, socket.vol.size + ): continue yield task_name, pid, socket def _generator(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - for task_name, pid, socket in self.list_sockets(self.context, - self.config['kernel'], - filter_func = filter_func): + for task_name, pid, socket in self.list_sockets( + self.context, self.config["kernel"], filter_func=filter_func + ): family = socket.get_family() @@ -95,8 +111,19 @@ class Netstat(plugins.PluginInterface): except exceptions.InvalidAddressException: continue - yield (0, (format_hints.Hex(socket.vol.offset), "UNIX", path, 0, "", 0, "", - f"{task_name}/{pid:d}")) + yield ( + 0, + ( + format_hints.Hex(socket.vol.offset), + "UNIX", + path, + 0, + "", + 0, + "", + f"{task_name}/{pid:d}", + ), + ) elif family in [2, 30]: state = socket.get_state() @@ -107,10 +134,31 @@ class Netstat(plugins.PluginInterface): if vals: (lip, lport, rip, rport) = vals - yield (0, (format_hints.Hex(socket.vol.offset), proto, lip, lport, rip, rport, state, - f"{task_name}/{pid:d}")) + yield ( + 0, + ( + format_hints.Hex(socket.vol.offset), + proto, + lip, + lport, + rip, + rport, + state, + f"{task_name}/{pid:d}", + ), + ) def run(self): - return renderers.TreeGrid([("Offset", format_hints.Hex), ("Proto", str), ("Local IP", str), ("Local Port", int), - ("Remote IP", str), ("Remote Port", int), ("State", str), ("Process", str)], - self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Proto", str), + ("Local IP", str), + ("Local Port", int), + ("Remote IP", str), + ("Remote Port", int), + ("State", str), + ("Process", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 70c9684cd..781b3ed66 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -17,13 +17,20 @@ class Maps(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] def _generator(self, tasks): @@ -32,20 +39,41 @@ class Maps(interfaces.plugins.PluginInterface): process_pid = task.p_pid for vma in task.get_map_iter(): - path = vma.get_path(self.context, self.context.modules[self.config['kernel']].symbol_table_name) + path = vma.get_path( + self.context, + self.context.modules[self.config["kernel"]].symbol_table_name, + ) if path == "": path = vma.get_special_path() - yield (0, (process_pid, process_name, format_hints.Hex(vma.links.start), - format_hints.Hex(vma.links.end), vma.get_perms(), path)) + yield ( + 0, + ( + process_pid, + process_name, + format_hints.Hex(vma.links.start), + format_hints.Hex(vma.links.end), + vma.get_perms(), + path, + ), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - list_tasks = pslist.PsList.get_list_tasks(self.config.get('pslist_method', pslist.PsList.pslist_methods[0])) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + list_tasks = pslist.PsList.get_list_tasks( + self.config.get("pslist_method", pslist.PsList.pslist_methods[0]) + ) - return renderers.TreeGrid([("PID", int), ("Process", str), ("Start", format_hints.Hex), - ("End", format_hints.Hex), ("Protection", str), ("Map Name", str)], - self._generator( - list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Start", format_hints.Hex), + ("End", format_hints.Hex), + ("Protection", str), + ("Map Name", str), + ], + self._generator( + list_tasks(self.context, self.config["kernel"], filter_func=filter_func) + ), + ) diff --git a/volatility3/framework/plugins/mac/psaux.py b/volatility3/framework/plugins/mac/psaux.py index e3fcdc0bb..28c238263 100644 --- a/volatility3/framework/plugins/mac/psaux.py +++ b/volatility3/framework/plugins/mac/psaux.py @@ -19,16 +19,25 @@ class Psaux(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] - def _generator(self, tasks: Iterator[Any]) -> Generator[Tuple[int, Tuple[int, str, int, str]], None, None]: + def _generator( + self, tasks: Iterator[Any] + ) -> Generator[Tuple[int, Tuple[int, str, int, str]], None, None]: for task in tasks: proc_layer_name = task.add_process_layer() if proc_layer_name is None: @@ -38,7 +47,11 @@ class Psaux(plugins.PluginInterface): argsstart = task.user_stack - task.p_argslen - if not proc_layer.is_valid(argsstart) or task.p_argslen == 0 or task.p_argc == 0: + if ( + not proc_layer.is_valid(argsstart) + or task.p_argslen == 0 + or task.p_argc == 0 + ): continue # Add one because the first two are usually duplicates @@ -58,7 +71,7 @@ class Psaux(plugins.PluginInterface): except exceptions.InvalidAddressException: break - idx = arg.find(b'\x00') + idx = arg.find(b"\x00") if idx != -1: arg = arg[:idx] @@ -85,16 +98,19 @@ class Psaux(plugins.PluginInterface): argc = argc - 1 - args_str = " ".join([s.decode("utf-8", errors = 'replace') for s in args]) + args_str = " ".join([s.decode("utf-8", errors="replace") for s in args]) yield (0, (task.p_pid, task_name, task.p_argc, args_str)) def run(self) -> renderers.TreeGrid: - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - list_tasks = pslist.PsList.get_list_tasks(self.config.get('pslist_method', pslist.PsList.pslist_methods[0])) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + list_tasks = pslist.PsList.get_list_tasks( + self.config.get("pslist_method", pslist.PsList.pslist_methods[0]) + ) - return renderers.TreeGrid([("PID", int), ("Process", str), ("Argc", int), ("Arguments", str)], - self._generator( - list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) + return renderers.TreeGrid( + [("PID", int), ("Process", str), ("Argc", int), ("Arguments", str)], + self._generator( + list_tasks(self.context, self.config["kernel"], filter_func=filter_func) + ), + ) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index e92609b3a..c2ae71e7e 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -18,30 +18,41 @@ class PsList(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (3, 0, 0) - pslist_methods = ['tasks', 'allproc', 'process_group', 'sessions', 'pid_hash_table'] + pslist_methods = ["tasks", "allproc", "process_group", "sessions", "pid_hash_table"] @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 1, 0)), - requirements.ChoiceRequirement(name = 'pslist_method', - description = 'Method to determine for processes', - choices = cls.pslist_methods, - default = cls.pslist_methods[0], - optional = True), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 1, 0) + ), + requirements.ChoiceRequirement( + name="pslist_method", + description="Method to determine for processes", + choices=cls.pslist_methods, + default=cls.pslist_methods[0], + optional=True, + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), ] @classmethod def get_list_tasks( - cls, method: str - ) -> Callable[[interfaces.context.ContextInterface, str, Callable[[int], bool]], - Iterable[interfaces.objects.ObjectInterface]]: + cls, method: str + ) -> Callable[ + [interfaces.context.ContextInterface, str, Callable[[int], bool]], + Iterable[interfaces.objects.ObjectInterface], + ]: """Returns the list_tasks method based on the selector Args: @@ -49,20 +60,20 @@ class PsList(interfaces.plugins.PluginInterface): Returns: list_tasks method for listing tasks - """ + """ # Ensure method is one of the suitable choices if method not in cls.pslist_methods: method = cls.pslist_methods[0] - if method == 'allproc': + if method == "allproc": list_tasks = cls.list_tasks_allproc - elif method == 'tasks': + elif method == "tasks": list_tasks = cls.list_tasks_tasks - elif method == 'process_group': + elif method == "process_group": list_tasks = cls.list_tasks_process_group - elif method == 'sessions': + elif method == "sessions": list_tasks = cls.list_tasks_sessions - elif method == 'pid_hash_table': + elif method == "pid_hash_table": list_tasks = cls.list_tasks_pid_hash_table else: raise ValueError("Impossible method choice chosen") @@ -86,22 +97,27 @@ class PsList(interfaces.plugins.PluginInterface): return filter_func def _generator(self): - list_tasks = self.get_list_tasks(self.config.get('pslist_method', self.pslist_methods[0])) + list_tasks = self.get_list_tasks( + self.config.get("pslist_method", self.pslist_methods[0]) + ) - for task in list_tasks(self.context, - self.config['kernel'], - filter_func = self.create_pid_filter(self.config.get('pid', None))): + for task in list_tasks( + self.context, + self.config["kernel"], + filter_func=self.create_pid_filter(self.config.get("pid", None)), + ): pid = task.p_pid ppid = task.p_ppid name = utility.array_to_string(task.p_comm) yield (0, (pid, ppid, name)) @classmethod - def list_tasks_allproc(cls, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def list_tasks_allproc( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + filter_func: Callable[[int], bool] = lambda _: False, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the processes in the primary layer based on the allproc method Args: @@ -117,17 +133,22 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - proc = kernel.object_from_symbol(symbol_name = "allproc").lh_first + proc = kernel.object_from_symbol(symbol_name="allproc").lh_first seen: Dict[int, int] = {} while proc is not None and proc.vol.offset != 0: if proc.vol.offset in seen: - vollog.log(logging.INFO, "Recursive process list detected (a result of non-atomic acquisition).") + vollog.log( + logging.INFO, + "Recursive process list detected (a result of non-atomic acquisition).", + ) break else: seen[proc.vol.offset] = 1 - if kernel_layer.is_valid(proc.vol.offset, proc.vol.size) and not filter_func(proc): + if kernel_layer.is_valid( + proc.vol.offset, proc.vol.size + ) and not filter_func(proc): yield proc try: @@ -136,11 +157,12 @@ class PsList(interfaces.plugins.PluginInterface): break @classmethod - def list_tasks_tasks(cls, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def list_tasks_tasks( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + filter_func: Callable[[int], bool] = lambda _: False, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer based on the tasks queue Args: @@ -155,12 +177,15 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object_from_symbol(symbol_name = "tasks") + queue_entry = kernel.object_from_symbol(symbol_name="tasks") seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): if task.vol.offset in seen: - vollog.log(logging.INFO, "Recursive process list detected (a result of non-atomic acquisition).") + vollog.log( + logging.INFO, + "Recursive process list detected (a result of non-atomic acquisition).", + ) break else: seen[task.vol.offset] = 1 @@ -170,15 +195,18 @@ class PsList(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: continue - if kernel_layer.is_valid(proc.vol.offset, proc.vol.size) and not filter_func(proc): + if kernel_layer.is_valid( + proc.vol.offset, proc.vol.size + ) and not filter_func(proc): yield proc @classmethod - def list_tasks_sessions(cls, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def list_tasks_sessions( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + filter_func: Callable[[int], bool] = lambda _: False, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer using sessions Args: @@ -191,14 +219,16 @@ class PsList(interfaces.plugins.PluginInterface): """ kernel = context.modules[kernel_module_name] - table_size = kernel.object_from_symbol(symbol_name = "sesshash") + table_size = kernel.object_from_symbol(symbol_name="sesshash") - sesshashtbl = kernel.object_from_symbol(symbol_name = "sesshashtbl") + sesshashtbl = kernel.object_from_symbol(symbol_name="sesshashtbl") - proc_array = kernel.object(object_type = "array", - offset = sesshashtbl, - count = table_size + 1, - subtype = kernel.get_type("sesshashhead")) + proc_array = kernel.object( + object_type="array", + offset=sesshashtbl, + count=table_size + 1, + subtype=kernel.get_type("sesshashhead"), + ) for proc_list in proc_array: for proc in mac.MacUtilities.walk_list_head(proc_list, "s_hash"): @@ -206,11 +236,12 @@ class PsList(interfaces.plugins.PluginInterface): yield proc.s_leader @classmethod - def list_tasks_process_group(cls, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def list_tasks_process_group( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + filter_func: Callable[[int], bool] = lambda _: False, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer using process groups Args: @@ -223,27 +254,32 @@ class PsList(interfaces.plugins.PluginInterface): """ kernel = context.modules[kernel_module_name] - table_size = kernel.object_from_symbol(symbol_name = "pgrphash") + table_size = kernel.object_from_symbol(symbol_name="pgrphash") - pgrphashtbl = kernel.object_from_symbol(symbol_name = "pgrphashtbl") + pgrphashtbl = kernel.object_from_symbol(symbol_name="pgrphashtbl") - proc_array = kernel.object(object_type = "array", - offset = pgrphashtbl, - count = table_size + 1, - subtype = kernel.get_type("pgrphashhead")) + proc_array = kernel.object( + object_type="array", + offset=pgrphashtbl, + count=table_size + 1, + subtype=kernel.get_type("pgrphashhead"), + ) for proc_list in proc_array: for pgrp in mac.MacUtilities.walk_list_head(proc_list, "pg_hash"): - for proc in mac.MacUtilities.walk_list_head(pgrp.pg_members, "p_pglist"): + for proc in mac.MacUtilities.walk_list_head( + pgrp.pg_members, "p_pglist" + ): if not filter_func(proc): yield proc @classmethod - def list_tasks_pid_hash_table(cls, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def list_tasks_pid_hash_table( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + filter_func: Callable[[int], bool] = lambda _: False, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer using the pid hash table Args: @@ -257,14 +293,16 @@ class PsList(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] - table_size = kernel.object_from_symbol(symbol_name = "pidhash") + table_size = kernel.object_from_symbol(symbol_name="pidhash") - pidhashtbl = kernel.object_from_symbol(symbol_name = "pidhashtbl") + pidhashtbl = kernel.object_from_symbol(symbol_name="pidhashtbl") - proc_array = kernel.object(object_type = "array", - offset = pidhashtbl, - count = table_size + 1, - subtype = kernel.get_type("pidhashhead")) + proc_array = kernel.object( + object_type="array", + offset=pidhashtbl, + count=table_size + 1, + subtype=kernel.get_type("pidhashhead"), + ) for proc_list in proc_array: for proc in mac.MacUtilities.walk_list_head(proc_list, "p_hash"): @@ -272,4 +310,6 @@ class PsList(interfaces.plugins.PluginInterface): yield proc def run(self): - return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str)], self._generator()) + return renderers.TreeGrid( + [("PID", int), ("PPID", int), ("COMM", str)], self._generator() + ) diff --git a/volatility3/framework/plugins/mac/pstree.py b/volatility3/framework/plugins/mac/pstree.py index d7fb0eab4..e62d5eb72 100644 --- a/volatility3/framework/plugins/mac/pstree.py +++ b/volatility3/framework/plugins/mac/pstree.py @@ -24,9 +24,14 @@ class PsTree(plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), ] def _find_level(self, pid): @@ -35,7 +40,12 @@ class PsTree(plugins.PluginInterface): seen.add(pid) level = 0 proc = self._processes.get(pid, None) - while proc is not None and proc.vol.offset != 0 and proc.p_ppid != 0 and proc.p_ppid not in seen: + while ( + proc is not None + and proc.vol.offset != 0 + and proc.p_ppid != 0 + and proc.p_ppid not in seen + ): ppid = int(proc.p_ppid) child_list = self._children.get(ppid, set([])) child_list.add(proc.p_pid) @@ -46,9 +56,11 @@ class PsTree(plugins.PluginInterface): def _generator(self): """Generates the tree list of processes""" - list_tasks = pslist.PsList.get_list_tasks(self.config.get('pslist_method', pslist.PsList.pslist_methods[0])) + list_tasks = pslist.PsList.get_list_tasks( + self.config.get("pslist_method", pslist.PsList.pslist_methods[0]) + ) - for proc in list_tasks(self.context, self.config['kernel']): + for proc in list_tasks(self.context, self.config["kernel"]): self._processes[proc.p_pid] = proc # Build the child/level maps @@ -68,4 +80,6 @@ class PsTree(plugins.PluginInterface): yield from yield_processes(pid) def run(self): - return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str)], self._generator()) + return renderers.TreeGrid( + [("PID", int), ("PPID", int), ("COMM", str)], self._generator() + ) diff --git a/volatility3/framework/plugins/mac/socket_filters.py b/volatility3/framework/plugins/mac/socket_filters.py index a6e9d11fd..49e77163e 100644 --- a/volatility3/framework/plugins/mac/socket_filters.py +++ b/volatility3/framework/plugins/mac/socket_filters.py @@ -24,31 +24,54 @@ class Socket_filters(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) + mods = lsmod.Lsmod.list_modules(self.context, self.config["kernel"]) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info( + self.context, kernel.layer_name, kernel, mods + ) members_to_check = [ - "sf_unregistered", "sf_attach", "sf_detach", "sf_notify", "sf_getpeername", "sf_getsockname", "sf_data_in", - "sf_data_out", "sf_connect_in", "sf_connect_out", "sf_bind", "sf_setoption", "sf_getoption", "sf_listen", - "sf_ioctl" + "sf_unregistered", + "sf_attach", + "sf_detach", + "sf_notify", + "sf_getpeername", + "sf_getsockname", + "sf_data_in", + "sf_data_out", + "sf_connect_in", + "sf_connect_out", + "sf_bind", + "sf_setoption", + "sf_getoption", + "sf_listen", + "sf_ioctl", ] - filter_list = kernel.object_from_symbol(symbol_name = "sock_filter_head") + filter_list = kernel.object_from_symbol(symbol_name="sock_filter_head") - for filter_container in mac.MacUtilities.walk_tailq(filter_list, "sf_global_next"): + for filter_container in mac.MacUtilities.walk_tailq( + filter_list, "sf_global_next" + ): current_filter = filter_container.sf_filter - filter_name = utility.pointer_to_string(current_filter.sf_name, count = 128) + filter_name = utility.pointer_to_string(current_filter.sf_name, count=128) try: filter_socket = filter_container.sf_entry_head.sfe_socket.vol.offset @@ -56,16 +79,37 @@ class Socket_filters(plugins.PluginInterface): filter_socket = 0 for member in members_to_check: - check_addr = current_filter.member(attr = member) + check_addr = current_filter.member(attr=member) if check_addr == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, check_addr) + module_name, symbol_name = mac.MacUtilities.lookup_module_address( + self.context, handlers, check_addr + ) - yield (0, (format_hints.Hex(current_filter.vol.offset), filter_name, member, \ - format_hints.Hex(filter_socket), format_hints.Hex(check_addr), module_name, symbol_name)) + yield ( + 0, + ( + format_hints.Hex(current_filter.vol.offset), + filter_name, + member, + format_hints.Hex(filter_socket), + format_hints.Hex(check_addr), + module_name, + symbol_name, + ), + ) def run(self): - return renderers.TreeGrid([("Filter", format_hints.Hex), ("Name", str), ("Member", str), - ("Socket", format_hints.Hex), ("Handler", format_hints.Hex), ("Module", str), - ("Symbol", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Filter", format_hints.Hex), + ("Name", str), + ("Member", str), + ("Socket", format_hints.Hex), + ("Handler", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/timers.py b/volatility3/framework/plugins/mac/timers.py index 7bc5fd5d0..8a267bd55 100644 --- a/volatility3/framework/plugins/mac/timers.py +++ b/volatility3/framework/plugins/mac/timers.py @@ -23,33 +23,46 @@ class Timers(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 3, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 3, 0) + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) + mods = lsmod.Lsmod.list_modules(self.context, self.config["kernel"]) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info( + self.context, kernel.layer_name, kernel, mods + ) - real_ncpus = kernel.object_from_symbol(symbol_name = "real_ncpus") + real_ncpus = kernel.object_from_symbol(symbol_name="real_ncpus") cpu_data_ptrs_ptr = kernel.get_symbol("cpu_data_ptr").address # Returns the a pointer to the absolute address - cpu_data_ptrs_addr = kernel.object(object_type = "pointer", - offset = cpu_data_ptrs_ptr, - subtype = kernel.get_type('long unsigned int')) + cpu_data_ptrs_addr = kernel.object( + object_type="pointer", + offset=cpu_data_ptrs_ptr, + subtype=kernel.get_type("long unsigned int"), + ) - cpu_data_ptrs = kernel.object(object_type = "array", - offset = cpu_data_ptrs_addr, - absolute = True, - subtype = kernel.get_type('cpu_data'), - count = real_ncpus) + cpu_data_ptrs = kernel.object( + object_type="array", + offset=cpu_data_ptrs_addr, + absolute=True, + subtype=kernel.get_type("cpu_data"), + count=real_ncpus, + ) for cpu_data_ptr in cpu_data_ptrs: try: @@ -68,13 +81,33 @@ class Timers(plugins.PluginInterface): else: entry_time = -1 - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, handler, - self.config['kernel']) + module_name, symbol_name = mac.MacUtilities.lookup_module_address( + self.context, handlers, handler, self.config["kernel"] + ) - yield (0, (format_hints.Hex(handler), format_hints.Hex(timer.param0), format_hints.Hex(timer.param1), - timer.deadline, entry_time, module_name, symbol_name)) + yield ( + 0, + ( + format_hints.Hex(handler), + format_hints.Hex(timer.param0), + format_hints.Hex(timer.param1), + timer.deadline, + entry_time, + module_name, + symbol_name, + ), + ) def run(self): - return renderers.TreeGrid([("Function", format_hints.Hex), ("Param 0", format_hints.Hex), - ("Param 1", format_hints.Hex), ("Deadline", int), ("Entry Time", int), - ("Module", str), ("Symbol", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Function", format_hints.Hex), + ("Param 0", format_hints.Hex), + ("Param 1", format_hints.Hex), + ("Deadline", int), + ("Entry Time", int), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/trustedbsd.py b/volatility3/framework/plugins/mac/trustedbsd.py index 9efb1b9d6..a03e2a903 100644 --- a/volatility3/framework/plugins/mac/trustedbsd.py +++ b/volatility3/framework/plugins/mac/trustedbsd.py @@ -25,24 +25,37 @@ class Trustedbsd(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 3, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="macutils", component=mac.MacUtilities, version=(1, 3, 0) + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), ] def _generator(self, mods: Iterator[Any]): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info( + self.context, kernel.layer_name, kernel, mods + ) - policy_list = kernel.object_from_symbol(symbol_name = "mac_policy_list").cast("mac_policy_list") + policy_list = kernel.object_from_symbol(symbol_name="mac_policy_list").cast( + "mac_policy_list" + ) - entries = kernel.object(object_type = "array", - offset = policy_list.entries.dereference().vol.offset, - subtype = kernel.get_type('mac_policy_list_element'), - absolute = True, - count = policy_list.staticmax + 1) + entries = kernel.object( + object_type="array", + offset=policy_list.entries.dereference().vol.offset, + subtype=kernel.get_type("mac_policy_list_element"), + absolute=True, + count=policy_list.staticmax + 1, + ) for i, ent in enumerate(entries): # I don't know how this can happen, but the kernel makes this check all over the place @@ -64,13 +77,31 @@ class Trustedbsd(plugins.PluginInterface): if call_addr is None or call_addr == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, call_addr, - self.config['kernel']) + module_name, symbol_name = mac.MacUtilities.lookup_module_address( + self.context, handlers, call_addr, self.config["kernel"] + ) - yield (0, (check, ent_name, format_hints.Hex(call_addr), module_name, symbol_name)) + yield ( + 0, + ( + check, + ent_name, + format_hints.Hex(call_addr), + module_name, + symbol_name, + ), + ) def run(self): - return renderers.TreeGrid([("Member", str), ("Policy Name", str), ("Handler Address", format_hints.Hex), - ("Handler Module", str), ("Handler Symbol", str)], - self._generator( - lsmod.Lsmod.list_modules(self.context, self.config['kernel']))) + return renderers.TreeGrid( + [ + ("Member", str), + ("Policy Name", str), + ("Handler Address", format_hints.Hex), + ("Handler Module", str), + ("Handler Symbol", str), + ], + self._generator( + lsmod.Lsmod.list_modules(self.context, self.config["kernel"]) + ), + ) diff --git a/volatility3/framework/plugins/mac/vfsevents.py b/volatility3/framework/plugins/mac/vfsevents.py index bc5668495..5aca76467 100644 --- a/volatility3/framework/plugins/mac/vfsevents.py +++ b/volatility3/framework/plugins/mac/vfsevents.py @@ -8,20 +8,34 @@ from volatility3.framework.objects import utility class VFSevents(interfaces.plugins.PluginInterface): - """ Lists processes that are filtering file system events """ + """Lists processes that are filtering file system events""" _required_framework_version = (2, 0, 0) event_types = [ - "CREATE_FILE", "DELETE", "STAT_CHANGED", "RENAME", "CONTENT_MODIFIED", "EXCHANGE", "FINDER_INFO_CHANGED", - "CREATE_DIR", "CHOWN", "XATTR_MODIFIED", "XATTR_REMOVED", "DOCID_CREATED", "DOCID_CHANGED" + "CREATE_FILE", + "DELETE", + "STAT_CHANGED", + "RENAME", + "CONTENT_MODIFIED", + "EXCHANGE", + "FINDER_INFO_CHANGED", + "CREATE_DIR", + "CHOWN", + "XATTR_MODIFIED", + "XATTR_REMOVED", + "DOCID_CREATED", + "DOCID_CHANGED", ] @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), ] def _generator(self): @@ -30,7 +44,7 @@ class VFSevents(interfaces.plugins.PluginInterface): Also lists which event(s) a process is registered for """ - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] watcher_table = kernel.object_from_symbol("watcher_table") @@ -44,11 +58,13 @@ class VFSevents(interfaces.plugins.PluginInterface): events = [] try: - event_array = kernel.object(object_type = "array", - offset = watcher.event_list, - absolute = True, - count = 13, - subtype = kernel.get_type("unsigned char")) + event_array = kernel.object( + object_type="array", + offset=watcher.event_list, + absolute=True, + count=13, + subtype=kernel.get_type("unsigned char"), + ) except exceptions.InvalidAddressException: continue @@ -61,4 +77,6 @@ class VFSevents(interfaces.plugins.PluginInterface): yield (0, (task_name, task_pid, ",".join(events))) def run(self): - return renderers.TreeGrid([("Name", str), ("PID", int), ("Events", str)], self._generator()) + return renderers.TreeGrid( + [("Name", str), ("PID", int), ("Events", str)], self._generator() + ) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index c1d29062d..0776b6cc8 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -25,12 +25,14 @@ class TimeLinerType(enum.IntEnum): CHANGED = 4 -class TimeLinerInterface(metaclass = abc.ABCMeta): +class TimeLinerInterface(metaclass=abc.ABCMeta): """Interface defining methods that timeliner will use to generate a body file.""" @abc.abstractmethod - def generate_timeline(self) -> Generator[Tuple[str, TimeLinerType, datetime.datetime], None, None]: + def generate_timeline( + self, + ) -> Generator[Tuple[str, TimeLinerType, datetime.datetime], None, None]: """Method generates Tuples of (description, timestamp_type, timestamp) These need not be generated in any particular order, sorting @@ -69,72 +71,106 @@ class Timeliner(interfaces.plugins.PluginInterface): else: selected_list = [] - return [plugin_class for plugin_class in plugin_list if filter_func(plugin_class.__name__, selected_list)] + return [ + plugin_class + for plugin_class in plugin_list + if filter_func(plugin_class.__name__, selected_list) + ] @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ requirements.BooleanRequirement( - name = 'record-config', - description = "Whether to record the state of all the plugins once complete", - optional = True, - default = False), - requirements.ListRequirement(name = 'plugin-filter', - description = "Only run plugins featuring this substring", - element_type = str, - optional = True, - default = []), - requirements.BooleanRequirement(name = 'create-bodyfile', - description = "Whether to create a body file whilst producing results", - optional = True, - default = False) + name="record-config", + description="Whether to record the state of all the plugins once complete", + optional=True, + default=False, + ), + requirements.ListRequirement( + name="plugin-filter", + description="Only run plugins featuring this substring", + element_type=str, + optional=True, + default=[], + ), + requirements.BooleanRequirement( + name="create-bodyfile", + description="Whether to create a body file whilst producing results", + optional=True, + default=False, + ), ] def _sort_function(self, item): data = item[1] def sortable(timestamp): - max_date = datetime.datetime(day = 1, month = 12, year = datetime.MAXYEAR) + max_date = datetime.datetime(day=1, month=12, year=datetime.MAXYEAR) if isinstance(timestamp, interfaces.renderers.BaseAbsentValue): return max_date return timestamp return [sortable(timestamp) for timestamp in data[2:]] - def _generator(self, runnable_plugins: List[TimeLinerInterface]) -> Optional[Iterable[Tuple[int, Tuple]]]: + def _generator( + self, runnable_plugins: List[TimeLinerInterface] + ) -> Optional[Iterable[Tuple[int, Tuple]]]: """Takes a timeline, sorts it and output the data from each relevant row from each plugin.""" # Generate the results for each plugin data = [] # Open the bodyfile now, so we can start outputting to it immediately - if self.config.get('create-bodyfile', True): + if self.config.get("create-bodyfile", True): file_data = self.open("volatility.body") - fp = io.TextIOWrapper(file_data, write_through = True) + fp = io.TextIOWrapper(file_data, write_through=True) else: file_data = None fp = None for plugin in runnable_plugins: plugin_name = plugin.__class__.__name__ - self._progress_callback((runnable_plugins.index(plugin) * 100) // len(runnable_plugins), - f"Running plugin {plugin_name}...") + self._progress_callback( + (runnable_plugins.index(plugin) * 100) // len(runnable_plugins), + f"Running plugin {plugin_name}...", + ) try: vollog.log(logging.INFO, f"Running {plugin_name}") for (item, timestamp_type, timestamp) in plugin.generate_timeline(): times = self.timeline.get((plugin_name, item), {}) if times.get(timestamp_type, None) is not None: - vollog.debug("Multiple timestamps for the same plugin/file combination found: {} {}".format( - plugin_name, item)) + vollog.debug( + "Multiple timestamps for the same plugin/file combination found: {} {}".format( + plugin_name, item + ) + ) times[timestamp_type] = timestamp self.timeline[(plugin_name, item)] = times - data.append((0, [ - plugin_name, item, - times.get(TimeLinerType.CREATED, renderers.NotApplicableValue()), - times.get(TimeLinerType.MODIFIED, renderers.NotApplicableValue()), - times.get(TimeLinerType.ACCESSED, renderers.NotApplicableValue()), - times.get(TimeLinerType.CHANGED, renderers.NotApplicableValue()) - ])) + data.append( + ( + 0, + [ + plugin_name, + item, + times.get( + TimeLinerType.CREATED, + renderers.NotApplicableValue(), + ), + times.get( + TimeLinerType.MODIFIED, + renderers.NotApplicableValue(), + ), + times.get( + TimeLinerType.ACCESSED, + renderers.NotApplicableValue(), + ), + times.get( + TimeLinerType.CHANGED, + renderers.NotApplicableValue(), + ), + ], + ) + ) # Write each entry because the body file doesn't need to be sorted if fp: @@ -142,21 +178,35 @@ class Timeliner(interfaces.plugins.PluginInterface): # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime if self._any_time_present(times): - fp.write("|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format( - plugin_name, self._sanitize_body_format(item), - self._text_format(times.get(TimeLinerType.ACCESSED, "")), - self._text_format(times.get(TimeLinerType.MODIFIED, "")), - self._text_format(times.get(TimeLinerType.CHANGED, "")), - self._text_format(times.get(TimeLinerType.CREATED, "")))) + fp.write( + "|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format( + plugin_name, + self._sanitize_body_format(item), + self._text_format( + times.get(TimeLinerType.ACCESSED, "") + ), + self._text_format( + times.get(TimeLinerType.MODIFIED, "") + ), + self._text_format( + times.get(TimeLinerType.CHANGED, "") + ), + self._text_format( + times.get(TimeLinerType.CREATED, "") + ), + ) + ) except Exception: - vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}") + vollog.log( + logging.INFO, f"Exception occurred running plugin: {plugin_name}" + ) vollog.log(logging.DEBUG, traceback.format_exc()) - for data_item in sorted(data, key = self._sort_function): + for data_item in sorted(data, key=self._sort_function): yield data_item # Write out a body file if necessary - if self.config.get('create-bodyfile', True): + if self.config.get("create-bodyfile", True): if fp: fp.close() file_data.close() @@ -166,7 +216,10 @@ class Timeliner(interfaces.plugins.PluginInterface): def _any_time_present(self, times): for time in TimeLinerType: - if not isinstance(times.get(time, renderers.NotApplicableValue), interfaces.renderers.BaseAbsentValue): + if not isinstance( + times.get(time, renderers.NotApplicableValue), + interfaces.renderers.BaseAbsentValue, + ): return True return False @@ -187,7 +240,7 @@ class Timeliner(interfaces.plugins.PluginInterface): plugins_to_run = [] requirement_configs = {} - filter_list = self.config['plugin-filter'] + filter_list = self.config["plugin-filter"] # Identify plugins that we can run which output datetimes for plugin_class in self.usable_plugins: try: @@ -197,42 +250,72 @@ class Timeliner(interfaces.plugins.PluginInterface): if requirement.name in requirement_configs: config_req, config_value = requirement_configs[requirement.name] if requirement == config_req: - self.context.config[interfaces.configuration.path_join( - self.config_path, plugin_class.__name__)] = config_value + self.context.config[ + interfaces.configuration.path_join( + self.config_path, plugin_class.__name__ + ) + ] = config_value - plugin = plugins.construct_plugin(self.context, automagics, plugin_class, self.config_path, - self._progress_callback, self.open) + plugin = plugins.construct_plugin( + self.context, + automagics, + plugin_class, + self.config_path, + self._progress_callback, + self.open, + ) for requirement in plugin.get_requirements(): if requirement.name not in requirement_configs: config_value = plugin.config.get(requirement.name, None) if config_value: - requirement_configs[requirement.name] = (requirement, config_value) + requirement_configs[requirement.name] = ( + requirement, + config_value, + ) if isinstance(plugin, TimeLinerInterface): if not len(filter_list) or any( - [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]): + [ + filter + in plugin.__module__ + "." + plugin.__class__.__name__ + for filter in filter_list + ] + ): plugins_to_run.append(plugin) except exceptions.UnsatisfiedException as excp: # Remove the failed plugin from the list and continue - vollog.debug(f"Unable to satisfy {plugin_class.__name__}: {excp.unsatisfied}") + vollog.debug( + f"Unable to satisfy {plugin_class.__name__}: {excp.unsatisfied}" + ) continue - if self.config.get('record-config', False): + if self.config.get("record-config", False): total_config = {} for plugin in plugins_to_run: old_dict = dict(plugin.build_configuration()) for entry in old_dict: - total_config[interfaces.configuration.path_join(plugin.__class__.__name__, entry)] = old_dict[entry] + total_config[ + interfaces.configuration.path_join( + plugin.__class__.__name__, entry + ) + ] = old_dict[entry] with self.open("config.json") as file_data: - with io.TextIOWrapper(file_data, write_through = True) as fp: - json.dump(total_config, fp, sort_keys = True, indent = 2) + with io.TextIOWrapper(file_data, write_through=True) as fp: + json.dump(total_config, fp, sort_keys=True, indent=2) - return renderers.TreeGrid(columns = [("Plugin", str), ("Description", str), ("Created Date", datetime.datetime), - ("Modified Date", datetime.datetime), ("Accessed Date", datetime.datetime), - ("Changed Date", datetime.datetime)], - generator = self._generator(plugins_to_run)) + return renderers.TreeGrid( + columns=[ + ("Plugin", str), + ("Description", str), + ("Created Date", datetime.datetime), + ("Modified Date", datetime.datetime), + ("Accessed Date", datetime.datetime), + ("Changed Date", datetime.datetime), + ], + generator=self._generator(plugins_to_run), + ) def build_configuration(self): """Builds the configuration to save for the plugin such that it can be diff --git a/volatility3/framework/plugins/windows/bigpools.py b/volatility3/framework/plugins/windows/bigpools.py index 9e120446f..1a51a0b81 100644 --- a/volatility3/framework/plugins/windows/bigpools.py +++ b/volatility3/framework/plugins/windows/bigpools.py @@ -27,25 +27,34 @@ class BigPools(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.StringRequirement(name = 'tags', - description = "Comma separated list of pool tags to filter pools returned", - optional = True, - default = None), - requirements.BooleanRequirement(name = 'show-free', - description = 'Show freed regions (otherwise only show allocations in use)', - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.StringRequirement( + name="tags", + description="Comma separated list of pool tags to filter pools returned", + optional=True, + default=None, + ), + requirements.BooleanRequirement( + name="show-free", + description="Show freed regions (otherwise only show allocations in use)", + default=False, + optional=True, + ), ] @classmethod - def list_big_pools(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - tags: Optional[list] = None, - show_free: bool = False): + def list_big_pools( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + tags: Optional[list] = None, + show_free: bool = False, + ): """Returns the big page pool objects from the kernel PoolBigPageTable array. Args: @@ -57,14 +66,18 @@ class BigPools(interfaces.plugins.PluginInterface): Yields: A big page pool object """ - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) big_page_table_offset = ntkrnlmp.get_symbol("PoolBigPageTable").address - big_page_table = ntkrnlmp.object(object_type = "unsigned long long", offset = big_page_table_offset) + big_page_table = ntkrnlmp.object( + object_type="unsigned long long", offset=big_page_table_offset + ) big_page_table_size_offset = ntkrnlmp.get_symbol("PoolBigPageTableSize").address - big_page_table_size = ntkrnlmp.object(object_type = "unsigned long", offset = big_page_table_size_offset) + big_page_table_size = ntkrnlmp.object( + object_type="unsigned long", offset=big_page_table_size_offset + ) try: big_page_table_type = ntkrnlmp.get_type("_POOL_TRACKER_BIG_PAGES") @@ -85,38 +98,49 @@ class BigPools(interfaces.plugins.PluginInterface): big_pools_json_filename += "-x86" new_table_name = intermed.IntermediateSymbolTable.create( - context = context, - config_path = configuration.path_join(context.symbol_space[symbol_table].config_path, "bigpools"), - sub_path = os.path.join("windows", "bigpools"), - filename = big_pools_json_filename, - table_mapping = {'nt_symbols': symbol_table}, - class_types = {'_POOL_TRACKER_BIG_PAGES': extensions.pool.POOL_TRACKER_BIG_PAGES}) - module = context.module(new_table_name, layer_name, offset = 0) + context=context, + config_path=configuration.path_join( + context.symbol_space[symbol_table].config_path, "bigpools" + ), + sub_path=os.path.join("windows", "bigpools"), + filename=big_pools_json_filename, + table_mapping={"nt_symbols": symbol_table}, + class_types={ + "_POOL_TRACKER_BIG_PAGES": extensions.pool.POOL_TRACKER_BIG_PAGES + }, + ) + module = context.module(new_table_name, layer_name, offset=0) big_page_table_type = module.get_type("_POOL_TRACKER_BIG_PAGES") - big_pools = ntkrnlmp.object(object_type = "array", - offset = big_page_table, - subtype = big_page_table_type, - count = big_page_table_size, - absolute = True) + big_pools = ntkrnlmp.object( + object_type="array", + offset=big_page_table, + subtype=big_page_table_type, + count=big_page_table_size, + absolute=True, + ) for big_pool in big_pools: if big_pool.is_valid(): - if (tags is None or big_pool.get_key() in tags) and (show_free or not big_pool.is_free()): + if (tags is None or big_pool.get_key() in tags) and ( + show_free or not big_pool.is_free() + ): yield big_pool def _generator(self) -> Iterator[Tuple[int, Tuple[int, str]]]: # , str, int]]]: if self.config.get("tags"): - tags = [tag for tag in self.config["tags"].split(',')] + tags = [tag for tag in self.config["tags"].split(",")] else: tags = None - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for big_pool in self.list_big_pools(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - tags = tags, - show_free = self.config.get("show-free")): + for big_pool in self.list_big_pools( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + tags=tags, + show_free=self.config.get("show-free"), + ): num_bytes = big_pool.get_number_of_bytes() if not isinstance(num_bytes, interfaces.renderers.BaseAbsentValue): @@ -127,13 +151,25 @@ class BigPools(interfaces.plugins.PluginInterface): else: status = "Allocated" - yield (0, (format_hints.Hex(big_pool.Va), big_pool.get_key(), big_pool.get_pool_type(), num_bytes, status)) + yield ( + 0, + ( + format_hints.Hex(big_pool.Va), + big_pool.get_key(), + big_pool.get_pool_type(), + num_bytes, + status, + ), + ) def run(self): - return renderers.TreeGrid([ - ('Allocation', format_hints.Hex), - ('Tag', str), - ('PoolType', str), - ('NumberOfBytes', format_hints.Hex), - ('Status', str), - ], self._generator()) + return renderers.TreeGrid( + [ + ("Allocation", format_hints.Hex), + ("Tag", str), + ("PoolType", str), + ("NumberOfBytes", format_hints.Hex), + ("Status", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index 59ea656f3..7d3093ed7 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -27,16 +27,29 @@ class Cachedump(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsadump', plugin = lsadump.Lsadump, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'hashdump', plugin = hashdump.Hashdump, version = (1, 1, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="lsadump", plugin=lsadump.Lsadump, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="hashdump", plugin=hashdump.Hashdump, version=(1, 1, 0) + ), ] @staticmethod - def get_nlkm(sechive: registry.RegistryHive, lsakey: bytes, is_vista_or_later: bool): - return lsadump.Lsadump.get_secret_by_name(sechive, 'NL$KM', lsakey, is_vista_or_later) + def get_nlkm( + sechive: registry.RegistryHive, lsakey: bytes, is_vista_or_later: bool + ): + return lsadump.Lsadump.get_secret_by_name( + sechive, "NL$KM", lsakey, is_vista_or_later + ) @staticmethod def decrypt_hash(edata: bytes, nlkm: bytes, ch, xp: bool): @@ -44,13 +57,13 @@ class Cachedump(interfaces.plugins.PluginInterface): hmac_md5 = HMAC.new(nlkm, ch) rc4key = hmac_md5.digest() rc4 = ARC4.new(rc4key) - data = rc4.encrypt(edata) # lgtm [py/weak-cryptographic-algorithm] + data = rc4.encrypt(edata) # lgtm [py/weak-cryptographic-algorithm] else: # Based on code from http://lab.mediaservice.net/code/cachedump.rb aes = AES.new(nlkm[16:32], AES.MODE_CBC, ch) data = b"" for i in range(0, len(edata), 16): - buf = edata[i:i + 16] + buf = edata[i : i + 16] if len(buf) < 16: buf += (16 - len(buf)) * b"\00" data += aes.decrypt(buf) @@ -60,15 +73,16 @@ class Cachedump(interfaces.plugins.PluginInterface): def parse_cache_entry(cache_data: bytes) -> Tuple[int, int, int, bytes, bytes]: (uname_len, domain_len) = unpack(" Tuple[str, str, str, bytes]: + def parse_decrypted_cache( + dec_data: bytes, uname_len: int, domain_len: int, domain_name_len: int + ) -> Tuple[str, str, str, bytes]: """Get the data from the cache and separate it into the username, domain name, and hash data""" uname_offset = 72 pad = 2 * ((uname_len / 2) % 2) @@ -76,43 +90,50 @@ class Cachedump(interfaces.plugins.PluginInterface): pad = 2 * ((domain_len / 2) % 2) domain_name_offset = int(domain_offset + domain_len + pad) hashh = dec_data[:0x10] - username = dec_data[uname_offset:uname_offset + uname_len].decode('utf-16-le', 'replace') - domain = dec_data[domain_offset:domain_offset + domain_len].decode('utf-16-le', 'replace') - domain_name = dec_data[domain_name_offset:domain_name_offset + domain_name_len].decode('utf-16-le', 'replace') + username = dec_data[uname_offset : uname_offset + uname_len].decode( + "utf-16-le", "replace" + ) + domain = dec_data[domain_offset : domain_offset + domain_len].decode( + "utf-16-le", "replace" + ) + domain_name = dec_data[ + domain_name_offset : domain_name_offset + domain_name_len + ].decode("utf-16-le", "replace") return (username, domain, domain_name, hashh) def _generator(self, syshive, sechive): if not syshive or not sechive: if syshive is None: - vollog.warning('Unable to locate SYSTEM hive') + vollog.warning("Unable to locate SYSTEM hive") if sechive is None: - vollog.warning('Unable to locate SECURITY hive') + vollog.warning("Unable to locate SECURITY hive") return bootkey = hashdump.Hashdump.get_bootkey(syshive) if not bootkey: - vollog.warning('Unable to find bootkey') + vollog.warning("Unable to find bootkey") return - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - vista_or_later = versions.is_vista_or_later(context = self.context, - symbol_table = kernel.symbol_table_name) + vista_or_later = versions.is_vista_or_later( + context=self.context, symbol_table=kernel.symbol_table_name + ) lsakey = lsadump.Lsadump.get_lsa_key(sechive, bootkey, vista_or_later) if not lsakey: - vollog.warning('Unable to find lsa key') + vollog.warning("Unable to find lsa key") return nlkm = self.get_nlkm(sechive, lsakey, vista_or_later) if not nlkm: - vollog.warning('Unable to find nlkma key') + vollog.warning("Unable to find nlkma key") return cache = hashdump.Hashdump.get_hive_key(sechive, "Cache") if not cache: - vollog.warning('Unable to find cache key') + vollog.warning("Unable to find cache key") return for cache_item in cache.get_values(): @@ -122,32 +143,43 @@ class Cachedump(interfaces.plugins.PluginInterface): data = sechive.read(cache_item.Data + 4, cache_item.DataLength) if data is None: continue - (uname_len, domain_len, domain_name_len, enc_data, ch) = self.parse_cache_entry(data) + ( + uname_len, + domain_len, + domain_name_len, + enc_data, + ch, + ) = self.parse_cache_entry(data) # Skip if nothing in this cache entry if uname_len == 0 or len(ch) == 0: continue dec_data = self.decrypt_hash(enc_data, nlkm, ch, not vista_or_later) - (username, domain, domain_name, hashh) = self.parse_decrypted_cache(dec_data, uname_len, domain_len, - domain_name_len) + (username, domain, domain_name, hashh) = self.parse_decrypted_cache( + dec_data, uname_len, domain_len, domain_name_len + ) yield (0, (username, domain, domain_name, hashh)) def run(self): - offset = self.config.get('offset', None) + offset = self.config.get("offset", None) syshive = sechive = None - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for hive in hivelist.HiveList.list_hives(self.context, - self.config_path, - kernel.layer_name, - kernel.symbol_table_name, - hive_offsets = None if offset is None else [offset]): + for hive in hivelist.HiveList.list_hives( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + hive_offsets=None if offset is None else [offset], + ): - if hive.get_name().split('\\')[-1].upper() == 'SYSTEM': + if hive.get_name().split("\\")[-1].upper() == "SYSTEM": syshive = hive - if hive.get_name().split('\\')[-1].upper() == 'SECURITY': + if hive.get_name().split("\\")[-1].upper() == "SECURITY": sechive = hive - return renderers.TreeGrid([("Username", str), ("Domain", str), ("Domain name", str), ('Hash', bytes)], - self._generator(syshive, sechive)) + return renderers.TreeGrid( + [("Username", str), ("Domain", str), ("Domain name", str), ("Hash", bytes)], + self._generator(syshive, sechive), + ) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 2195671df..3bde95cf3 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -24,13 +24,22 @@ class Callbacks(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'ssdt', plugin = ssdt.SSDT, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), ] @staticmethod - def create_callback_table(context: interfaces.context.ContextInterface, symbol_table: str, config_path: str) -> str: + def create_callback_table( + context: interfaces.context.ContextInterface, + symbol_table: str, + config_path: str, + ) -> str: """Creates a symbol table for a set of callbacks. Args: @@ -50,16 +59,23 @@ class Callbacks(interfaces.plugins.PluginInterface): else: symbol_filename = "callbacks-x86" - return intermed.IntermediateSymbolTable.create(context, - config_path, - "windows", - symbol_filename, - native_types = native_types, - table_mapping = table_mapping) + return intermed.IntermediateSymbolTable.create( + context, + config_path, + "windows", + symbol_filename, + native_types=native_types, + table_mapping=table_mapping, + ) @classmethod - def list_notify_routines(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - callback_table_name: str) -> Iterable[Tuple[str, int, Optional[str]]]: + def list_notify_routines( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + callback_table_name: str, + ) -> Iterable[Tuple[str, int, Optional[str]]]: """Lists all kernel notification routines. Args: @@ -72,14 +88,19 @@ class Callbacks(interfaces.plugins.PluginInterface): A name, location and optional detail string """ - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) - is_vista_or_later = versions.is_vista_or_later(context = context, symbol_table = symbol_table) + is_vista_or_later = versions.is_vista_or_later( + context=context, symbol_table=symbol_table + ) full_type_name = callback_table_name + constants.BANG + "_GENERIC_CALLBACK" - symbol_names = [("PspLoadImageNotifyRoutine", False), ("PspCreateThreadNotifyRoutine", True), - ("PspCreateProcessNotifyRoutine", True)] + symbol_names = [ + ("PspLoadImageNotifyRoutine", False), + ("PspCreateThreadNotifyRoutine", True), + ("PspCreateProcessNotifyRoutine", True), + ] for symbol_name, extended_list in symbol_names: @@ -94,10 +115,12 @@ class Callbacks(interfaces.plugins.PluginInterface): else: count = 8 - fast_refs = ntkrnlmp.object(object_type = "array", - offset = symbol_offset, - subtype = ntkrnlmp.get_type("_EX_FAST_REF"), - count = count) + fast_refs = ntkrnlmp.object( + object_type="array", + offset=symbol_offset, + subtype=ntkrnlmp.get_type("_EX_FAST_REF"), + count=count, + ) for fast_ref in fast_refs: try: @@ -109,29 +132,39 @@ class Callbacks(interfaces.plugins.PluginInterface): yield symbol_name, callback.Callback, None @classmethod - def _list_registry_callbacks_legacy(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + def _list_registry_callbacks_legacy( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + callback_table_name: str, + ) -> Iterable[Tuple[str, int, None]]: """ Lists all registry callbacks from the old format via the CmpCallBackVector. """ - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) - full_type_name = callback_table_name + constants.BANG + "_EX_CALLBACK_ROUTINE_BLOCK" + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) + full_type_name = ( + callback_table_name + constants.BANG + "_EX_CALLBACK_ROUTINE_BLOCK" + ) symbol_offset = ntkrnlmp.get_symbol("CmpCallBackVector").address symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address - - callback_count = ntkrnlmp.object(object_type = "unsigned int", offset = symbol_count_offset) + callback_count = ntkrnlmp.object( + object_type="unsigned int", offset=symbol_count_offset + ) if callback_count == 0: return - fast_refs = ntkrnlmp.object(object_type = "array", - offset = symbol_offset, - subtype = ntkrnlmp.get_type("_EX_FAST_REF"), - count = callback_count) + fast_refs = ntkrnlmp.object( + object_type="array", + offset=symbol_offset, + subtype=ntkrnlmp.get_type("_EX_FAST_REF"), + count=callback_count, + ) for fast_ref in fast_refs: try: @@ -143,31 +176,43 @@ class Callbacks(interfaces.plugins.PluginInterface): yield "CmRegisterCallback", callback.Function, None @classmethod - def _list_registry_callbacks_new(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + def _list_registry_callbacks_new( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + callback_table_name: str, + ) -> Iterable[Tuple[str, int, None]]: """ Lists all registry callbacks via the CallbackListHead. """ - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) full_type_name = callback_table_name + constants.BANG + "_CM_CALLBACK_ENTRY" symbol_offset = ntkrnlmp.get_symbol("CallbackListHead").address symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address - callback_count = ntkrnlmp.object(object_type = "unsigned int", offset = symbol_count_offset) + callback_count = ntkrnlmp.object( + object_type="unsigned int", offset=symbol_count_offset + ) if callback_count == 0: return - callback_list = ntkrnlmp.object(object_type = "_LIST_ENTRY", offset = symbol_offset) + callback_list = ntkrnlmp.object(object_type="_LIST_ENTRY", offset=symbol_offset) for callback in callback_list.to_list(full_type_name, "Link"): yield "CmRegisterCallbackEx", callback.Function, f"Altitude: {callback.Altitude.String}" @classmethod - def list_registry_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + def list_registry_callbacks( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + callback_table_name: str, + ) -> Iterable[Tuple[str, int, None]]: """Lists all registry callbacks. Args: @@ -180,15 +225,27 @@ class Callbacks(interfaces.plugins.PluginInterface): A name, location and optional detail string """ - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) - if ntkrnlmp.has_symbol("CmpCallBackVector") and ntkrnlmp.has_symbol("CmpCallBackCount"): - yield from cls._list_registry_callbacks_legacy(context, layer_name, symbol_table, callback_table_name) - elif ntkrnlmp.has_symbol("CallbackListHead") and ntkrnlmp.has_symbol("CmpCallBackCount"): - yield from cls._list_registry_callbacks_new(context, layer_name, symbol_table, callback_table_name) + if ntkrnlmp.has_symbol("CmpCallBackVector") and ntkrnlmp.has_symbol( + "CmpCallBackCount" + ): + yield from cls._list_registry_callbacks_legacy( + context, layer_name, symbol_table, callback_table_name + ) + elif ntkrnlmp.has_symbol("CallbackListHead") and ntkrnlmp.has_symbol( + "CmpCallBackCount" + ): + yield from cls._list_registry_callbacks_new( + context, layer_name, symbol_table, callback_table_name + ) else: - symbols_to_check = ["CmpCallBackVector", "CmpCallBackCount", "CallbackListHead"] + symbols_to_check = [ + "CmpCallBackVector", + "CmpCallBackCount", + "CallbackListHead", + ] vollog.debug("Failed to get registry callbacks!") for symbol_name in symbols_to_check: symbol_status = "does not exist" @@ -199,8 +256,13 @@ class Callbacks(interfaces.plugins.PluginInterface): return @classmethod - def list_bugcheck_reason_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, - symbol_table: str, callback_table_name: str) -> Iterable[Tuple[str, int, str]]: + def list_bugcheck_reason_callbacks( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + callback_table_name: str, + ) -> Iterable[Tuple[str, int, str]]: """Lists all kernel bugcheck reason callbacks. Args: @@ -213,19 +275,23 @@ class Callbacks(interfaces.plugins.PluginInterface): A name, location and optional detail string """ - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) try: - list_offset = ntkrnlmp.get_symbol("KeBugCheckReasonCallbackListHead").address + list_offset = ntkrnlmp.get_symbol( + "KeBugCheckReasonCallbackListHead" + ).address except exceptions.SymbolError: vollog.debug("Cannot find KeBugCheckReasonCallbackListHead") return - full_type_name = callback_table_name + constants.BANG + "_KBUGCHECK_REASON_CALLBACK_RECORD" - callback_record = context.object(object_type = full_type_name, - offset = kvo + list_offset, - layer_name = layer_name) + full_type_name = ( + callback_table_name + constants.BANG + "_KBUGCHECK_REASON_CALLBACK_RECORD" + ) + callback_record = context.object( + object_type=full_type_name, offset=kvo + list_offset, layer_name=layer_name + ) for callback in callback_record.Entry: if not context.layers[layer_name].is_valid(callback.CallbackRoutine, 64): @@ -233,8 +299,14 @@ class Callbacks(interfaces.plugins.PluginInterface): try: component: Union[ - interfaces.renderers.BaseAbsentValue, interfaces.objects.ObjectInterface] = ntkrnlmp.object( - "string", absolute = True, offset = callback.Component, max_length = 64, errors = "replace" + interfaces.renderers.BaseAbsentValue, + interfaces.objects.ObjectInterface, + ] = ntkrnlmp.object( + "string", + absolute=True, + offset=callback.Component, + max_length=64, + errors="replace", ) except exceptions.InvalidAddressException: component = renderers.UnreadableValue() @@ -242,8 +314,13 @@ class Callbacks(interfaces.plugins.PluginInterface): yield "KeBugCheckReasonCallbackListHead", callback.CallbackRoutine, component @classmethod - def list_bugcheck_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - callback_table_name: str) -> Iterable[Tuple[str, int, str]]: + def list_bugcheck_callbacks( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + callback_table_name: str, + ) -> Iterable[Tuple[str, int, str]]: """Lists all kernel bugcheck callbacks. Args: @@ -256,8 +333,8 @@ class Callbacks(interfaces.plugins.PluginInterface): A name, location and optional detail string """ - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) try: list_offset = ntkrnlmp.get_symbol("KeBugCheckCallbackListHead").address @@ -265,8 +342,12 @@ class Callbacks(interfaces.plugins.PluginInterface): vollog.debug("Cannot find KeBugCheckCallbackListHead") return - full_type_name = callback_table_name + constants.BANG + "_KBUGCHECK_CALLBACK_RECORD" - callback_record = context.object(full_type_name, offset = kvo + list_offset, layer_name = layer_name) + full_type_name = ( + callback_table_name + constants.BANG + "_KBUGCHECK_CALLBACK_RECORD" + ) + callback_record = context.object( + full_type_name, offset=kvo + list_offset, layer_name=layer_name + ) for callback in callback_record.Entry: @@ -274,11 +355,13 @@ class Callbacks(interfaces.plugins.PluginInterface): continue try: - component = context.object(symbol_table + constants.BANG + "string", - layer_name = layer_name, - offset = callback.Component, - max_length = 64, - errors = "replace") + component = context.object( + symbol_table + constants.BANG + "string", + layer_name=layer_name, + offset=callback.Component, + max_length=64, + errors="replace", + ) except exceptions.InvalidAddressException: component = renderers.UnreadableValue() @@ -286,28 +369,39 @@ class Callbacks(interfaces.plugins.PluginInterface): def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - callback_table_name = self.create_callback_table(self.context, kernel.symbol_table_name, - self.config_path) + callback_table_name = self.create_callback_table( + self.context, kernel.symbol_table_name, self.config_path + ) - collection = ssdt.SSDT.build_module_collection(self.context, kernel.layer_name, kernel.symbol_table_name) + collection = ssdt.SSDT.build_module_collection( + self.context, kernel.layer_name, kernel.symbol_table_name + ) - callback_methods = (self.list_notify_routines, self.list_bugcheck_callbacks, - self.list_bugcheck_reason_callbacks, self.list_registry_callbacks) + callback_methods = ( + self.list_notify_routines, + self.list_bugcheck_callbacks, + self.list_bugcheck_reason_callbacks, + self.list_registry_callbacks, + ) for callback_method in callback_methods: - for callback_type, callback_address, callback_detail in callback_method(self.context, - kernel.layer_name, - kernel.symbol_table_name, - callback_table_name): + for callback_type, callback_address, callback_detail in callback_method( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + callback_table_name, + ): if callback_detail is None: detail = renderers.NotApplicableValue() else: detail = callback_detail - module_symbols = list(collection.get_module_symbols_by_absolute_location(callback_address)) + module_symbols = list( + collection.get_module_symbols_by_absolute_location(callback_address) + ) if module_symbols: for module_name, symbol_generator in module_symbols: @@ -316,19 +410,51 @@ class Callbacks(interfaces.plugins.PluginInterface): # we might have multiple symbols pointing to the same location for symbol in symbol_generator: symbols_found = True - yield (0, (callback_type, format_hints.Hex(callback_address), module_name, - symbol.split(constants.BANG)[1], detail)) + yield ( + 0, + ( + callback_type, + format_hints.Hex(callback_address), + module_name, + symbol.split(constants.BANG)[1], + detail, + ), + ) # no symbols, but we at least can report the module name if not symbols_found: - yield (0, (callback_type, format_hints.Hex(callback_address), module_name, - renderers.NotAvailableValue(), detail)) + yield ( + 0, + ( + callback_type, + format_hints.Hex(callback_address), + module_name, + renderers.NotAvailableValue(), + detail, + ), + ) else: # no module was found at the absolute location - yield (0, (callback_type, format_hints.Hex(callback_address), renderers.NotAvailableValue(), - renderers.NotAvailableValue(), detail)) + yield ( + 0, + ( + callback_type, + format_hints.Hex(callback_address), + renderers.NotAvailableValue(), + renderers.NotAvailableValue(), + detail, + ), + ) def run(self): - return renderers.TreeGrid([("Type", str), ("Callback", format_hints.Hex), ("Module", str), ("Symbol", str), - ("Detail", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Type", str), + ("Callback", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ("Detail", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/cmdline.py b/volatility3/framework/plugins/windows/cmdline.py index af6035abb..8cfb5576c 100644 --- a/volatility3/framework/plugins/windows/cmdline.py +++ b/volatility3/framework/plugins/windows/cmdline.py @@ -22,17 +22,26 @@ class CmdLine(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), ] @classmethod - def get_cmdline(cls, context: interfaces.context.ContextInterface, kernel_table_name: str, proc): + def get_cmdline( + cls, context: interfaces.context.ContextInterface, kernel_table_name: str, proc + ): """Extracts the cmdline from PEB Args: @@ -46,15 +55,17 @@ class CmdLine(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() - peb = context.object(kernel_table_name + constants.BANG + "_PEB", - layer_name = proc_layer_name, - offset = proc.Peb) + peb = context.object( + kernel_table_name + constants.BANG + "_PEB", + layer_name=proc_layer_name, + offset=proc.Peb, + ) result_text = peb.ProcessParameters.CommandLine.get_string() return result_text def _generator(self, procs): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] for proc in procs: process_name = utility.array_to_string(proc.ImageFileName) @@ -62,7 +73,9 @@ class CmdLine(interfaces.plugins.PluginInterface): try: proc_id = proc.UniqueProcessId - result_text = self.get_cmdline(self.context, kernel.symbol_table_name, proc) + result_text = self.get_cmdline( + self.context, kernel.symbol_table_name, proc + ) except exceptions.SwappedInvalidAddressException as exp: result_text = f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)" @@ -72,17 +85,23 @@ class CmdLine(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException as exp: result_text = "Process {}: Required memory at {:#x} is not valid (incomplete layer {}?)".format( - proc_id, exp.invalid_address, exp.layer_name) + proc_id, exp.invalid_address, exp.layer_name + ) yield (0, (proc.UniqueProcessId, process_name, result_text)) def run(self): - kernel = self.context.modules[self.config['kernel']] - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config["kernel"]] + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - return renderers.TreeGrid([("PID", int), ("Process", str), ("Args", str)], - self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [("PID", int), ("Process", str), ("Args", str)], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/crashinfo.py b/volatility3/framework/plugins/windows/crashinfo.py index d66d86cd1..a9f32f63d 100644 --- a/volatility3/framework/plugins/windows/crashinfo.py +++ b/volatility3/framework/plugins/windows/crashinfo.py @@ -19,14 +19,16 @@ class Crashinfo(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), ] def _generator(self, layer: crash.WindowsCrashDump32Layer): header = layer.get_header() - uptime = datetime.timedelta(microseconds = int(header.SystemUpTime) / 10) + uptime = datetime.timedelta(microseconds=int(header.SystemUpTime) / 10) if header.DumpType == 0x1: dump_type = "Full Dump (0x1)" @@ -42,27 +44,32 @@ class Crashinfo(interfaces.plugins.PluginInterface): bitmap_size = format_hints.Hex(summary_header.BitmapSize) bitmap_pages = format_hints.Hex(summary_header.Pages) else: - bitmap_header_size = bitmap_size = bitmap_pages = renderers.NotApplicableValue() + bitmap_header_size = ( + bitmap_size + ) = bitmap_pages = renderers.NotApplicableValue() - yield (0, ( - utility.array_to_string(header.Signature), - header.MajorVersion, - header.MinorVersion, - format_hints.Hex(header.DirectoryTableBase), - format_hints.Hex(header.PfnDataBase), - format_hints.Hex(header.PsLoadedModuleList), - format_hints.Hex(header.PsActiveProcessHead), - header.MachineImageType, - header.NumberProcessors, - format_hints.Hex(header.KdDebuggerDataBlock), - dump_type, - str(uptime), - utility.array_to_string(header.Comment), - conversion.wintime_to_datetime(header.SystemTime), - bitmap_header_size, - bitmap_size, - bitmap_pages, - )) + yield ( + 0, + ( + utility.array_to_string(header.Signature), + header.MajorVersion, + header.MinorVersion, + format_hints.Hex(header.DirectoryTableBase), + format_hints.Hex(header.PfnDataBase), + format_hints.Hex(header.PsLoadedModuleList), + format_hints.Hex(header.PsActiveProcessHead), + header.MachineImageType, + header.NumberProcessors, + format_hints.Hex(header.KdDebuggerDataBlock), + dump_type, + str(uptime), + utility.array_to_string(header.Comment), + conversion.wintime_to_datetime(header.SystemTime), + bitmap_header_size, + bitmap_size, + bitmap_pages, + ), + ) def run(self): crash_layer = None @@ -76,22 +83,25 @@ class Crashinfo(interfaces.plugins.PluginInterface): vollog.error("This plugin requires a Windows crash dump") raise - return renderers.TreeGrid([ - ("Signature", str), - ("MajorVersion", int), - ("MinorVersion", int), - ("DirectoryTableBase", format_hints.Hex), - ("PfnDataBase", format_hints.Hex), - ("PsLoadedModuleList", format_hints.Hex), - ("PsActiveProcessHead", format_hints.Hex), - ("MachineImageType", int), - ("NumberProcessors", int), - ("KdDebuggerDataBlock", format_hints.Hex), - ("DumpType", str), - ("SystemUpTime", str), - ("Comment", str), - ("SystemTime", datetime.datetime), - ("BitmapHeaderSize", format_hints.Hex), - ("BitmapSize", format_hints.Hex), - ("BitmapPages", format_hints.Hex), - ], self._generator(crash_layer)) + return renderers.TreeGrid( + [ + ("Signature", str), + ("MajorVersion", int), + ("MinorVersion", int), + ("DirectoryTableBase", format_hints.Hex), + ("PfnDataBase", format_hints.Hex), + ("PsLoadedModuleList", format_hints.Hex), + ("PsActiveProcessHead", format_hints.Hex), + ("MachineImageType", int), + ("NumberProcessors", int), + ("KdDebuggerDataBlock", format_hints.Hex), + ("DumpType", str), + ("SystemUpTime", str), + ("Comment", str), + ("SystemTime", datetime.datetime), + ("BitmapHeaderSize", format_hints.Hex), + ("BitmapSize", format_hints.Hex), + ("BitmapPages", format_hints.Hex), + ], + self._generator(crash_layer), + ) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index 1b6b55cb7..2541629d5 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -12,68 +12,69 @@ from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import driverscan DEVICE_CODES = { - 0x00000027 : "FILE_DEVICE_8042_PORT", - 0x00000032 : "FILE_DEVICE_ACPI", - 0x00000029 : "FILE_DEVICE_BATTERY", - 0x00000001 : "FILE_DEVICE_BEEP", - 0x0000002a : "FILE_DEVICE_BUS_EXTENDER", - 0x00000002 : "FILE_DEVICE_CD_ROM", - 0x00000003 : "FILE_DEVICE_CD_ROM_FILE_SYSTEM", - 0x00000030 : "FILE_DEVICE_CHANGER", - 0x00000004 : "FILE_DEVICE_CONTROLLER", - 0x00000005 : "FILE_DEVICE_DATALINK", - 0x00000006 : "FILE_DEVICE_DFS", - 0x00000035 : "FILE_DEVICE_DFS_FILE_SYSTEM", - 0x00000036 : "FILE_DEVICE_DFS_VOLUME", - 0x00000007 : "FILE_DEVICE_DISK", - 0x00000008 : "FILE_DEVICE_DISK_FILE_SYSTEM", - 0x00000033 : "FILE_DEVICE_DVD", - 0x00000009 : "FILE_DEVICE_FILE_SYSTEM", - 0x0000003a : "FILE_DEVICE_FIPS", - 0x00000034 : "FILE_DEVICE_FULLSCREEN_VIDEO", - 0x0000000a : "FILE_DEVICE_INPORT_PORT", - 0x0000000b : "FILE_DEVICE_KEYBOARD", - 0x0000002f : "FILE_DEVICE_KS", - 0x00000039 : "FILE_DEVICE_KSEC", - 0x0000000c : "FILE_DEVICE_MAILSLOT", - 0x0000002d : "FILE_DEVICE_MASS_STORAGE", - 0x0000000d : "FILE_DEVICE_MIDI_IN", - 0x0000000e : "FILE_DEVICE_MIDI_OUT", - 0x0000002b : "FILE_DEVICE_MODEM", - 0x0000000f : "FILE_DEVICE_MOUSE", - 0x00000010 : "FILE_DEVICE_MULTI_UNC_PROVIDER", - 0x00000011 : "FILE_DEVICE_NAMED_PIPE", - 0x00000012 : "FILE_DEVICE_NETWORK", - 0x00000013 : "FILE_DEVICE_NETWORK_BROWSER", - 0x00000014 : "FILE_DEVICE_NETWORK_FILE_SYSTEM", - 0x00000028 : "FILE_DEVICE_NETWORK_REDIRECTOR", - 0x00000015 : "FILE_DEVICE_NULL", - 0x00000016 : "FILE_DEVICE_PARALLEL_PORT", - 0x00000017 : "FILE_DEVICE_PHYSICAL_NETCARD", - 0x00000018 : "FILE_DEVICE_PRINTER", - 0x00000019 : "FILE_DEVICE_SCANNER", - 0x0000001c : "FILE_DEVICE_SCREEN", - 0x00000037 : "FILE_DEVICE_SERENUM", - 0x0000001a : "FILE_DEVICE_SERIAL_MOUSE_PORT", - 0x0000001b : "FILE_DEVICE_SERIAL_PORT", - 0x00000031 : "FILE_DEVICE_SMARTCARD", - 0x0000002e : "FILE_DEVICE_SMB", - 0x0000001d : "FILE_DEVICE_SOUND", - 0x0000001e : "FILE_DEVICE_STREAMS", - 0x0000001f : "FILE_DEVICE_TAPE", - 0x00000020 : "FILE_DEVICE_TAPE_FILE_SYSTEM", - 0x00000038 : "FILE_DEVICE_TERMSRV", - 0x00000021 : "FILE_DEVICE_TRANSPORT", - 0x00000022 : "FILE_DEVICE_UNKNOWN", - 0x0000002c : "FILE_DEVICE_VDM", - 0x00000023 : "FILE_DEVICE_VIDEO", - 0x00000024 : "FILE_DEVICE_VIRTUAL_DISK", - 0x00000025 : "FILE_DEVICE_WAVE_IN", - 0x00000026 : "FILE_DEVICE_WAVE_OUT", + 0x00000027: "FILE_DEVICE_8042_PORT", + 0x00000032: "FILE_DEVICE_ACPI", + 0x00000029: "FILE_DEVICE_BATTERY", + 0x00000001: "FILE_DEVICE_BEEP", + 0x0000002A: "FILE_DEVICE_BUS_EXTENDER", + 0x00000002: "FILE_DEVICE_CD_ROM", + 0x00000003: "FILE_DEVICE_CD_ROM_FILE_SYSTEM", + 0x00000030: "FILE_DEVICE_CHANGER", + 0x00000004: "FILE_DEVICE_CONTROLLER", + 0x00000005: "FILE_DEVICE_DATALINK", + 0x00000006: "FILE_DEVICE_DFS", + 0x00000035: "FILE_DEVICE_DFS_FILE_SYSTEM", + 0x00000036: "FILE_DEVICE_DFS_VOLUME", + 0x00000007: "FILE_DEVICE_DISK", + 0x00000008: "FILE_DEVICE_DISK_FILE_SYSTEM", + 0x00000033: "FILE_DEVICE_DVD", + 0x00000009: "FILE_DEVICE_FILE_SYSTEM", + 0x0000003A: "FILE_DEVICE_FIPS", + 0x00000034: "FILE_DEVICE_FULLSCREEN_VIDEO", + 0x0000000A: "FILE_DEVICE_INPORT_PORT", + 0x0000000B: "FILE_DEVICE_KEYBOARD", + 0x0000002F: "FILE_DEVICE_KS", + 0x00000039: "FILE_DEVICE_KSEC", + 0x0000000C: "FILE_DEVICE_MAILSLOT", + 0x0000002D: "FILE_DEVICE_MASS_STORAGE", + 0x0000000D: "FILE_DEVICE_MIDI_IN", + 0x0000000E: "FILE_DEVICE_MIDI_OUT", + 0x0000002B: "FILE_DEVICE_MODEM", + 0x0000000F: "FILE_DEVICE_MOUSE", + 0x00000010: "FILE_DEVICE_MULTI_UNC_PROVIDER", + 0x00000011: "FILE_DEVICE_NAMED_PIPE", + 0x00000012: "FILE_DEVICE_NETWORK", + 0x00000013: "FILE_DEVICE_NETWORK_BROWSER", + 0x00000014: "FILE_DEVICE_NETWORK_FILE_SYSTEM", + 0x00000028: "FILE_DEVICE_NETWORK_REDIRECTOR", + 0x00000015: "FILE_DEVICE_NULL", + 0x00000016: "FILE_DEVICE_PARALLEL_PORT", + 0x00000017: "FILE_DEVICE_PHYSICAL_NETCARD", + 0x00000018: "FILE_DEVICE_PRINTER", + 0x00000019: "FILE_DEVICE_SCANNER", + 0x0000001C: "FILE_DEVICE_SCREEN", + 0x00000037: "FILE_DEVICE_SERENUM", + 0x0000001A: "FILE_DEVICE_SERIAL_MOUSE_PORT", + 0x0000001B: "FILE_DEVICE_SERIAL_PORT", + 0x00000031: "FILE_DEVICE_SMARTCARD", + 0x0000002E: "FILE_DEVICE_SMB", + 0x0000001D: "FILE_DEVICE_SOUND", + 0x0000001E: "FILE_DEVICE_STREAMS", + 0x0000001F: "FILE_DEVICE_TAPE", + 0x00000020: "FILE_DEVICE_TAPE_FILE_SYSTEM", + 0x00000038: "FILE_DEVICE_TERMSRV", + 0x00000021: "FILE_DEVICE_TRANSPORT", + 0x00000022: "FILE_DEVICE_UNKNOWN", + 0x0000002C: "FILE_DEVICE_VDM", + 0x00000023: "FILE_DEVICE_VIDEO", + 0x00000024: "FILE_DEVICE_VIRTUAL_DISK", + 0x00000025: "FILE_DEVICE_WAVE_IN", + 0x00000026: "FILE_DEVICE_WAVE_OUT", } vollog = logging.getLogger(__name__) + class DeviceTree(interfaces.plugins.PluginInterface): """Listing tree based on drivers and attached devices in a particular windows memory image.""" @@ -83,85 +84,118 @@ class DeviceTree(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = "kernel", description = "Windows kernel", - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = "driverscan", plugin = driverscan.DriverScan, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="driverscan", plugin=driverscan.DriverScan, version=(1, 0, 0) + ), ] def _generator(self) -> Iterator[Tuple]: kernel = self.context.modules[self.config["kernel"]] # Scan the Layer for drivers - for driver in driverscan.DriverScan.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): + for driver in driverscan.DriverScan.scan_drivers( + self.context, kernel.layer_name, kernel.symbol_table_name + ): try: try: driver_name = driver.get_driver_name() except (ValueError, exceptions.InvalidAddressException): - vollog.log(constants.LOGLEVEL_VVVV, - f"Failed to get Driver name : {driver.vol.offset:x}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Failed to get Driver name : {driver.vol.offset:x}", + ) driver_name = renderers.UnparsableValue() - yield (0, ( - format_hints.Hex(driver.vol.offset), - "DRV", - driver_name, - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue() - )) + yield ( + 0, + ( + format_hints.Hex(driver.vol.offset), + "DRV", + driver_name, + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + ), + ) # Scan to get the device information of driver. for device in driver.get_devices(): try: device_name = device.get_device_name() except (ValueError, exceptions.InvalidAddressException): - vollog.log(constants.LOGLEVEL_VVVV, - f"Failed to get Device name : {device.vol.offset:x}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Failed to get Device name : {device.vol.offset:x}", + ) device_name = renderers.UnparsableValue() - + device_type = DEVICE_CODES.get(device.DeviceType, "UNKNOWN") - yield (1, ( - format_hints.Hex(driver.vol.offset), - "DEV", - driver_name, - device_name, - renderers.NotApplicableValue(), - device_type - )) - + yield ( + 1, + ( + format_hints.Hex(driver.vol.offset), + "DEV", + driver_name, + device_name, + renderers.NotApplicableValue(), + device_type, + ), + ) + # Scan to get the attached devices information of device. - for level, attached_device in enumerate(device.get_attached_devices(), start=2): + for level, attached_device in enumerate( + device.get_attached_devices(), start=2 + ): try: device_name = attached_device.get_device_name() except (ValueError, exceptions.InvalidAddressException): - vollog.log(constants.LOGLEVEL_VVVV, - f"Failed to get Attached Device Name: {attached_device.vol.offset:x}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Failed to get Attached Device Name: {attached_device.vol.offset:x}", + ) device_name = renderers.UnparsableValue() - - attached_device_driver_name = attached_device.DriverObject.DriverName.get_string() - attached_device_type = DEVICE_CODES.get(attached_device.DeviceType, "UNKNOWN") - yield (level, ( - format_hints.Hex(driver.vol.offset), - "ATT", - driver_name, - device_name, - attached_device_driver_name, - attached_device_type - )) - - except(exceptions.InvalidAddressException): - vollog.log(constants.LOGLEVEL_VVVV, - f"Invalid address identified in drivers and devices: {driver.vol.offset:x}") + attached_device_driver_name = ( + attached_device.DriverObject.DriverName.get_string() + ) + attached_device_type = DEVICE_CODES.get( + attached_device.DeviceType, "UNKNOWN" + ) + + yield ( + level, + ( + format_hints.Hex(driver.vol.offset), + "ATT", + driver_name, + device_name, + attached_device_driver_name, + attached_device_type, + ), + ) + + except (exceptions.InvalidAddressException): + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid address identified in drivers and devices: {driver.vol.offset:x}", + ) continue def run(self) -> renderers.TreeGrid: - return renderers.TreeGrid([ - ("Offset", format_hints.Hex), - ("Type", str), - ("DriverName", str), - ("DeviceName", str), - ("DriverNameOfAttDevice", str), - ("DeviceType", str), - ], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Type", str), + ("DriverName", str), + ("DeviceName", str), + ("DriverNameOfAttDevice", str), + ("DeviceType", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index cb7626dfa..c1593b836 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -28,28 +28,41 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True), - requirements.BooleanRequirement(name = 'dump', - description = "Extract listed DLLs", - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="info", component=info.Info, version=(1, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed DLLs", + default=False, + optional=True, + ), ] @classmethod - def dump_pe(cls, - context: interfaces.context.ContextInterface, - pe_table_name: str, - dll_entry: interfaces.objects.ObjectInterface, - open_method: Type[interfaces.plugins.FileHandlerInterface], - layer_name: str = None, - prefix: str = '') -> Optional[interfaces.plugins.FileHandlerInterface]: + def dump_pe( + cls, + context: interfaces.context.ContextInterface, + pe_table_name: str, + dll_entry: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + layer_name: str = None, + prefix: str = "", + ) -> Optional[interfaces.plugins.FileHandlerInterface]: """Extracts the complete data for a process as a FileInterface Args: @@ -61,45 +74,60 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Returns: An open FileHandlerInterface object containing the complete data for the DLL or None in the case of failure - """ + """ try: try: name = dll_entry.FullDllName.get_string() except exceptions.InvalidAddressException: - name = 'UnreadableDLLName' + name = "UnreadableDLLName" if layer_name is None: layer_name = dll_entry.vol.layer_name - file_handle = open_method("{}{}.{:#x}.{:#x}.dmp".format(prefix, ntpath.basename(name), dll_entry.vol.offset, - dll_entry.DllBase)) + file_handle = open_method( + "{}{}.{:#x}.{:#x}.dmp".format( + prefix, + ntpath.basename(name), + dll_entry.vol.offset, + dll_entry.DllBase, + ) + ) - dos_header = context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset = dll_entry.DllBase, - layer_name = layer_name) + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=dll_entry.DllBase, + layer_name=layer_name, + ) for offset, data in dos_header.reconstruct(): file_handle.seek(offset) file_handle.write(data) - except (IOError, exceptions.VolatilityException, OverflowError, ValueError) as excp: + except ( + IOError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: vollog.debug(f"Unable to dump dll at offset {dll_entry.DllBase}: {excp}") return None return file_handle def _generator(self, procs): - pe_table_name = intermed.IntermediateSymbolTable.create(self.context, - self.config_path, - "windows", - "pe", - class_types = pe.class_types) + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "windows", "pe", class_types=pe.class_types + ) - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - kuser = info.Info.get_kuser_structure(self.context, kernel.layer_name, kernel.symbol_table_name) + kuser = info.Info.get_kuser_structure( + self.context, kernel.layer_name, kernel.symbol_table_name + ) nt_major_version = int(kuser.NtMajorVersion) nt_minor_version = int(kuser.NtMinorVersion) # LoadTime only applies to versions higher or equal to Window 7 (6.1 and higher) - dll_load_time_field = (nt_major_version > 6) or (nt_major_version == 6 and nt_minor_version >= 1) + dll_load_time_field = (nt_major_version > 6) or ( + nt_major_version == 6 and nt_minor_version >= 1 + ) for proc in procs: proc_id = proc.UniqueProcessId @@ -117,20 +145,24 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Versions prior to 6.1 won't have the LoadTime attribute # and 32bit version shouldn't have the Quadpart according to MSDN try: - DllLoadTime = conversion.wintime_to_datetime(entry.LoadTime.QuadPart) + DllLoadTime = conversion.wintime_to_datetime( + entry.LoadTime.QuadPart + ) except exceptions.InvalidAddressException: DllLoadTime = renderers.UnreadableValue() else: DllLoadTime = renderers.NotApplicableValue() file_output = "Disabled" - if self.config['dump']: - file_handle = self.dump_pe(self.context, - pe_table_name, - entry, - self.open, - proc_layer_name, - prefix = f"pid.{proc_id}.") + if self.config["dump"]: + file_handle = self.dump_pe( + self.context, + pe_table_name, + entry, + self.open, + proc_layer_name, + prefix=f"pid.{proc_id}.", + ) file_output = "Error outputting file" if file_handle: file_handle.close() @@ -145,34 +177,69 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): except exceptions.InvalidAddressException: size_of_image = renderers.NotAvailableValue() - yield (0, (proc.UniqueProcessId, - proc.ImageFileName.cast("string", - max_length = proc.ImageFileName.vol.count, - errors = 'replace'), dllbase, size_of_image, BaseDllName, - FullDllName, DllLoadTime, file_output)) + yield ( + 0, + ( + proc.UniqueProcessId, + proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ), + dllbase, + size_of_image, + BaseDllName, + FullDllName, + DllLoadTime, + file_output, + ), + ) def generate_timeline(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] for row in self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name)): + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) + ): _depth, row_data = row if not isinstance(row_data[6], datetime.datetime): continue - description = "DLL Load: Process {} {} Loaded {} ({}) Size {} Offset {}".format( - row_data[0], row_data[1], row_data[4], row_data[5], row_data[3], row_data[2]) + description = ( + "DLL Load: Process {} {} Loaded {} ({}) Size {} Offset {}".format( + row_data[0], + row_data[1], + row_data[4], + row_data[5], + row_data[3], + row_data[2], + ) + ) yield (description, timeliner.TimeLinerType.CREATED, row_data[6]) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] - return renderers.TreeGrid([("PID", int), ("Process", str), ("Base", format_hints.Hex), - ("Size", format_hints.Hex), ("Name", str), ("Path", str), - ("LoadTime", datetime.datetime), ("File output", str)], - self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Base", format_hints.Hex), + ("Size", format_hints.Hex), + ("Name", str), + ("Path", str), + ("LoadTime", datetime.datetime), + ("File output", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index 7f9bc6b08..4d2c24dea 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -9,13 +9,34 @@ from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import ssdt, driverscan MAJOR_FUNCTIONS = [ - 'IRP_MJ_CREATE', 'IRP_MJ_CREATE_NAMED_PIPE', 'IRP_MJ_CLOSE', 'IRP_MJ_READ', 'IRP_MJ_WRITE', - 'IRP_MJ_QUERY_INFORMATION', 'IRP_MJ_SET_INFORMATION', 'IRP_MJ_QUERY_EA', 'IRP_MJ_SET_EA', 'IRP_MJ_FLUSH_BUFFERS', - 'IRP_MJ_QUERY_VOLUME_INFORMATION', 'IRP_MJ_SET_VOLUME_INFORMATION', 'IRP_MJ_DIRECTORY_CONTROL', - 'IRP_MJ_FILE_SYSTEM_CONTROL', 'IRP_MJ_DEVICE_CONTROL', 'IRP_MJ_INTERNAL_DEVICE_CONTROL', 'IRP_MJ_SHUTDOWN', - 'IRP_MJ_LOCK_CONTROL', 'IRP_MJ_CLEANUP', 'IRP_MJ_CREATE_MAILSLOT', 'IRP_MJ_QUERY_SECURITY', 'IRP_MJ_SET_SECURITY', - 'IRP_MJ_POWER', 'IRP_MJ_SYSTEM_CONTROL', 'IRP_MJ_DEVICE_CHANGE', 'IRP_MJ_QUERY_QUOTA', 'IRP_MJ_SET_QUOTA', - 'IRP_MJ_PNP' + "IRP_MJ_CREATE", + "IRP_MJ_CREATE_NAMED_PIPE", + "IRP_MJ_CLOSE", + "IRP_MJ_READ", + "IRP_MJ_WRITE", + "IRP_MJ_QUERY_INFORMATION", + "IRP_MJ_SET_INFORMATION", + "IRP_MJ_QUERY_EA", + "IRP_MJ_SET_EA", + "IRP_MJ_FLUSH_BUFFERS", + "IRP_MJ_QUERY_VOLUME_INFORMATION", + "IRP_MJ_SET_VOLUME_INFORMATION", + "IRP_MJ_DIRECTORY_CONTROL", + "IRP_MJ_FILE_SYSTEM_CONTROL", + "IRP_MJ_DEVICE_CONTROL", + "IRP_MJ_INTERNAL_DEVICE_CONTROL", + "IRP_MJ_SHUTDOWN", + "IRP_MJ_LOCK_CONTROL", + "IRP_MJ_CLEANUP", + "IRP_MJ_CREATE_MAILSLOT", + "IRP_MJ_QUERY_SECURITY", + "IRP_MJ_SET_SECURITY", + "IRP_MJ_POWER", + "IRP_MJ_SYSTEM_CONTROL", + "IRP_MJ_DEVICE_CHANGE", + "IRP_MJ_QUERY_QUOTA", + "IRP_MJ_SET_QUOTA", + "IRP_MJ_PNP", ] @@ -27,18 +48,29 @@ class DriverIrp(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'ssdt', plugin = ssdt.SSDT, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'driverscan', plugin = driverscan.DriverScan, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="driverscan", plugin=driverscan.DriverScan, version=(1, 0, 0) + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - collection = ssdt.SSDT.build_module_collection(self.context, kernel.layer_name, kernel.symbol_table_name) + collection = ssdt.SSDT.build_module_collection( + self.context, kernel.layer_name, kernel.symbol_table_name + ) - for driver in driverscan.DriverScan.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): + for driver in driverscan.DriverScan.scan_drivers( + self.context, kernel.layer_name, kernel.symbol_table_name + ): try: driver_name = driver.get_driver_name() @@ -46,27 +78,50 @@ class DriverIrp(interfaces.plugins.PluginInterface): driver_name = renderers.NotApplicableValue() for i, address in enumerate(driver.MajorFunction): - module_symbols = collection.get_module_symbols_by_absolute_location(address) + module_symbols = collection.get_module_symbols_by_absolute_location( + address + ) for module_name, symbol_generator in module_symbols: symbols_found = False for symbol in symbol_generator: symbols_found = True - yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], - format_hints.Hex(address), module_name, symbol.split(constants.BANG)[1])) + yield ( + 0, + ( + format_hints.Hex(driver.vol.offset), + driver_name, + MAJOR_FUNCTIONS[i], + format_hints.Hex(address), + module_name, + symbol.split(constants.BANG)[1], + ), + ) if not symbols_found: - yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], - format_hints.Hex(address), module_name, renderers.NotAvailableValue())) + yield ( + 0, + ( + format_hints.Hex(driver.vol.offset), + driver_name, + MAJOR_FUNCTIONS[i], + format_hints.Hex(address), + module_name, + renderers.NotAvailableValue(), + ), + ) def run(self): - return renderers.TreeGrid([ - ("Offset", format_hints.Hex), - ("Driver Name", str), - ("IRP", str), - ("Address", format_hints.Hex), - ("Module", str), - ("Symbol", str), - ], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Driver Name", str), + ("IRP", str), + ("Address", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/drivermodule.py b/volatility3/framework/plugins/windows/drivermodule.py index 31f4711b1..cc735db30 100644 --- a/volatility3/framework/plugins/windows/drivermodule.py +++ b/volatility3/framework/plugins/windows/drivermodule.py @@ -8,12 +8,8 @@ from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import ssdt, driverscan # built in Windows-components that trigger false positives -KNOWN_DRIVERS = ["ACPI_HAL", - "PnpManager", - "RAW", - "WMIxWDM", - "Win32k", - "Fs_Rec"] +KNOWN_DRIVERS = ["ACPI_HAL", "PnpManager", "RAW", "WMIxWDM", "Win32k", "Fs_Rec"] + class DriverModule(interfaces.plugins.PluginInterface): """Determines if any loaded drivers were hidden by a rootkit""" @@ -24,10 +20,17 @@ class DriverModule(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'ssdt', plugin = ssdt.SSDT, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'driverscan', plugin = driverscan.DriverScan, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="driverscan", plugin=driverscan.DriverScan, version=(1, 0, 0) + ), ] def _generator(self) -> Iterator[Tuple]: @@ -36,26 +39,48 @@ class DriverModule(interfaces.plugins.PluginInterface): A common rootkit technique is to register drivers from modules that are hidden, which allows us to detect the disconnect between a malicious driver and its hidden module. """ - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - collection = ssdt.SSDT.build_module_collection(self.context, kernel.layer_name, kernel.symbol_table_name) + collection = ssdt.SSDT.build_module_collection( + self.context, kernel.layer_name, kernel.symbol_table_name + ) - for driver in driverscan.DriverScan.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): + for driver in driverscan.DriverScan.scan_drivers( + self.context, kernel.layer_name, kernel.symbol_table_name + ): # we do not care about actual symbol names, we just want to know if the driver points to a known module - module_symbols = list(collection.get_module_symbols_by_absolute_location(driver.DriverStart)) + module_symbols = list( + collection.get_module_symbols_by_absolute_location(driver.DriverStart) + ) if not module_symbols: - driver_name, service_key, name = driverscan.DriverScan.get_names_for_driver(driver) + ( + driver_name, + service_key, + name, + ) = driverscan.DriverScan.get_names_for_driver(driver) known_exception = driver_name in KNOWN_DRIVERS - yield (0, (format_hints.Hex(driver.vol.offset), known_exception, driver_name, service_key, name)) + yield ( + 0, + ( + format_hints.Hex(driver.vol.offset), + known_exception, + driver_name, + service_key, + name, + ), + ) def run(self) -> renderers.TreeGrid: - return renderers.TreeGrid([ - ("Offset", format_hints.Hex), - ("Known Exception", bool), - ("Driver Name", str), - ("Service Key", str), - ("Alternative Name", str), - ], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Known Exception", bool), + ("Driver Name", str), + ("Service Key", str), + ("Alternative Name", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/driverscan.py b/volatility3/framework/plugins/windows/driverscan.py index 60ac0d67a..d8df80702 100644 --- a/volatility3/framework/plugins/windows/driverscan.py +++ b/volatility3/framework/plugins/windows/driverscan.py @@ -19,17 +19,23 @@ class DriverScan(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'poolscanner', plugin = poolscanner.PoolScanner, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), ] @classmethod - def scan_drivers(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def scan_drivers( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Scans for drivers using the poolscanner module and constraints. Args: @@ -41,9 +47,13 @@ class DriverScan(interfaces.plugins.PluginInterface): A list of Driver objects as found from the `layer_name` layer based on Driver pool signatures """ - constraints = poolscanner.PoolScanner.builtin_constraints(symbol_table, [b'Dri\xf6', b'Driv']) + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"Dri\xf6", b"Driv"] + ) - for result in poolscanner.PoolScanner.generate_pool_scan(context, layer_name, symbol_table, constraints): + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): _constraint, mem_object, _header = result yield mem_object @@ -78,15 +88,34 @@ class DriverScan(interfaces.plugins.PluginInterface): return driver_name, service_key, name def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for driver in self.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): + for driver in self.scan_drivers( + self.context, kernel.layer_name, kernel.symbol_table_name + ): driver_name, service_key, name = self.get_names_for_driver(driver) - yield (0, (format_hints.Hex(driver.vol.offset), format_hints.Hex(driver.DriverStart), - format_hints.Hex(driver.DriverSize), service_key, driver_name, name)) + yield ( + 0, + ( + format_hints.Hex(driver.vol.offset), + format_hints.Hex(driver.DriverStart), + format_hints.Hex(driver.DriverSize), + service_key, + driver_name, + name, + ), + ) def run(self): - return renderers.TreeGrid([("Offset", format_hints.Hex), - ("Start", format_hints.Hex), ("Size", format_hints.Hex), ("Service Key", str), - ("Driver Name", str), ("Name", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Start", format_hints.Hex), + ("Size", format_hints.Hex), + ("Service Key", str), + ("Driver Name", str), + ("Name", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index b20efa45d..af9568897 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -33,28 +33,43 @@ class DumpFiles(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', - description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.IntRequirement(name = 'pid', - description = "Process ID to include (all other processes are excluded)", - optional = True), - requirements.IntRequirement(name = 'virtaddr', - description = "Dump a single _FILE_OBJECT at this virtual address", - optional = True), - requirements.IntRequirement(name = 'physaddr', - description = "Dump a single _FILE_OBJECT at this physical address", - optional = True), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'handles', component = handles.Handles, version = (1, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.IntRequirement( + name="pid", + description="Process ID to include (all other processes are excluded)", + optional=True, + ), + requirements.IntRequirement( + name="virtaddr", + description="Dump a single _FILE_OBJECT at this virtual address", + optional=True, + ), + requirements.IntRequirement( + name="physaddr", + description="Dump a single _FILE_OBJECT at this physical address", + optional=True, + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="handles", component=handles.Handles, version=(1, 0, 0) + ), ] @classmethod - def dump_file_producer(cls, file_object: interfaces.objects.ObjectInterface, - memory_object: interfaces.objects.ObjectInterface, - open_method: Type[interfaces.plugins.FileHandlerInterface], - layer: interfaces.layers.DataLayerInterface, - desired_file_name: str) -> Optional[interfaces.plugins.FileHandlerInterface]: + def dump_file_producer( + cls, + file_object: interfaces.objects.ObjectInterface, + memory_object: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + layer: interfaces.layers.DataLayerInterface, + desired_file_name: str, + ) -> Optional[interfaces.plugins.FileHandlerInterface]: """Produce a file from the memory object's get_available_pages() interface. :param file_object: the parent _FILE_OBJECT @@ -74,7 +89,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): bytes_written = 0 try: for memoffset, fileoffset, datasize in memory_object.get_available_pages(): - data = layer.read(memoffset, datasize, pad = True) + data = layer.read(memoffset, datasize, pad=True) bytes_written += len(data) filedata.seek(fileoffset) filedata.write(data) @@ -82,16 +97,22 @@ class DumpFiles(interfaces.plugins.PluginInterface): vollog.debug(f"Unable to dump file at {file_object.vol.offset:#x}") return None if not bytes_written: - vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") + vollog.debug( + f"No data is cached for the file at {file_object.vol.offset:#x}" + ) return None vollog.debug(f"Stored {filedata.preferred_filename}") return filedata @classmethod - def process_file_object(cls, context: interfaces.context.ContextInterface, primary_layer_name: str, - open_method: Type[interfaces.plugins.FileHandlerInterface], - file_obj: interfaces.objects.ObjectInterface) -> Generator[Tuple, None, None]: + def process_file_object( + cls, + context: interfaces.context.ContextInterface, + primary_layer_name: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + file_obj: interfaces.objects.ObjectInterface, + ) -> Generator[Tuple, None, None]: """Given a FILE_OBJECT, dump data to separate files for each of the three file caches. :param context: the context to operate upon @@ -101,13 +122,19 @@ class DumpFiles(interfaces.plugins.PluginInterface): """ # Filtering by these types of devices prevents us from processing other types of devices that # use the "File" object type, such as \Device\Tcp and \Device\NamedPipe. - if file_obj.DeviceObject.DeviceType not in [FILE_DEVICE_DISK, FILE_DEVICE_NETWORK_FILE_SYSTEM]: - vollog.log(constants.LOGLEVEL_VVV, f"The file object at {file_obj.vol.offset:#x} is not a file on disk") + if file_obj.DeviceObject.DeviceType not in [ + FILE_DEVICE_DISK, + FILE_DEVICE_NETWORK_FILE_SYSTEM, + ]: + vollog.log( + constants.LOGLEVEL_VVV, + f"The file object at {file_obj.vol.offset:#x} is not a file on disk", + ) return # Depending on the type of object (DataSection, ImageSection, SharedCacheMap) we may need to # read from the memory layer or the primary layer. - memory_layer_name = context.layers[primary_layer_name].config['memory_layer'] + memory_layer_name = context.layers[primary_layer_name].config["memory_layer"] memory_layer = context.layers[memory_layer_name] primary_layer = context.layers[primary_layer_name] @@ -123,14 +150,20 @@ class DumpFiles(interfaces.plugins.PluginInterface): # The DataSectionObject and ImageSectionObject caches are handled in basically the same way. # We carve these "pages" from the memory_layer. - for member_name, extension in [("DataSectionObject", "dat"), ("ImageSectionObject", "img")]: + for member_name, extension in [ + ("DataSectionObject", "dat"), + ("ImageSectionObject", "img"), + ]: try: section_obj = getattr(file_obj.SectionObjectPointer, member_name) control_area = section_obj.dereference().cast("_CONTROL_AREA") if control_area.is_valid(): dump_parameters.append((control_area, memory_layer, extension)) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"{member_name} is unavailable for file {file_obj.vol.offset:#x}", + ) # The SharedCacheMap is handled differently than the caches above. # We carve these "pages" from the primary_layer. @@ -140,15 +173,24 @@ class DumpFiles(interfaces.plugins.PluginInterface): if shared_cache_map.is_valid(): dump_parameters.append((shared_cache_map, primary_layer, "vacb")) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}", + ) for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] - desired_file_name = "file.{0:#x}.{1:#x}.{2}.{3}.{4}".format(file_obj.vol.offset, - memory_object.vol.offset, cache_name, - ntpath.basename(obj_name), extension) + desired_file_name = "file.{0:#x}.{1:#x}.{2}.{3}.{4}".format( + file_obj.vol.offset, + memory_object.vol.offset, + cache_name, + ntpath.basename(obj_name), + extension, + ) - file_handle = cls.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) + file_handle = cls.dump_file_producer( + file_obj, memory_object, open_method, layer, desired_file_name + ) file_output = "Error dumping file" if file_handle: @@ -158,31 +200,43 @@ class DumpFiles(interfaces.plugins.PluginInterface): yield ( cache_name, format_hints.Hex(file_obj.vol.offset), - ntpath.basename(obj_name), # temporary, so its easier to visualize output - file_output) + ntpath.basename( + obj_name + ), # temporary, so its easier to visualize output + file_output, + ) def _generator(self, procs: List, offsets: List): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing # private variables, so we need an instance (for now, anyway). We _could_ call Handles._generator() # to do some of the other work that is duplicated here, but then we'd need to parse the TreeGrid # results instead of just dealing with them as direct objects here. - handles_plugin = handles.Handles(context = self.context, config_path = self._config_path) - type_map = handles_plugin.get_type_map(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name) - cookie = handles_plugin.find_cookie(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name) + handles_plugin = handles.Handles( + context=self.context, config_path=self._config_path + ) + type_map = handles_plugin.get_type_map( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) + cookie = handles_plugin.find_cookie( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) for proc in procs: try: object_table = proc.ObjectTable except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}", + ) continue for entry in handles_plugin.handles(object_table): @@ -190,12 +244,15 @@ class DumpFiles(interfaces.plugins.PluginInterface): obj_type = entry.get_object_type(type_map, cookie) if obj_type == "File": file_obj = entry.Body.cast("_FILE_OBJECT") - for result in self.process_file_object(self.context, kernel.layer_name, self.open, - file_obj): + for result in self.process_file_object( + self.context, kernel.layer_name, self.open, file_obj + ): yield (0, result) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot extract file from _OBJECT_HEADER at {entry.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Cannot extract file from _OBJECT_HEADER at {entry.vol.offset:#x}", + ) # Pull file objects from the VADs. This will produce DLLs and EXEs that are # mapped into the process as images, but that the process doesn't have an @@ -207,17 +264,24 @@ class DumpFiles(interfaces.plugins.PluginInterface): file_obj = vad.ControlArea.FilePointer.dereference() elif vad.has_member("Subsection"): # Vista and beyond - file_obj = vad.Subsection.ControlArea.FilePointer.dereference().cast("_FILE_OBJECT") + file_obj = vad.Subsection.ControlArea.FilePointer.dereference().cast( + "_FILE_OBJECT" + ) else: continue if not file_obj.is_valid(): continue - for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): + for result in self.process_file_object( + self.context, kernel.layer_name, self.open, file_obj + ): yield (0, result) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, f"Cannot extract file from VAD at {vad.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Cannot extract file from VAD at {vad.vol.offset:#x}", + ) elif offsets: # Now process any offsets explicitly requested by the user. @@ -226,34 +290,53 @@ class DumpFiles(interfaces.plugins.PluginInterface): layer_name = kernel.layer_name # switch to a memory layer if the user provided --physaddr instead of --virtaddr if not is_virtual: - layer_name = self.context.layers[layer_name].config["memory_layer"] + layer_name = self.context.layers[layer_name].config[ + "memory_layer" + ] - file_obj = self.context.object(kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", - layer_name = layer_name, - native_layer_name = kernel.layer_name, - offset = offset) - for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): + file_obj = self.context.object( + kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", + layer_name=layer_name, + native_layer_name=kernel.layer_name, + offset=offset, + ) + for result in self.process_file_object( + self.context, kernel.layer_name, self.open, file_obj + ): yield (0, result) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, f"Cannot extract file at {offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, f"Cannot extract file at {offset:#x}" + ) def run(self): # a list of tuples (, ) where is the address and is True for virtual. offsets = list() # a list of processes matching the pid filter. all files for these process(es) will be dumped. procs = list() - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] if self.config.get("virtaddr", None) is not None: offsets.append((self.config["virtaddr"], True)) elif self.config.get("physaddr", None) is not None: offsets.append((self.config["physaddr"], False)) else: - filter_func = pslist.PsList.create_pid_filter([self.config.get("pid", None)]) - procs = pslist.PsList.list_processes(self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func = filter_func) + filter_func = pslist.PsList.create_pid_filter( + [self.config.get("pid", None)] + ) + procs = pslist.PsList.list_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=filter_func, + ) - return renderers.TreeGrid([("Cache", str), ("FileObject", format_hints.Hex), ("FileName", str), - ("Result", str)], self._generator(procs, offsets)) + return renderers.TreeGrid( + [ + ("Cache", str), + ("FileObject", format_hints.Hex), + ("FileName", str), + ("Result", str), + ], + self._generator(procs, offsets), + ) diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index e9015280a..a1dbd7665 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -23,17 +23,28 @@ class Envars(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True), - requirements.BooleanRequirement(name = 'silent', - description = 'Suppress common and non-persistent variables', - optional = True), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.BooleanRequirement( + name="silent", + description="Suppress common and non-persistent variables", + optional=True, + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), ] def _get_silent_vars(self) -> List[str]: @@ -47,23 +58,29 @@ class Envars(interfaces.plugins.PluginInterface): """ values = [] - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for hive in hivelist.HiveList.list_hives(context = self.context, - base_config_path = self.config_path, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - hive_offsets = None): + for hive in hivelist.HiveList.list_hives( + context=self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + hive_offsets=None, + ): sys = False ntuser = False ## The global variables try: - key = hive.get_key('CurrentControlSet\\Control\\Session Manager\\Environment') + key = hive.get_key( + "CurrentControlSet\\Control\\Session Manager\\Environment" + ) sys = True except KeyError: with contextlib.suppress(KeyError): - key = hive.get_key('ControlSet001\\Control\\Session Manager\\Environment') + key = hive.get_key( + "ControlSet001\\Control\\Session Manager\\Environment" + ) sys = True if sys: with contextlib.suppress(KeyError): @@ -72,15 +89,19 @@ class Envars(interfaces.plugins.PluginInterface): value_node_name = node.get_name() if value_node_name: values.append(value_node_name) - except (exceptions.InvalidAddressException, registry.RegistryFormatException) as excp: + except ( + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ) as excp: vollog.log( constants.LOGLEVEL_VVV, - "Error while parsing global environment variables keys (some keys might be excluded)") + "Error while parsing global environment variables keys (some keys might be excluded)", + ) continue ## The user-specific variables with contextlib.suppress(KeyError): - key = hive.get_key('Environment') + key = hive.get_key("Environment") ntuser = True if ntuser: with contextlib.suppress(KeyError): @@ -89,15 +110,19 @@ class Envars(interfaces.plugins.PluginInterface): value_node_name = node.get_name() if value_node_name: values.append(value_node_name) - except (exceptions.InvalidAddressException, registry.RegistryFormatException) as excp: + except ( + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ) as excp: vollog.log( constants.LOGLEVEL_VVV, - "Error while parsing user environment variables keys (some keys might be excluded)") + "Error while parsing user environment variables keys (some keys might be excluded)", + ) continue ## The volatile user variables try: - key = hive.get_key('Volatile Environment') + key = hive.get_key("Volatile Environment") except KeyError: continue try: @@ -106,89 +131,114 @@ class Envars(interfaces.plugins.PluginInterface): value_node_name = node.get_name() if value_node_name: values.append(value_node_name) - except (exceptions.InvalidAddressException, registry.RegistryFormatException) as excp: + except ( + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ) as excp: vollog.log( constants.LOGLEVEL_VVV, - "Error while parsing volatile environment variables keys (some keys might be excluded)") + "Error while parsing volatile environment variables keys (some keys might be excluded)", + ) continue except KeyError: continue ## These are variables set explicitly but are ## common enough to ignore safely. - values.extend([ - "ProgramFiles", - "CommonProgramFiles", - "SystemDrive", - "SystemRoot", - "ProgramData", - "PUBLIC", - "ALLUSERSPROFILE", - "COMPUTERNAME", - "SESSIONNAME", - "USERNAME", - "USERPROFILE", - "PROMPT", - "USERDOMAIN", - "AppData", - "CommonFiles", - "CommonDesktop", - "CommonProgramGroups", - "CommonStartMenu", - "CommonStartUp", - "Cookies", - "DesktopDirectory", - "Favorites", - "History", - "NetHood", - "PersonalDocuments", - "RecycleBin", - "StartMenu", - "Templates", - "AltStartup", - "CommonFavorites", - "ConnectionWizard", - "DocAndSettingRoot", - "InternetCache", - "windir", - "Path", - "HOMEDRIVE", - "PROCESSOR_ARCHITECTURE", - "NUMBER_OF_PROCESSORS", - "ProgramFiles(x86)", - "CommonProgramFiles(x86)", - "CommonProgramW6432", - "PSModulePath", - "PROCESSOR_IDENTIFIER", - "FP_NO_HOST_CHECK", - "LOCALAPPDATA", - "TMP", - "ProgramW6432", - ]) + values.extend( + [ + "ProgramFiles", + "CommonProgramFiles", + "SystemDrive", + "SystemRoot", + "ProgramData", + "PUBLIC", + "ALLUSERSPROFILE", + "COMPUTERNAME", + "SESSIONNAME", + "USERNAME", + "USERPROFILE", + "PROMPT", + "USERDOMAIN", + "AppData", + "CommonFiles", + "CommonDesktop", + "CommonProgramGroups", + "CommonStartMenu", + "CommonStartUp", + "Cookies", + "DesktopDirectory", + "Favorites", + "History", + "NetHood", + "PersonalDocuments", + "RecycleBin", + "StartMenu", + "Templates", + "AltStartup", + "CommonFavorites", + "ConnectionWizard", + "DocAndSettingRoot", + "InternetCache", + "windir", + "Path", + "HOMEDRIVE", + "PROCESSOR_ARCHITECTURE", + "NUMBER_OF_PROCESSORS", + "ProgramFiles(x86)", + "CommonProgramFiles(x86)", + "CommonProgramW6432", + "PSModulePath", + "PROCESSOR_IDENTIFIER", + "FP_NO_HOST_CHECK", + "LOCALAPPDATA", + "TMP", + "ProgramW6432", + ] + ) return values def _generator(self, data): silent_vars = [] - if self.config.get('SILENT', None): + if self.config.get("SILENT", None): silent_vars = self._get_silent_vars() for task in data: for var, val in task.environment_variables(): - if self.config.get('silent', None): + if self.config.get("silent", None): if var in silent_vars: continue - yield (0, (int(task.UniqueProcessId), str(objects.utility.array_to_string(task.ImageFileName)), - hex(task.get_peb().ProcessParameters.Environment.vol.offset), str(var), str(val))) + yield ( + 0, + ( + int(task.UniqueProcessId), + str(objects.utility.array_to_string(task.ImageFileName)), + hex(task.get_peb().ProcessParameters.Environment.vol.offset), + str(var), + str(val), + ), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] - return renderers.TreeGrid([("PID", int), ("Process", str), ("Block", str), ("Variable", str), ("Value", str)], - self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Block", str), + ("Variable", str), + ("Value", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/filescan.py b/volatility3/framework/plugins/windows/filescan.py index 79d85eb8d..de3331e16 100644 --- a/volatility3/framework/plugins/windows/filescan.py +++ b/volatility3/framework/plugins/windows/filescan.py @@ -18,17 +18,23 @@ class FileScan(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'poolscanner', plugin = poolscanner.PoolScanner, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), ] @classmethod - def scan_files(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def scan_files( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Scans for file objects using the poolscanner module and constraints. Args: @@ -40,17 +46,23 @@ class FileScan(interfaces.plugins.PluginInterface): A list of File objects as found from the `layer_name` layer based on File pool signatures """ - constraints = poolscanner.PoolScanner.builtin_constraints(symbol_table, [b'Fil\xe5', b'File']) + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"Fil\xe5", b"File"] + ) - for result in poolscanner.PoolScanner.generate_pool_scan(context, layer_name, symbol_table, constraints): + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): _constraint, mem_object, _header = result yield mem_object def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for fileobj in self.scan_files(self.context, kernel.layer_name, kernel.symbol_table_name): + for fileobj in self.scan_files( + self.context, kernel.layer_name, kernel.symbol_table_name + ): try: file_name = fileobj.FileName.String @@ -60,4 +72,7 @@ class FileScan(interfaces.plugins.PluginInterface): yield (0, (format_hints.Hex(fileobj.vol.offset), file_name, fileobj.Size)) def run(self): - return renderers.TreeGrid([("Offset", format_hints.Hex), ("Name", str), ("Size", int)], self._generator()) + return renderers.TreeGrid( + [("Offset", format_hints.Hex), ("Name", str), ("Size", int)], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index a47b1178a..c4088426f 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -16,15 +16,17 @@ vollog = logging.getLogger(__name__) def createservicesid(svc) -> str: - """ Calculate the Service SID """ - uni = ''.join([c + '\x00' for c in svc]) - sha = hashlib.sha1(uni.upper().encode("utf-8")).digest() # pylint: disable-msg=E1101 + """Calculate the Service SID""" + uni = "".join([c + "\x00" for c in svc]) + sha = hashlib.sha1( + uni.upper().encode("utf-8") + ).digest() # pylint: disable-msg=E1101 dec = list() for i in range(5): ## The use of struct here is OK. It doesn't make much sense ## to leverage obj.Object inside this loop. - dec.append(struct.unpack(' List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] # Get the system hive - for hive in hivelist.HiveList.list_hives(context = self.context, - base_config_path = self.config_path, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_string = 'machine\\system', - hive_offsets = None): + for hive in hivelist.HiveList.list_hives( + context=self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_string="machine\\system", + hive_offsets=None, + ): # Get ControlSet\Services. try: services = hive.get_key(r"CurrentControlSet\Services") diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index da6fa71c9..2334a328d 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -8,7 +8,14 @@ import os import re from typing import List, Dict, Union -from volatility3.framework import renderers, interfaces, objects, exceptions, constants, layers +from volatility3.framework import ( + renderers, + interfaces, + objects, + exceptions, + constants, + layers, +) from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.windows.extensions import registry @@ -18,7 +25,9 @@ from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) -def find_sid_re(sid_string, sid_re_list) -> Union[str, interfaces.renderers.BaseAbsentValue]: +def find_sid_re( + sid_string, sid_re_list +) -> Union[str, interfaces.renderers.BaseAbsentValue]: for reg, name in sid_re_list: if reg.search(sid_string): return name @@ -34,33 +43,52 @@ class GetSIDs(interfaces.plugins.PluginInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) for plugin_dir in constants.PLUGINS_PATH: - sids_json_file_name = os.path.join(plugin_dir, os.path.join("windows", "sids_and_privileges.json")) + sids_json_file_name = os.path.join( + plugin_dir, os.path.join("windows", "sids_and_privileges.json") + ) if os.path.exists(sids_json_file_name): break else: - vollog.log(constants.LOGLEVEL_VVV, 'sids_and_privileges.json file is missing plugin error') - raise RuntimeError("The sids_and_privileges.json file missed from you plugin directory") + vollog.log( + constants.LOGLEVEL_VVV, + "sids_and_privileges.json file is missing plugin error", + ) + raise RuntimeError( + "The sids_and_privileges.json file missed from you plugin directory" + ) # Get all the sids from the json file. - with open(sids_json_file_name, 'r') as file_handle: + with open(sids_json_file_name, "r") as file_handle: sids_json_data = json.load(file_handle) - self.servicesids = sids_json_data['service sids'] - self.well_known_sids = sids_json_data['well known'] + self.servicesids = sids_json_data["service sids"] + self.well_known_sids = sids_json_data["well known"] # Compile all the sids regex. - self.well_known_sid_re = [(re.compile(c_list[0]), c_list[1]) for c_list in sids_json_data['sids re']] + self.well_known_sid_re = [ + (re.compile(c_list[0]), c_list[1]) + for c_list in sids_json_data["sids re"] + ] @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), ] def lookup_user_sids(self) -> Dict[str, str]: @@ -73,15 +101,17 @@ class GetSIDs(interfaces.plugins.PluginInterface): key = "Microsoft\\Windows NT\\CurrentVersion\\ProfileList" val = "ProfileImagePath" - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] sids = {} - for hive in hivelist.HiveList.list_hives(context = self.context, - base_config_path = self.config_path, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_string = 'config\\software', - hive_offsets = None): + for hive in hivelist.HiveList.list_hives( + context=self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_string="config\\software", + hive_offsets=None, + ): try: for subkey in hive.get_key(key).get_subkeys(): @@ -90,26 +120,44 @@ class GetSIDs(interfaces.plugins.PluginInterface): for node in subkey.get_values(): try: value_node_name = node.get_name() or "(Default)" - except (exceptions.InvalidAddressException, layers.registry.RegistryFormatException) as excp: + except ( + exceptions.InvalidAddressException, + layers.registry.RegistryFormatException, + ) as excp: continue try: value_data = node.decode_data() if isinstance(value_data, int): - value_data = format_hints.MultiTypeData(value_data, encoding = 'utf-8') - elif registry.RegValueTypes(node.Type) == registry.RegValueTypes.REG_BINARY: - value_data = format_hints.MultiTypeData(value_data, show_hex = True) - elif registry.RegValueTypes(node.Type) == registry.RegValueTypes.REG_MULTI_SZ: - value_data = format_hints.MultiTypeData(value_data, - encoding = 'utf-16-le', - split_nulls = True) + value_data = format_hints.MultiTypeData( + value_data, encoding="utf-8" + ) + elif ( + registry.RegValueTypes(node.Type) + == registry.RegValueTypes.REG_BINARY + ): + value_data = format_hints.MultiTypeData( + value_data, show_hex=True + ) + elif ( + registry.RegValueTypes(node.Type) + == registry.RegValueTypes.REG_MULTI_SZ + ): + value_data = format_hints.MultiTypeData( + value_data, encoding="utf-16-le", split_nulls=True + ) else: - value_data = format_hints.MultiTypeData(value_data, encoding = 'utf-16-le') + value_data = format_hints.MultiTypeData( + value_data, encoding="utf-16-le" + ) if value_node_name == val: - path = str(value_data).replace('\\x00', '')[:-1] + path = str(value_data).replace("\\x00", "")[:-1] user = ntpath.basename(path) sids[sid] = user - except (ValueError, exceptions.InvalidAddressException, - layers.registry.RegistryFormatException) as excp: + except ( + ValueError, + exceptions.InvalidAddressException, + layers.registry.RegistryFormatException, + ) as excp: continue except (KeyError, exceptions.InvalidAddressException): continue @@ -129,7 +177,15 @@ class GetSIDs(interfaces.plugins.PluginInterface): token = False if not token or not isinstance(token, interfaces.objects.ObjectInterface): - yield (0, [int(task.UniqueProcessId), str(task.ImageFileName), "Token unreadable", ""]) + yield ( + 0, + [ + int(task.UniqueProcessId), + str(task.ImageFileName), + "Token unreadable", + "", + ], + ) continue # Go all over the sids and try to translate them with one of the tables we have @@ -147,17 +203,29 @@ class GetSIDs(interfaces.plugins.PluginInterface): else: sid_name = "" - yield (0, (task.UniqueProcessId, objects.utility.array_to_string(task.ImageFileName), sid_string, - sid_name)) + yield ( + 0, + ( + task.UniqueProcessId, + objects.utility.array_to_string(task.ImageFileName), + sid_string, + sid_name, + ), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] - return renderers.TreeGrid([("PID", int), ("Process", str), ("SID", str), ("Name", str)], - self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [("PID", int), ("Process", str), ("SID", str), ("Name", str)], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index bdff88075..2f25a0597 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -38,13 +38,20 @@ class Handles(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), ] def _decode_pointer(self, value, magic): @@ -67,7 +74,7 @@ class Handles(interfaces.plugins.PluginInterface): process' handle table, determine where the corresponding object's _OBJECT_HEADER can be found.""" - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] virtual = kernel.layer_name @@ -80,7 +87,9 @@ class Handles(interfaces.plugins.PluginInterface): object_header.GrantedAccess = handle_table_entry.GrantedAccess except AttributeError: # starting with windows 8 - is_64bit = symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) + is_64bit = symbols.symbol_table_is_64bit( + self.context, kernel.symbol_table_name + ) if is_64bit: if handle_table_entry.LowValue == 0: @@ -91,10 +100,14 @@ class Handles(interfaces.plugins.PluginInterface): # is this the right thing to raise here? if magic is None: if has_capstone: - raise AttributeError("Unable to find the SAR value for decoding handle table pointers") + raise AttributeError( + "Unable to find the SAR value for decoding handle table pointers" + ) else: raise exceptions.MissingModuleException( - "capstone", "Requires capstone to find the SAR value for decoding handle table pointers") + "capstone", + "Requires capstone to find the SAR value for decoding handle table pointers", + ) offset = self._decode_pointer(handle_table_entry.LowValue, magic) else: @@ -104,8 +117,11 @@ class Handles(interfaces.plugins.PluginInterface): offset = handle_table_entry.InfoTable & ~7 # print("LowValue: {0:#x} Magic: {1:#x} Offset: {2:#x}".format(handle_table_entry.InfoTable, magic, offset)) - object_header = self.context.object(kernel.symbol_table_name + constants.BANG + "_OBJECT_HEADER", virtual, - offset = offset) + object_header = self.context.object( + kernel.symbol_table_name + constants.BANG + "_OBJECT_HEADER", + virtual, + offset=offset, + ) object_header.GrantedAccess = handle_table_entry.GrantedAccessBits object_header.HandleValue = handle_value @@ -123,11 +139,15 @@ class Handles(interfaces.plugins.PluginInterface): if not has_capstone: return None - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] virtual_layer_name = kernel.layer_name - kvo = self.context.layers[virtual_layer_name].config['kernel_virtual_offset'] - ntkrnlmp = self.context.module(kernel.symbol_table_name, layer_name = virtual_layer_name, offset = kvo) + kvo = self.context.layers[virtual_layer_name].config[ + "kernel_virtual_offset" + ] + ntkrnlmp = self.context.module( + kernel.symbol_table_name, layer_name=virtual_layer_name, offset=kvo + ) try: func_addr = ntkrnlmp.get_symbol("ObpCaptureHandleInformationEx").address @@ -140,7 +160,9 @@ class Handles(interfaces.plugins.PluginInterface): md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) - for (address, size, mnemonic, op_str) in md.disasm_lite(data, kvo + func_addr): + for (address, size, mnemonic, op_str) in md.disasm_lite( + data, kvo + func_addr + ): # print("{} {} {} {}".format(address, size, mnemonic, op_str)) if mnemonic.startswith("sar"): @@ -152,8 +174,12 @@ class Handles(interfaces.plugins.PluginInterface): return self._sar_value @classmethod - def get_type_map(cls, context: interfaces.context.ContextInterface, layer_name: str, - symbol_table: str) -> Dict[int, str]: + def get_type_map( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Dict[int, str]: """List the executive object types (_OBJECT_TYPE) using the ObTypeIndexTable or ObpObjectTypes symbol (differs per OS). This method will be necessary for determining what type of object we have given an @@ -174,8 +200,8 @@ class Handles(interfaces.plugins.PluginInterface): type_map: Dict[int, str] = {} - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) try: table_addr = ntkrnlmp.get_symbol("ObTypeIndexTable").address @@ -187,10 +213,12 @@ class Handles(interfaces.plugins.PluginInterface): if not trans_layer.is_valid(kvo + table_addr): return type_map - ptrs = ntkrnlmp.object(object_type = "array", - offset = table_addr, - subtype = ntkrnlmp.get_type("pointer"), - count = 100) + ptrs = ntkrnlmp.object( + object_type="array", + offset=table_addr, + subtype=ntkrnlmp.get_type("pointer"), + count=100, + ) for i, ptr in enumerate(ptrs): # type: ignore # the first entry in the table is always null. break the @@ -199,11 +227,15 @@ class Handles(interfaces.plugins.PluginInterface): break try: - objt = ptr.dereference().cast(symbol_table + constants.BANG + "_OBJECT_TYPE") + objt = ptr.dereference().cast( + symbol_table + constants.BANG + "_OBJECT_TYPE" + ) type_name = objt.Name.String except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot access _OBJECT_HEADER Name at {objt.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Cannot access _OBJECT_HEADER Name at {objt.vol.offset:#x}", + ) continue type_map[i] = type_name @@ -211,28 +243,40 @@ class Handles(interfaces.plugins.PluginInterface): return type_map @classmethod - def find_cookie(cls, context: interfaces.context.ContextInterface, layer_name: str, - symbol_table: str) -> Optional[interfaces.objects.ObjectInterface]: + def find_cookie( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Optional[interfaces.objects.ObjectInterface]: """Find the ObHeaderCookie value (if it exists)""" try: - offset = context.symbol_space.get_symbol(symbol_table + constants.BANG + "ObHeaderCookie").address + offset = context.symbol_space.get_symbol( + symbol_table + constants.BANG + "ObHeaderCookie" + ).address except exceptions.SymbolError: return None - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - return context.object(symbol_table + constants.BANG + "unsigned int", layer_name, offset = kvo + offset) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + return context.object( + symbol_table + constants.BANG + "unsigned int", + layer_name, + offset=kvo + offset, + ) - def _make_handle_array(self, offset, level, depth = 0): + def _make_handle_array(self, offset, level, depth=0): """Parse a process' handle table and yield valid handle table entries, going as deep into the table "levels" as necessary.""" - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] virtual = kernel.layer_name - kvo = self.context.layers[virtual].config['kernel_virtual_offset'] + kvo = self.context.layers[virtual].config["kernel_virtual_offset"] - ntkrnlmp = self.context.module(kernel.symbol_table_name, layer_name = virtual, offset = kvo) + ntkrnlmp = self.context.module( + kernel.symbol_table_name, layer_name=virtual, offset=kvo + ) if level > 0: subtype = ntkrnlmp.get_type("pointer") @@ -244,14 +288,16 @@ class Handles(interfaces.plugins.PluginInterface): if not self.context.layers[virtual].is_valid(offset): return - table = ntkrnlmp.object(object_type = "array", - offset = offset, - subtype = subtype, - count = int(count), - absolute = True) + table = ntkrnlmp.object( + object_type="array", + offset=offset, + subtype=subtype, + count=int(count), + absolute=True, + ) layer_object = self.context.layers[virtual] - masked_offset = (offset & layer_object.maximum_address) + masked_offset = offset & layer_object.maximum_address for entry in table: @@ -263,8 +309,10 @@ class Handles(interfaces.plugins.PluginInterface): handle_multiplier = 4 handle_level_base = depth * count * handle_multiplier - handle_value = ((entry.vol.offset - masked_offset) / - (subtype.size / handle_multiplier)) + handle_level_base + handle_value = ( + (entry.vol.offset - masked_offset) + / (subtype.size / handle_multiplier) + ) + handle_level_base item = self._get_item(entry, handle_value) @@ -286,29 +334,38 @@ class Handles(interfaces.plugins.PluginInterface): TableCode = handle_table.TableCode & ~self._level_mask table_levels = handle_table.TableCode & self._level_mask except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, "Handle table parsing was aborted due to an invalid address exception") + vollog.log( + constants.LOGLEVEL_VVV, + "Handle table parsing was aborted due to an invalid address exception", + ) return for handle_table_entry in self._make_handle_array(TableCode, table_levels): yield handle_table_entry def _generator(self, procs): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - type_map = self.get_type_map(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name) + type_map = self.get_type_map( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) - cookie = self.find_cookie(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name) + cookie = self.find_cookie( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) for proc in procs: try: object_table = proc.ObjectTable except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot access _EPROCESS.ObjectType at {proc.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Cannot access _EPROCESS.ObjectType at {proc.vol.offset:#x}", + ) continue process_name = utility.array_to_string(proc.ImageFileName) @@ -326,7 +383,9 @@ class Handles(interfaces.plugins.PluginInterface): obj_name = f"{utility.array_to_string(item.ImageFileName)} Pid {item.UniqueProcessId}" elif obj_type == "Thread": item = entry.Body.cast("_ETHREAD") - obj_name = f"Tid {item.Cid.UniqueThread} Pid {item.Cid.UniqueProcess}" + obj_name = ( + f"Tid {item.Cid.UniqueThread} Pid {item.Cid.UniqueProcess}" + ) elif obj_type == "Key": item = entry.Body.cast("_CM_KEY_BODY") obj_name = item.get_full_key_name() @@ -337,24 +396,46 @@ class Handles(interfaces.plugins.PluginInterface): obj_name = "" except (exceptions.InvalidAddressException): - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot access _OBJECT_HEADER at {entry.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Cannot access _OBJECT_HEADER at {entry.vol.offset:#x}", + ) continue - yield (0, (proc.UniqueProcessId, process_name, format_hints.Hex(entry.Body.vol.offset), - format_hints.Hex(entry.HandleValue), obj_type, format_hints.Hex(entry.GrantedAccess), - obj_name)) + yield ( + 0, + ( + proc.UniqueProcessId, + process_name, + format_hints.Hex(entry.Body.vol.offset), + format_hints.Hex(entry.HandleValue), + obj_type, + format_hints.Hex(entry.GrantedAccess), + obj_name, + ), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] - return renderers.TreeGrid([("PID", int), ("Process", str), ("Offset", format_hints.Hex), - ("HandleValue", format_hints.Hex), ("Type", str), - ("GrantedAccess", format_hints.Hex), ("Name", str)], - self._generator( - pslist.PsList.list_processes(self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Offset", format_hints.Hex), + ("HandleValue", format_hints.Hex), + ("Type", str), + ("GrantedAccess", format_hints.Hex), + ("Name", str), + ], + self._generator( + pslist.PsList.list_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index e9f8047e0..72bea2c8b 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -27,27 +27,294 @@ class Hashdump(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), ] odd_parity = [ - 1, 1, 2, 2, 4, 4, 7, 7, 8, 8, 11, 11, 13, 13, 14, 14, 16, 16, 19, 19, 21, 21, 22, 22, 25, 25, 26, 26, 28, 28, - 31, 31, 32, 32, 35, 35, 37, 37, 38, 38, 41, 41, 42, 42, 44, 44, 47, 47, 49, 49, 50, 50, 52, 52, 55, 55, 56, 56, - 59, 59, 61, 61, 62, 62, 64, 64, 67, 67, 69, 69, 70, 70, 73, 73, 74, 74, 76, 76, 79, 79, 81, 81, 82, 82, 84, 84, - 87, 87, 88, 88, 91, 91, 93, 93, 94, 94, 97, 97, 98, 98, 100, 100, 103, 103, 104, 104, 107, 107, 109, 109, 110, - 110, 112, 112, 115, 115, 117, 117, 118, 118, 121, 121, 122, 122, 124, 124, 127, 127, 128, 128, 131, 131, 133, - 133, 134, 134, 137, 137, 138, 138, 140, 140, 143, 143, 145, 145, 146, 146, 148, 148, 151, 151, 152, 152, 155, - 155, 157, 157, 158, 158, 161, 161, 162, 162, 164, 164, 167, 167, 168, 168, 171, 171, 173, 173, 174, 174, 176, - 176, 179, 179, 181, 181, 182, 182, 185, 185, 186, 186, 188, 188, 191, 191, 193, 193, 194, 194, 196, 196, 199, - 199, 200, 200, 203, 203, 205, 205, 206, 206, 208, 208, 211, 211, 213, 213, 214, 214, 217, 217, 218, 218, 220, - 220, 223, 223, 224, 224, 227, 227, 229, 229, 230, 230, 233, 233, 234, 234, 236, 236, 239, 239, 241, 241, 242, - 242, 244, 244, 247, 247, 248, 248, 251, 251, 253, 253, 254, 254 + 1, + 1, + 2, + 2, + 4, + 4, + 7, + 7, + 8, + 8, + 11, + 11, + 13, + 13, + 14, + 14, + 16, + 16, + 19, + 19, + 21, + 21, + 22, + 22, + 25, + 25, + 26, + 26, + 28, + 28, + 31, + 31, + 32, + 32, + 35, + 35, + 37, + 37, + 38, + 38, + 41, + 41, + 42, + 42, + 44, + 44, + 47, + 47, + 49, + 49, + 50, + 50, + 52, + 52, + 55, + 55, + 56, + 56, + 59, + 59, + 61, + 61, + 62, + 62, + 64, + 64, + 67, + 67, + 69, + 69, + 70, + 70, + 73, + 73, + 74, + 74, + 76, + 76, + 79, + 79, + 81, + 81, + 82, + 82, + 84, + 84, + 87, + 87, + 88, + 88, + 91, + 91, + 93, + 93, + 94, + 94, + 97, + 97, + 98, + 98, + 100, + 100, + 103, + 103, + 104, + 104, + 107, + 107, + 109, + 109, + 110, + 110, + 112, + 112, + 115, + 115, + 117, + 117, + 118, + 118, + 121, + 121, + 122, + 122, + 124, + 124, + 127, + 127, + 128, + 128, + 131, + 131, + 133, + 133, + 134, + 134, + 137, + 137, + 138, + 138, + 140, + 140, + 143, + 143, + 145, + 145, + 146, + 146, + 148, + 148, + 151, + 151, + 152, + 152, + 155, + 155, + 157, + 157, + 158, + 158, + 161, + 161, + 162, + 162, + 164, + 164, + 167, + 167, + 168, + 168, + 171, + 171, + 173, + 173, + 174, + 174, + 176, + 176, + 179, + 179, + 181, + 181, + 182, + 182, + 185, + 185, + 186, + 186, + 188, + 188, + 191, + 191, + 193, + 193, + 194, + 194, + 196, + 196, + 199, + 199, + 200, + 200, + 203, + 203, + 205, + 205, + 206, + 206, + 208, + 208, + 211, + 211, + 213, + 213, + 214, + 214, + 217, + 217, + 218, + 218, + 220, + 220, + 223, + 223, + 224, + 224, + 227, + 227, + 229, + 229, + 230, + 230, + 233, + 233, + 234, + 234, + 236, + 236, + 239, + 239, + 241, + 241, + 242, + 242, + 244, + 244, + 247, + 247, + 248, + 248, + 251, + 251, + 253, + 253, + 254, + 254, ] # Permutation matrix for boot key - bootkey_perm_table = [0x8, 0x5, 0x4, 0x2, 0xb, 0x9, 0xd, 0x3, 0x0, 0x6, 0x1, 0xc, 0xe, 0xa, 0xf, 0x7] + bootkey_perm_table = [ + 0x8, + 0x5, + 0x4, + 0x2, + 0xB, + 0x9, + 0xD, + 0x3, + 0x0, + 0x6, + 0x1, + 0xC, + 0xE, + 0xA, + 0xF, + 0x7, + ] # Constants for SAM decrypt algorithm aqwerty = b"!@#$%^&*()qwertyUIOPAzxcvbnmQQQQQQQQQQQQ)(*@&%\0" @@ -67,11 +334,14 @@ class Hashdump(interfaces.plugins.PluginInterface): result = hive.get_key(key) except KeyError: vollog.info( - f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image") + f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image" + ) return result @classmethod - def get_user_keys(cls, samhive: registry.RegistryHive) -> List[interfaces.objects.ObjectInterface]: + def get_user_keys( + cls, samhive: registry.RegistryHive + ) -> List[interfaces.objects.ObjectInterface]: user_key_path = "SAM\\Domains\\Account\\Users" user_key = cls.get_hive_key(samhive, user_key_path) @@ -91,24 +361,28 @@ class Hashdump(interfaces.plugins.PluginInterface): if not lsa: return None - bootkey = '' + bootkey = "" for lk in lsa_keys: - key = cls.get_hive_key(syshive, lsa_base + '\\' + lk) + key = cls.get_hive_key(syshive, lsa_base + "\\" + lk) class_data = None if key: class_data = syshive.read(key.Class + 4, key.ClassLength) if class_data is None: return None - bootkey += class_data.decode('utf-16-le') + bootkey += class_data.decode("utf-16-le") bootkey_str = binascii.unhexlify(bootkey) - bootkey_scrambled = bytes([bootkey_str[cls.bootkey_perm_table[i]] for i in range(len(bootkey_str))]) + bootkey_scrambled = bytes( + [bootkey_str[cls.bootkey_perm_table[i]] for i in range(len(bootkey_str))] + ) return bootkey_scrambled @classmethod - def get_hbootkey(cls, samhive: registry.RegistryHive, bootkey: bytes) -> Optional[bytes]: + def get_hbootkey( + cls, samhive: registry.RegistryHive, bootkey: bytes + ) -> Optional[bytes]: sam_account_path = "SAM\\Domains\\Account" if not bootkey: @@ -120,7 +394,7 @@ class Hashdump(interfaces.plugins.PluginInterface): sam_data = None for v in sam_account_key.get_values(): - if v.get_name() == 'F': + if v.get_name() == "F": sam_data = samhive.read(v.Data + 4, v.DataLength) if not sam_data: return None @@ -133,7 +407,9 @@ class Hashdump(interfaces.plugins.PluginInterface): rc4_key = md5.digest() rc4 = ARC4.new(rc4_key) - hbootkey = rc4.encrypt(sam_data[0x80:0xA0]) # lgtm [py/weak-cryptographic-algorithm] + hbootkey = rc4.encrypt( + sam_data[0x80:0xA0] + ) # lgtm [py/weak-cryptographic-algorithm] return hbootkey elif revision == 3: # AES encrypted @@ -145,18 +421,22 @@ class Hashdump(interfaces.plugins.PluginInterface): return None @classmethod - def decrypt_single_salted_hash(cls, rid, hbootkey: bytes, enc_hash: bytes, _lmntstr, - salt: bytes) -> Optional[bytes]: + def decrypt_single_salted_hash( + cls, rid, hbootkey: bytes, enc_hash: bytes, _lmntstr, salt: bytes + ) -> Optional[bytes]: (des_k1, des_k2) = cls.sid_to_key(rid) des1 = DES.new(des_k1, DES.MODE_ECB) des2 = DES.new(des_k2, DES.MODE_ECB) cipher = AES.new(hbootkey[:16], AES.MODE_CBC, salt) obfkey = cipher.decrypt(enc_hash) - return des1.decrypt(obfkey[:8]) + des2.decrypt(obfkey[8:16]) # lgtm [py/weak-cryptographic-algorithm] + return des1.decrypt(obfkey[:8]) + des2.decrypt( + obfkey[8:16] + ) # lgtm [py/weak-cryptographic-algorithm] @classmethod - def get_user_hashes(cls, user: registry.CM_KEY_NODE, samhive: registry.RegistryHive, - hbootkey: bytes) -> Optional[Tuple[bytes, bytes]]: + def get_user_hashes( + cls, user: registry.CM_KEY_NODE, samhive: registry.RegistryHive, hbootkey: bytes + ) -> Optional[Tuple[bytes, bytes]]: ## Will sometimes find extra user with rid = NAMES, returns empty strings right now try: rid = int(str(user.get_name()), 16) @@ -164,64 +444,88 @@ class Hashdump(interfaces.plugins.PluginInterface): return None sam_data = None for v in user.get_values(): - if v.get_name() == 'V': + if v.get_name() == "V": sam_data = samhive.read(v.Data + 4, v.DataLength) if not sam_data: return None - lm_offset = unpack(" Tuple[bytes, bytes]: """Takes rid of a user and converts it to a key to be used by the DES cipher""" - bytestr1 = [sid & 0xFF, (sid >> 8) & 0xFF, (sid >> 16) & 0xFF, (sid >> 24) & 0xFF] + bytestr1 = [ + sid & 0xFF, + (sid >> 8) & 0xFF, + (sid >> 16) & 0xFF, + (sid >> 24) & 0xFF, + ] bytestr1 += bytestr1[0:3] bytestr2 = [bytestr1[3]] + bytestr1[0:3] bytestr2 += bytestr2[0:3] - return cls.sidbytes_to_key(bytes(bytestr1)), cls.sidbytes_to_key(bytes(bytestr2)) + return cls.sidbytes_to_key(bytes(bytestr1)), cls.sidbytes_to_key( + bytes(bytestr2) + ) @classmethod def sidbytes_to_key(cls, s: bytes) -> bytes: """Builds final DES key from the strings generated in sid_to_key""" - key = [s[0] >> 1, ((s[0] & 0x01) << 6) | (s[1] >> 2), ((s[1] & 0x03) << 5) | (s[2] >> 3), - ((s[2] & 0x07) << 4) | (s[3] >> 4), ((s[3] & 0x0F) << 3) | (s[4] >> 5), - ((s[4] & 0x1F) << 2) | (s[5] >> 6), ((s[5] & 0x3F) << 1) | (s[6] >> 7), s[6] & 0x7F] + key = [ + s[0] >> 1, + ((s[0] & 0x01) << 6) | (s[1] >> 2), + ((s[1] & 0x03) << 5) | (s[2] >> 3), + ((s[2] & 0x07) << 4) | (s[3] >> 4), + ((s[3] & 0x0F) << 3) | (s[4] >> 5), + ((s[4] & 0x1F) << 2) | (s[5] >> 6), + ((s[5] & 0x3F) << 1) | (s[6] >> 7), + s[6] & 0x7F, + ] for i in range(8): - key[i] = (key[i] << 1) + key[i] = key[i] << 1 key[i] = cls.odd_parity[key[i]] return bytes(key) @classmethod - def decrypt_single_hash(cls, rid: int, hbootkey: bytes, enc_hash: bytes, lmntstr: bytes): + def decrypt_single_hash( + cls, rid: int, hbootkey: bytes, enc_hash: bytes, lmntstr: bytes + ): (des_k1, des_k2) = cls.sid_to_key(rid) des1 = DES.new(des_k1, DES.MODE_ECB) des2 = DES.new(des_k2, DES.MODE_ECB) @@ -232,27 +536,33 @@ class Hashdump(interfaces.plugins.PluginInterface): rc4 = ARC4.new(rc4_key) obfkey = rc4.encrypt(enc_hash) # lgtm [py/weak-cryptographic-algorithm] - return des1.decrypt(obfkey[:8]) + des2.decrypt(obfkey[8:]) # lgtm [py/weak-cryptographic-algorithm] + return des1.decrypt(obfkey[:8]) + des2.decrypt( + obfkey[8:] + ) # lgtm [py/weak-cryptographic-algorithm] @classmethod - def get_user_name(cls, user: registry.CM_KEY_NODE, samhive: registry.RegistryHive) -> Optional[bytes]: + def get_user_name( + cls, user: registry.CM_KEY_NODE, samhive: registry.RegistryHive + ) -> Optional[bytes]: value = None for v in user.get_values(): - if v.get_name() == 'V': + if v.get_name() == "V": value = samhive.read(v.Data + 4, v.DataLength) if not value: return None - name_offset = unpack(" len(value): return None - username = value[name_offset:name_offset + name_length] + username = value[name_offset : name_offset + name_length] return username # replaces the dump_hashes method in vol2 - def _generator(self, syshive: registry.RegistryHive, samhive: registry.RegistryHive): + def _generator( + self, syshive: registry.RegistryHive, samhive: registry.RegistryHive + ): if syshive is None: vollog.debug("SYSTEM address is None: No system hive found") if samhive is None: @@ -271,30 +581,34 @@ class Hashdump(interfaces.plugins.PluginInterface): if name is None: name = renderers.NotAvailableValue() else: - name = str(name, 'utf-16-le', errors = 'ignore') + name = str(name, "utf-16-le", errors="ignore") - lmout = str(binascii.hexlify(lmhash or self.empty_lm), 'latin-1') - ntout = str(binascii.hexlify(nthash or self.empty_nt), 'latin-1') + lmout = str(binascii.hexlify(lmhash or self.empty_lm), "latin-1") + ntout = str(binascii.hexlify(nthash or self.empty_nt), "latin-1") rid = int(str(user.get_name()), 16) yield (0, (name, rid, lmout, ntout)) else: vollog.warning("Hbootkey is not valid") def run(self): - offset = self.config.get('offset', None) + offset = self.config.get("offset", None) syshive = None samhive = None - kernel = self.context.modules[self.config['kernel']] - for hive in hivelist.HiveList.list_hives(self.context, - self.config_path, - kernel.layer_name, - kernel.symbol_table_name, - hive_offsets = None if offset is None else [offset]): + kernel = self.context.modules[self.config["kernel"]] + for hive in hivelist.HiveList.list_hives( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + hive_offsets=None if offset is None else [offset], + ): - if hive.get_name().split('\\')[-1].upper() == 'SYSTEM': + if hive.get_name().split("\\")[-1].upper() == "SYSTEM": syshive = hive - if hive.get_name().split('\\')[-1].upper() == 'SAM': + if hive.get_name().split("\\")[-1].upper() == "SAM": samhive = hive - return renderers.TreeGrid([("User", str), ("rid", int), ("lmhash", str), ("nthash", str)], - self._generator(syshive, samhive)) + return renderers.TreeGrid( + [("User", str), ("rid", int), ("lmhash", str), ("nthash", str)], + self._generator(syshive, samhive), + ) diff --git a/volatility3/framework/plugins/windows/info.py b/volatility3/framework/plugins/windows/info.py index 172664aef..aa7837029 100644 --- a/volatility3/framework/plugins/windows/info.py +++ b/volatility3/framework/plugins/windows/info.py @@ -22,15 +22,20 @@ class Info(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), ] @classmethod - def get_depends(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - index: int = 0) -> Iterable[Tuple[int, interfaces.layers.DataLayerInterface]]: + def get_depends( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + index: int = 0, + ) -> Iterable[Tuple[int, interfaces.layers.DataLayerInterface]]: """List the dependencies of a given layer. Args: @@ -52,7 +57,12 @@ class Info(plugins.PluginInterface): pass @classmethod - def get_kernel_module(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str): + def get_kernel_module( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ): """Returns the kernel module based on the layer and symbol_table""" virtual_layer = context.layers[layer_name] if not isinstance(virtual_layer, layers.intel.Intel): @@ -60,12 +70,17 @@ class Info(plugins.PluginInterface): kvo = virtual_layer.config["kernel_virtual_offset"] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) return ntkrnlmp @classmethod - def get_kdbg_structure(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, - symbol_table: str) -> interfaces.objects.ObjectInterface: + def get_kdbg_structure( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + symbol_table: str, + ) -> interfaces.objects.ObjectInterface: """Returns the KDDEBUGGER_DATA64 structure for a kernel""" ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table) @@ -73,23 +88,30 @@ class Info(plugins.PluginInterface): kdbg_offset = ntkrnlmp.get_symbol("KdDebuggerDataBlock").address - kdbg_table_name = intermed.IntermediateSymbolTable.create(context, - interfaces.configuration.path_join( - config_path, 'kdbg'), - "windows", - "kdbg", - native_types = native_types, - class_types = extensions.kdbg.class_types) + kdbg_table_name = intermed.IntermediateSymbolTable.create( + context, + interfaces.configuration.path_join(config_path, "kdbg"), + "windows", + "kdbg", + native_types=native_types, + class_types=extensions.kdbg.class_types, + ) - kdbg = context.object(kdbg_table_name + constants.BANG + "_KDDEBUGGER_DATA64", - offset = ntkrnlmp.offset + kdbg_offset, - layer_name = layer_name) + kdbg = context.object( + kdbg_table_name + constants.BANG + "_KDDEBUGGER_DATA64", + offset=ntkrnlmp.offset + kdbg_offset, + layer_name=layer_name, + ) return kdbg @classmethod - def get_kuser_structure(cls, context: interfaces.context.ContextInterface, layer_name: str, - symbol_table: str) -> interfaces.objects.ObjectInterface: + def get_kuser_structure( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> interfaces.objects.ObjectInterface: """Returns the _KUSER_SHARED_DATA structure for a kernel""" virtual_layer = context.layers[layer_name] if not isinstance(virtual_layer, layers.intel.Intel): @@ -103,28 +125,42 @@ class Info(plugins.PluginInterface): else: kuser_addr = 0xFFFFF78000000000 - kuser = ntkrnlmp.object(object_type = "_KUSER_SHARED_DATA", - layer_name = layer_name, - offset = kuser_addr, - absolute = True) + kuser = ntkrnlmp.object( + object_type="_KUSER_SHARED_DATA", + layer_name=layer_name, + offset=kuser_addr, + absolute=True, + ) return kuser @classmethod - def get_version_structure(cls, context: interfaces.context.ContextInterface, layer_name: str, - symbol_table: str) -> interfaces.objects.ObjectInterface: + def get_version_structure( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> interfaces.objects.ObjectInterface: """Returns the KdVersionBlock information from a kernel""" ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table) vers_offset = ntkrnlmp.get_symbol("KdVersionBlock").address - vers = ntkrnlmp.object(object_type = "_DBGKD_GET_VERSION64", layer_name = layer_name, offset = vers_offset) + vers = ntkrnlmp.object( + object_type="_DBGKD_GET_VERSION64", + layer_name=layer_name, + offset=vers_offset, + ) return vers @classmethod - def get_ntheader_structure(cls, context: interfaces.context.ContextInterface, config_path: str, - layer_name: str) -> interfaces.objects.ObjectInterface: + def get_ntheader_structure( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + ) -> interfaces.objects.ObjectInterface: """Gets the ntheader structure for the kernel of the specified layer""" virtual_layer = context.layers[layer_name] if not isinstance(virtual_layer, layers.intel.Intel): @@ -132,15 +168,19 @@ class Info(plugins.PluginInterface): kvo = virtual_layer.config["kernel_virtual_offset"] - pe_table_name = intermed.IntermediateSymbolTable.create(context, - interfaces.configuration.path_join(config_path, 'pe'), - "windows", - "pe", - class_types = extensions.pe.class_types) + pe_table_name = intermed.IntermediateSymbolTable.create( + context, + interfaces.configuration.path_join(config_path, "pe"), + "windows", + "pe", + class_types=extensions.pe.class_types, + ) - dos_header = context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset = kvo, - layer_name = layer_name) + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=kvo, + layer_name=layer_name, + ) nt_header = dos_header.get_nt_header() @@ -148,20 +188,28 @@ class Info(plugins.PluginInterface): def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name symbol_table = kernel.symbol_table_name layer = self.context.layers[layer_name] table = self.context.symbol_space[symbol_table] - kdbg = self.get_kdbg_structure(self.context, self.config_path, layer_name, symbol_table) + kdbg = self.get_kdbg_structure( + self.context, self.config_path, layer_name, symbol_table + ) yield (0, ("Kernel Base", hex(layer.config["kernel_virtual_offset"]))) yield (0, ("DTB", hex(layer.config["page_map_offset"]))) yield (0, ("Symbols", table.config["isf_url"])) - yield (0, ("Is64Bit", str(symbols.symbol_table_is_64bit(self.context, symbol_table)))) - yield (0, ("IsPAE", str(self.context.layers[layer_name].metadata.get("pae", False)))) + yield ( + 0, + ("Is64Bit", str(symbols.symbol_table_is_64bit(self.context, symbol_table))), + ) + yield ( + 0, + ("IsPAE", str(self.context.layers[layer_name].metadata.get("pae", False))), + ) for i, layer in self.get_depends(self.context, layer_name): yield (0, (layer.name, f"{i} {layer.__class__.__name__}")) @@ -182,28 +230,59 @@ class Info(plugins.PluginInterface): cpu_count_offset = ntkrnlmp.get_symbol("KeNumberProcessors").address - cpu_count = ntkrnlmp.object(object_type = "unsigned int", layer_name = layer_name, offset = cpu_count_offset) + cpu_count = ntkrnlmp.object( + object_type="unsigned int", layer_name=layer_name, offset=cpu_count_offset + ) yield (0, ("KeNumberProcessors", str(cpu_count))) kuser = self.get_kuser_structure(self.context, layer_name, symbol_table) yield (0, ("SystemTime", str(kuser.SystemTime.get_time()))) - yield (0, ("NtSystemRoot", - str(kuser.NtSystemRoot.cast("string", encoding = "utf-16", errors = "replace", max_length = 260)))) + yield ( + 0, + ( + "NtSystemRoot", + str( + kuser.NtSystemRoot.cast( + "string", encoding="utf-16", errors="replace", max_length=260 + ) + ), + ), + ) yield (0, ("NtProductType", str(kuser.NtProductType.description))) yield (0, ("NtMajorVersion", str(kuser.NtMajorVersion))) yield (0, ("NtMinorVersion", str(kuser.NtMinorVersion))) # yield (0, ("KdDebuggerEnabled", "True" if kuser.KdDebuggerEnabled else "False")) # yield (0, ("SafeBootMode", "True" if kuser.SafeBootMode else "False")) - nt_header = self.get_ntheader_structure(self.context, self.config_path, layer_name) + nt_header = self.get_ntheader_structure( + self.context, self.config_path, layer_name + ) - yield (0, ("PE MajorOperatingSystemVersion", str(nt_header.OptionalHeader.MajorOperatingSystemVersion))) - yield (0, ("PE MinorOperatingSystemVersion", str(nt_header.OptionalHeader.MinorOperatingSystemVersion))) + yield ( + 0, + ( + "PE MajorOperatingSystemVersion", + str(nt_header.OptionalHeader.MajorOperatingSystemVersion), + ), + ) + yield ( + 0, + ( + "PE MinorOperatingSystemVersion", + str(nt_header.OptionalHeader.MinorOperatingSystemVersion), + ), + ) yield (0, ("PE Machine", str(nt_header.FileHeader.Machine))) - yield (0, ("PE TimeDateStamp", time.asctime(time.gmtime(nt_header.FileHeader.TimeDateStamp)))) + yield ( + 0, + ( + "PE TimeDateStamp", + time.asctime(time.gmtime(nt_header.FileHeader.TimeDateStamp)), + ), + ) def run(self): diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 40d09b9ea..354ef31c9 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -24,52 +24,109 @@ class JobLinks(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', - description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.BooleanRequirement(name = 'physical', - description = "Display physical offset instead of virtual", - default = False, - optional = True), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.BooleanRequirement( + name="physical", + description="Display physical offset instead of virtual", + default=False, + optional=True, + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), ] def _generator(self) -> Iterator[Tuple]: - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] memory = self.context.layers[kernel.layer_name] - for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, kernel.symbol_table_name): + for proc in pslist.PsList.list_processes( + self.context, kernel.layer_name, kernel.symbol_table_name + ): try: - if not self.config['physical']: + if not self.config["physical"]: offset = proc.vol.offset else: - (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + (_, _, offset, _, _) = list( + memory.mapping(offset=proc.vol.offset, length=0) + )[0] job = proc.Job.dereference() - yield (0, (format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, - proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), - job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, - renderers.NotApplicableValue(), "(Original Process)")) + yield ( + 0, + ( + format_hints.Hex(offset), + utility.array_to_string(proc.ImageFileName), + proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, + proc.get_session_id(), + job.SessionId, + proc.get_is_wow64(), + job.TotalProcesses, + job.ActiveProcesses, + job.TotalTerminatedProcesses, + renderers.NotApplicableValue(), + "(Original Process)", + ), + ) - for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): - if not self.config['physical']: + for entry in job.ProcessListHead.to_list( + proc.vol.type_name, "JobLinks" + ): + if not self.config["physical"]: offset = entry.vol.offset else: - (_, _, offset, _, _) = list(memory.mapping(offset = entry.vol.offset, length = 0))[0] + (_, _, offset, _, _) = list( + memory.mapping(offset=entry.vol.offset, length=0) + )[0] - yield (1, (format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), - entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, - entry.get_is_wow64(), 0, 0, 0, "Yes", - entry.get_peb().ProcessParameters.ImagePathName.get_string())) + yield ( + 1, + ( + format_hints.Hex(offset), + utility.array_to_string(entry.ImageFileName), + entry.UniqueProcessId, + entry.InheritedFromUniqueProcessId, + entry.get_session_id(), + 0, + entry.get_is_wow64(), + 0, + 0, + 0, + "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string(), + ), + ) except (exceptions.InvalidAddressException): continue def run(self) -> renderers.TreeGrid: - offsettype = "(V)" if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT) else "(P)" + offsettype = ( + "(V)" + if not self.config.get("physical", pslist.PsList.PHYSICAL_DEFAULT) + else "(P)" + ) - return renderers.TreeGrid([(f"Offset{offsettype}", format_hints.Hex), ("Name", str), - ("PID", int), ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), - ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str)], - self._generator()) + return renderers.TreeGrid( + [ + (f"Offset{offsettype}", format_hints.Hex), + ("Name", str), + ("PID", int), + ("PPID", int), + ("Sess", int), + ("JobSess", int), + ("Wow64", bool), + ("Total", int), + ("Active", int), + ("Term", int), + ("JobLink", str), + ("Process", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index ba8d049a6..a9b229048 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -5,29 +5,38 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import pslist, vadinfo + class LdrModules(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) - + @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True), - ] + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), + ] def _generator(self, procs): - pe_table_name = intermed.IntermediateSymbolTable.create(self.context, - self.config_path, - "windows", - "pe", - class_types = pe.class_types) + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "windows", "pe", class_types=pe.class_types + ) def filter_function(x: interfaces.objects.ObjectInterface) -> bool: try: @@ -35,25 +44,28 @@ class LdrModules(interfaces.plugins.PluginInterface): except AttributeError: return False - filter_func = filter_function + filter_func = filter_function for proc in procs: proc_layer_name = proc.add_process_layer() # Build dictionaries from different module lists, where the DllBase address is the key and value is the module object - load_order_mod = dict((mod.DllBase, mod) - for mod in proc.load_order_modules()) - init_order_mod = dict((mod.DllBase, mod) - for mod in proc.init_order_modules()) - mem_order_mod = dict((mod.DllBase, mod) - for mod in proc.mem_order_modules()) + load_order_mod = dict( + (mod.DllBase, mod) for mod in proc.load_order_modules() + ) + init_order_mod = dict( + (mod.DllBase, mod) for mod in proc.init_order_modules() + ) + mem_order_mod = dict((mod.DllBase, mod) for mod in proc.mem_order_modules()) # Build dictionary of mapped files, where the VAD start address is the key and value is the file name of the mapped file mapped_files = {} - for vad in vadinfo.VadInfo.list_vads(proc, filter_func = filter_func): - dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset = vad.get_start(), - layer_name = proc_layer_name) + for vad in vadinfo.VadInfo.list_vads(proc, filter_func=filter_func): + dos_header = self.context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=vad.get_start(), + layer_name=proc_layer_name, + ) try: # Filter out VADs that do not start with a MZ header if dos_header.e_magic != 0x5A4D: @@ -69,29 +81,45 @@ class LdrModules(interfaces.plugins.PluginInterface): init_mod = init_order_mod.get(base, None) mem_mod = mem_order_mod.get(base, None) - yield (0, [int(proc.UniqueProcessId), - str(proc.ImageFileName.cast("string", - max_length = proc.ImageFileName.vol.count, - errors = 'replace')), - format_hints.Hex(base), - load_mod is not None, - init_mod is not None, - mem_mod is not None, - mapped_files[base]]) + yield ( + 0, + [ + int(proc.UniqueProcessId), + str( + proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ) + ), + format_hints.Hex(base), + load_mod is not None, + init_mod is not None, + mem_mod is not None, + mapped_files[base], + ], + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] - return renderers.TreeGrid([("Pid", int), - ("Process", str), - ("Base", format_hints.Hex), - ("InLoad", bool), - ("InInit", bool), - ("InMem", bool), - ("MappedPath", str)], - self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("Pid", int), + ("Process", str), + ("Base", format_hints.Hex), + ("InLoad", bool), + ("InInit", bool), + ("InMem", bool), + ("MappedPath", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/lsadump.py b/volatility3/framework/plugins/windows/lsadump.py index edf16416c..8cb239905 100644 --- a/volatility3/framework/plugins/windows/lsadump.py +++ b/volatility3/framework/plugins/windows/lsadump.py @@ -27,10 +27,17 @@ class Lsadump(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'hashdump', component = hashdump.Hashdump, version = (1, 1, 0)), - requirements.VersionRequirement(name = 'hivelist', component = hivelist.HiveList, version = (1, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="hashdump", component=hashdump.Hashdump, version=(1, 1, 0) + ), + requirements.VersionRequirement( + name="hivelist", component=hivelist.HiveList, version=(1, 0, 0) + ), ] @classmethod @@ -46,8 +53,8 @@ class Lsadump(interfaces.plugins.PluginInterface): data = b"" for i in range(60, len(secret), 16): - aes = AES.new(aeskey, AES.MODE_CBC, b'\x00' * 16) - buf = secret[i:i + 16] + aes = AES.new(aeskey, AES.MODE_CBC, b"\x00" * 16) + buf = secret[i : i + 16] if len(buf) < 16: buf += (16 - len(buf)) * "\00" data += aes.decrypt(buf) @@ -55,14 +62,16 @@ class Lsadump(interfaces.plugins.PluginInterface): return data @classmethod - def get_lsa_key(cls, sechive: registry.RegistryHive, bootkey: bytes, vista_or_later: bool) -> Optional[bytes]: + def get_lsa_key( + cls, sechive: registry.RegistryHive, bootkey: bytes, vista_or_later: bool + ) -> Optional[bytes]: if not bootkey: return None if vista_or_later: - policy_key = 'PolEKList' + policy_key = "PolEKList" else: - policy_key = 'PolSecretEncryptionKey' + policy_key = "PolSecretEncryptionKey" enc_reg_key = hashdump.Hashdump.get_hive_key(sechive, "Policy\\" + policy_key) if not enc_reg_key: @@ -84,7 +93,9 @@ class Lsadump(interfaces.plugins.PluginInterface): rc4key = md5.digest() rc4 = ARC4.new(rc4key) - lsa_key = rc4.decrypt(obf_lsa_key[12:60]) # lgtm [py/weak-cryptographic-algorithm] + lsa_key = rc4.decrypt( + obf_lsa_key[12:60] + ) # lgtm [py/weak-cryptographic-algorithm] lsa_key = lsa_key[0x10:0x20] else: lsa_key = cls.decrypt_aes(obf_lsa_key, bootkey) @@ -92,15 +103,25 @@ class Lsadump(interfaces.plugins.PluginInterface): return lsa_key @classmethod - def get_secret_by_name(cls, sechive: registry.RegistryHive, name: str, lsakey: bytes, is_vista_or_later: bool): - enc_secret_key = hashdump.Hashdump.get_hive_key(sechive, "Policy\\Secrets\\" + name + "\\CurrVal") + def get_secret_by_name( + cls, + sechive: registry.RegistryHive, + name: str, + lsakey: bytes, + is_vista_or_later: bool, + ): + enc_secret_key = hashdump.Hashdump.get_hive_key( + sechive, "Policy\\Secrets\\" + name + "\\CurrVal" + ) secret = None if enc_secret_key: enc_secret_value = next(enc_secret_key.get_values()) if enc_secret_value: - enc_secret = sechive.read(enc_secret_value.Data + 4, enc_secret_value.DataLength) + enc_secret = sechive.read( + enc_secret_value.Data + 4, enc_secret_value.DataLength + ) if enc_secret: if not is_vista_or_later: @@ -116,30 +137,35 @@ class Lsadump(interfaces.plugins.PluginInterface): Decrypts a block of data with DES using given key. Note that key can be longer than 7 bytes.""" - decrypted_data = b'' + decrypted_data = b"" j = 0 # key index for i in range(0, len(secret), 8): - enc_block = secret[i:i + 8] - block_key = key[j:j + 7] + enc_block = secret[i : i + 8] + block_key = key[j : j + 7] des_key = hashdump.Hashdump.sidbytes_to_key(block_key) des = DES.new(des_key, DES.MODE_ECB) enc_block = enc_block + b"\x00" * int(abs(8 - len(enc_block)) % 8) - decrypted_data += des.decrypt(enc_block) # lgtm [py/weak-cryptographic-algorithm] + decrypted_data += des.decrypt( + enc_block + ) # lgtm [py/weak-cryptographic-algorithm] j += 7 - if len(key[j:j + 7]) < 7: - j = len(key[j:j + 7]) + if len(key[j : j + 7]) < 7: + j = len(key[j : j + 7]) (dec_data_len,) = unpack(" Iterable[Tuple[interfaces.objects.ObjectInterface, bytes]]: + cls, + context: interfaces.context.ContextInterface, + kernel_layer_name: str, + symbol_table: str, + proc: interfaces.objects.ObjectInterface, + ) -> Iterable[Tuple[interfaces.objects.ObjectInterface, bytes]]: """Generate memory regions for a process that may contain injected code. @@ -87,40 +105,52 @@ class Malfind(interfaces.plugins.PluginInterface): proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: - vollog.debug("Process {}: invalid address {} in layer {}".format(proc_id, excp.invalid_address, - excp.layer_name)) + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) return proc_layer = context.layers[proc_layer_name] for vad in proc.get_vad_root().traverse(): protection_string = vad.get_protection( - vadinfo.VadInfo.protect_values(context, kernel_layer_name, symbol_table), vadinfo.winnt_protections) + vadinfo.VadInfo.protect_values( + context, kernel_layer_name, symbol_table + ), + vadinfo.winnt_protections, + ) write_exec = "EXECUTE" in protection_string and "WRITE" in protection_string # the write/exec check applies to everything if not write_exec: continue - if (vad.get_private_memory() == 1 - and vad.get_tag() == "VadS") or (vad.get_private_memory() == 0 - and protection_string != "PAGE_EXECUTE_WRITECOPY"): + if (vad.get_private_memory() == 1 and vad.get_tag() == "VadS") or ( + vad.get_private_memory() == 0 + and protection_string != "PAGE_EXECUTE_WRITECOPY" + ): if cls.is_vad_empty(proc_layer, vad): continue - data = proc_layer.read(vad.get_start(), 64, pad = True) + data = proc_layer.read(vad.get_start(), 64, pad=True) yield vad, data def _generator(self, procs): # determine if we're on a 32 or 64 bit kernel - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - is_32bit_arch = not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) + is_32bit_arch = not symbols.symbol_table_is_64bit( + self.context, kernel.symbol_table_name + ) for proc in procs: process_name = utility.array_to_string(proc.ImageFileName) - for vad, data in self.list_injections(self.context, kernel.layer_name, kernel.symbol_table_name, proc): + for vad, data in self.list_injections( + self.context, kernel.layer_name, kernel.symbol_table_name, proc + ): # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): @@ -128,37 +158,74 @@ class Malfind(interfaces.plugins.PluginInterface): else: architecture = "intel64" - disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) + disasm = interfaces.renderers.Disassembly( + data, vad.get_start(), architecture + ) file_output = "Disabled" - if self.config['dump']: + if self.config["dump"]: file_output = "Error outputting to file" try: - file_handle = vadinfo.VadInfo.vad_dump(self.context, proc, vad, self.open) + file_handle = vadinfo.VadInfo.vad_dump( + self.context, proc, vad, self.open + ) file_handle.close() file_output = file_handle.preferred_filename except (exceptions.InvalidAddressException, OverflowError) as excp: - vollog.debug("Unable to dump PE with pid {0}.{1:#x}: {2}".format( - proc.UniqueProcessId, vad.get_start(), excp)) + vollog.debug( + "Unable to dump PE with pid {0}.{1:#x}: {2}".format( + proc.UniqueProcessId, vad.get_start(), excp + ) + ) - yield (0, (proc.UniqueProcessId, process_name, format_hints.Hex(vad.get_start()), - format_hints.Hex(vad.get_end()), vad.get_tag(), - vad.get_protection( - vadinfo.VadInfo.protect_values(self.context, kernel.layer_name, - kernel.symbol_table_name), - vadinfo.winnt_protections), vad.get_commit_charge(), vad.get_private_memory(), - file_output, format_hints.HexBytes(data), disasm)) + yield ( + 0, + ( + proc.UniqueProcessId, + process_name, + format_hints.Hex(vad.get_start()), + format_hints.Hex(vad.get_end()), + vad.get_tag(), + vad.get_protection( + vadinfo.VadInfo.protect_values( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + ), + vadinfo.winnt_protections, + ), + vad.get_commit_charge(), + vad.get_private_memory(), + file_output, + format_hints.HexBytes(data), + disasm, + ), + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] - return renderers.TreeGrid([("PID", int), ("Process", str), ("Start VPN", format_hints.Hex), - ("End VPN", format_hints.Hex), ("Tag", str), ("Protection", str), - ("CommitCharge", int), ("PrivateMemory", int), ("File output", str), - ("Hexdump", format_hints.HexBytes), ("Disasm", interfaces.renderers.Disassembly)], - self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Start VPN", format_hints.Hex), + ("End VPN", format_hints.Hex), + ("Tag", str), + ("Protection", str), + ("CommitCharge", int), + ("PrivateMemory", int), + ("File output", str), + ("Hexdump", format_hints.HexBytes), + ("Disasm", interfaces.renderers.Disassembly), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index d064e7d29..ccf6eccea 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -16,6 +16,7 @@ from volatility3.framework.symbols.windows.extensions import mbr vollog = logging.getLogger(__name__) + class MBRScan(interfaces.plugins.PluginInterface): """Scans for and parses potential Master Boot Records (MBRs)""" @@ -23,55 +24,75 @@ class MBRScan(interfaces.plugins.PluginInterface): _version = (1, 0, 0) @classmethod - def get_requirements(cls)-> List[interfaces.configuration.RequirementInterface]: + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.BooleanRequirement(name = 'full', - description ="It analyzes and provides all the information in the partition entry and bootcode hexdump. (It returns a lot of information, so we recommend you render it in CSV.)", - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.BooleanRequirement( + name="full", + description="It analyzes and provides all the information in the partition entry and bootcode hexdump. (It returns a lot of information, so we recommend you render it in CSV.)", + default=False, + optional=True, + ), ] @classmethod - def get_hash(cls, data:bytes) -> str: + def get_hash(cls, data: bytes) -> str: return hashlib.md5(data).hexdigest() def _generator(self) -> Iterator[Tuple]: - kernel = self.context.modules[self.config['kernel']] - physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) - + kernel = self.context.modules[self.config["kernel"]] + physical_layer_name = self.context.layers[kernel.layer_name].config.get( + "memory_layer", None + ) + # Decide of Memory Dump Architecture layer = self.context.layers[physical_layer_name] - architecture = "intel" if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) else "intel64" + architecture = ( + "intel" + if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) + else "intel64" + ) # Read in the Symbol File - symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, - config_path = self.config_path, - sub_path = "windows", - filename = "mbr", - class_types = { - 'PARTITION_TABLE': mbr.PARTITION_TABLE, - 'PARTITION_ENTRY': mbr.PARTITION_ENTRY - }) + symbol_table = intermed.IntermediateSymbolTable.create( + context=self.context, + config_path=self.config_path, + sub_path="windows", + filename="mbr", + class_types={ + "PARTITION_TABLE": mbr.PARTITION_TABLE, + "PARTITION_ENTRY": mbr.PARTITION_ENTRY, + }, + ) partition_table_object = symbol_table + constants.BANG + "PARTITION_TABLE" - + # Define Signature and Data Length mbr_signature = b"\x55\xAA" mbr_length = 0x200 bootcode_length = 0x1B8 # Scan the Layer for Raw Master Boot Record (MBR) and parse the fields - for offset, _value in layer.scan(context = self.context, scanner = scanners.MultiStringScanner(patterns = [mbr_signature])): + for offset, _value in layer.scan( + context=self.context, + scanner=scanners.MultiStringScanner(patterns=[mbr_signature]), + ): try: mbr_start_offset = offset - (mbr_length - len(mbr_signature)) - partition_table = self.context.object(partition_table_object, offset = mbr_start_offset, layer_name = layer.name) + partition_table = self.context.object( + partition_table_object, + offset=mbr_start_offset, + layer_name=layer.name, + ) # Extract only BootCode - full_mbr = layer.read(mbr_start_offset, mbr_length, pad = True) + full_mbr = layer.read(mbr_start_offset, mbr_length, pad=True) bootcode = full_mbr[:bootcode_length] - + all_zeros = None if bootcode: @@ -80,121 +101,163 @@ class MBRScan(interfaces.plugins.PluginInterface): if not all_zeros: partition_entries = [ - partition_table.FirstEntry, partition_table.SecondEntry, - partition_table.ThirdEntry, partition_table.FourthEntry + partition_table.FirstEntry, + partition_table.SecondEntry, + partition_table.ThirdEntry, + partition_table.FourthEntry, ] if not self.config.get("full", True): - yield (0, ( - format_hints.Hex(offset), - partition_table.get_disk_signature(), - self.get_hash(bootcode), - self.get_hash(full_mbr), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - interfaces.renderers.Disassembly(bootcode, 0, architecture) - )) - else: - yield (0, ( - format_hints.Hex(offset), - partition_table.get_disk_signature(), - self.get_hash(bootcode), - self.get_hash(full_mbr), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - interfaces.renderers.Disassembly(bootcode, 0, architecture), - format_hints.HexBytes(bootcode) - )) - - for partition_index, partition_entry_object in enumerate(partition_entries, start=1): - - if not self.config.get("full", True): - yield (1, ( + yield ( + 0, + ( format_hints.Hex(offset), partition_table.get_disk_signature(), self.get_hash(bootcode), self.get_hash(full_mbr), - partition_index, - partition_entry_object.is_bootable(), - partition_entry_object.get_partition_type(), - format_hints.Hex(partition_entry_object.get_size_in_sectors()), - renderers.NotApplicableValue() - )) - else: - yield (1, ( - format_hints.Hex(offset), - partition_table.get_disk_signature(), - self.get_hash(bootcode), - self.get_hash(full_mbr), - partition_index, - partition_entry_object.is_bootable(), - format_hints.Hex(partition_entry_object.get_bootable_flag()), - partition_entry_object.get_partition_type(), - format_hints.Hex(partition_entry_object.PartitionType), - format_hints.Hex(partition_entry_object.get_starting_lba()), - partition_entry_object.get_starting_cylinder(), - partition_entry_object.get_starting_chs(), - partition_entry_object.get_starting_sector(), - partition_entry_object.get_ending_cylinder(), - partition_entry_object.get_ending_chs(), - partition_entry_object.get_ending_sector(), - format_hints.Hex(partition_entry_object.get_size_in_sectors()), renderers.NotApplicableValue(), - renderers.NotApplicableValue() - )) + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + interfaces.renderers.Disassembly( + bootcode, 0, architecture + ), + ), + ) + else: + yield ( + 0, + ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + interfaces.renderers.Disassembly( + bootcode, 0, architecture + ), + format_hints.HexBytes(bootcode), + ), + ) + + for partition_index, partition_entry_object in enumerate( + partition_entries, start=1 + ): + + if not self.config.get("full", True): + yield ( + 1, + ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_index, + partition_entry_object.is_bootable(), + partition_entry_object.get_partition_type(), + format_hints.Hex( + partition_entry_object.get_size_in_sectors() + ), + renderers.NotApplicableValue(), + ), + ) + else: + yield ( + 1, + ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_index, + partition_entry_object.is_bootable(), + format_hints.Hex( + partition_entry_object.get_bootable_flag() + ), + partition_entry_object.get_partition_type(), + format_hints.Hex( + partition_entry_object.PartitionType + ), + format_hints.Hex( + partition_entry_object.get_starting_lba() + ), + partition_entry_object.get_starting_cylinder(), + partition_entry_object.get_starting_chs(), + partition_entry_object.get_starting_sector(), + partition_entry_object.get_ending_cylinder(), + partition_entry_object.get_ending_chs(), + partition_entry_object.get_ending_sector(), + format_hints.Hex( + partition_entry_object.get_size_in_sectors() + ), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + ), + ) else: - vollog.log(constants.LOGLEVEL_VVVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}", + ) continue - + except exceptions.PagedInvalidAddressException as excp: - vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}", + ) continue - - def run(self)-> renderers.TreeGrid: + + def run(self) -> renderers.TreeGrid: if not self.config.get("full", True): - return renderers.TreeGrid([ - ("Potential MBR at Physical Offset", format_hints.Hex), - ("Disk Signature", str), - ("Bootcode MD5", str), - ("Full MBR MD5", str), - ("PartitionIndex", int), - ("Bootable", bool), - ("PartitionType", str), - ("SectorInSize", format_hints.Hex), - ("Disasm", interfaces.renderers.Disassembly) - ], self._generator()) + return renderers.TreeGrid( + [ + ("Potential MBR at Physical Offset", format_hints.Hex), + ("Disk Signature", str), + ("Bootcode MD5", str), + ("Full MBR MD5", str), + ("PartitionIndex", int), + ("Bootable", bool), + ("PartitionType", str), + ("SectorInSize", format_hints.Hex), + ("Disasm", interfaces.renderers.Disassembly), + ], + self._generator(), + ) else: - return renderers.TreeGrid([ - ("Potential MBR at Physical Offset", format_hints.Hex), - ("Disk Signature", str), - ("Bootcode MD5", str), - ("Full MBR MD5", str), - ("PartitionIndex", int), - ("Bootable", bool), - ("BootFlag", format_hints.Hex), - ("PartitionType", str), - ("PartitionTypeRaw", format_hints.Hex), - ("StartingLBA", format_hints.Hex), - ("StartingCylinder", int), - ("StartingCHS", int), - ("StartingSector", int), - ("EndingCylinder", int), - ("EndingCHS", int), - ("EndingSector", int), - ("SectorInSize", format_hints.Hex), - ("Disasm", interfaces.renderers.Disassembly), - ("Bootcode", format_hints.HexBytes) - ], self._generator()) + return renderers.TreeGrid( + [ + ("Potential MBR at Physical Offset", format_hints.Hex), + ("Disk Signature", str), + ("Bootcode MD5", str), + ("Full MBR MD5", str), + ("PartitionIndex", int), + ("Bootable", bool), + ("BootFlag", format_hints.Hex), + ("PartitionType", str), + ("PartitionTypeRaw", format_hints.Hex), + ("StartingLBA", format_hints.Hex), + ("StartingCylinder", int), + ("StartingCHS", int), + ("StartingSector", int), + ("EndingCylinder", int), + ("EndingCHS", int), + ("EndingSector", int), + ("SectorInSize", format_hints.Hex), + ("Disasm", interfaces.renderers.Disassembly), + ("Bootcode", format_hints.HexBytes), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/memmap.py b/volatility3/framework/plugins/windows/memmap.py index 74e5885da..b5c9a211e 100644 --- a/volatility3/framework/plugins/windows/memmap.py +++ b/volatility3/framework/plugins/windows/memmap.py @@ -22,19 +22,27 @@ class Memmap(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.IntRequirement(name = 'pid', - description = "Process ID to include (all other processes are excluded)", - optional = True), - requirements.BooleanRequirement(name = 'dump', - description = "Extract listed memory segments", - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.IntRequirement( + name="pid", + description="Process ID to include (all other processes are excluded)", + optional=True, + ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed memory segments", + default=False, + optional=True, + ), ] - def _generator(self, procs): for proc in procs: pid = "Unknown" @@ -44,47 +52,73 @@ class Memmap(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() proc_layer = self.context.layers[proc_layer_name] except exceptions.InvalidAddressException as excp: - vollog.debug("Process {}: invalid address {} in layer {}".format(pid, excp.invalid_address, - excp.layer_name)) + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + pid, excp.invalid_address, excp.layer_name + ) + ) continue - if self.config['dump']: + if self.config["dump"]: file_handle = self.open(f"pid.{pid}.dmp") else: # Ensure the file isn't actually created if not needed file_handle = contextlib.ExitStack() with file_handle as file_data: file_offset = 0 - for mapval in proc_layer.mapping(0x0, proc_layer.maximum_address, ignore_errors = True): + for mapval in proc_layer.mapping( + 0x0, proc_layer.maximum_address, ignore_errors=True + ): offset, size, mapped_offset, mapped_size, maplayer = mapval file_output = "Disabled" - if self.config['dump']: + if self.config["dump"]: try: - data = proc_layer.read(offset, size, pad = True) + data = proc_layer.read(offset, size, pad=True) file_data.write(data) file_output = file_handle.preferred_filename except exceptions.InvalidAddressException: file_output = "Error outputting to file" - vollog.debug("Unable to write {}'s address {} to {}".format( - proc_layer_name, offset, file_handle.preferred_filename)) + vollog.debug( + "Unable to write {}'s address {} to {}".format( + proc_layer_name, + offset, + file_handle.preferred_filename, + ) + ) - yield (0, (format_hints.Hex(offset), format_hints.Hex(mapped_offset), - format_hints.Hex(mapped_size), - format_hints.Hex(file_offset), file_output)) + yield ( + 0, + ( + format_hints.Hex(offset), + format_hints.Hex(mapped_offset), + format_hints.Hex(mapped_size), + format_hints.Hex(file_offset), + file_output, + ), + ) file_offset += mapped_size offset += mapped_size def run(self): - filter_func = pslist.PsList.create_pid_filter([self.config.get('pid', None)]) - kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter([self.config.get("pid", None)]) + kernel = self.context.modules[self.config["kernel"]] - return renderers.TreeGrid([("Virtual", format_hints.Hex), ("Physical", format_hints.Hex), - ("Size", format_hints.Hex), ("Offset in File", format_hints.Hex), - ("File output", str)], - self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("Virtual", format_hints.Hex), + ("Physical", format_hints.Hex), + ("Size", format_hints.Hex), + ("Offset in File", format_hints.Hex), + ("File output", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index c96fd9522..87416d274 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -23,28 +23,32 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, - version = (2, 0, 0)), + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), ] def _generator(self): - layer = self.context.layers[self.config['primary']] + layer = self.context.layers[self.config["primary"]] # Yara Rule to scan for MFT Header Signatures - rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) + rules = yarascan.YaraScan.process_yara_options( + {"yara_rules": "/FILE0|FILE\*|BAAD/"} + ) # Read in the Symbol File - symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, - config_path = self.config_path, - sub_path = "windows", - filename = "mft", - class_types = { - 'FILE_NAME_ENTRY': mft.MFTFileName, - 'MFT_ENTRY': mft.MFTEntry - }) + symbol_table = intermed.IntermediateSymbolTable.create( + context=self.context, + config_path=self.config_path, + sub_path="windows", + filename="mft", + class_types={"FILE_NAME_ENTRY": mft.MFTFileName, "MFT_ENTRY": mft.MFTEntry}, + ) # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" @@ -54,16 +58,21 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields - for offset, _rule_name, _name, _value in layer.scan(context = self.context, - scanner = yarascan.YaraScanner(rules = rules)): + for offset, _rule_name, _name, _value in layer.scan( + context=self.context, scanner=yarascan.YaraScanner(rules=rules) + ): with contextlib.suppress(exceptions.PagedInvalidAddressException): - mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) + mft_record = self.context.object( + mft_object, offset=offset, layer_name=layer.name + ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - attr_header = self.context.object(header_object, - offset = offset + attr_base_offset, - layer_name = layer.name) + attr_header = self.context.object( + header_object, + offset=offset + attr_base_offset, + layer_name=layer.name, + ) # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType @@ -72,8 +81,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( - attribute_object).relative_child_offset("Attr_Data") + attr_data_offset = ( + offset + + attr_base_offset + + self.context.symbol_space.get_type( + attribute_object + ).relative_child_offset("Attr_Data") + ) # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record @@ -83,8 +97,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType.lookup() == 'STANDARD_INFORMATION': - attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name) + if attr_header.AttrType.lookup() == "STANDARD_INFORMATION": + attr_data = self.context.object( + si_object, offset=attr_data_offset, layer_name=layer.name + ) yield 0, ( format_hints.Hex(attr_data_offset), @@ -102,8 +118,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType.lookup() == 'FILE_NAME': - attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name) + if attr_header.AttrType.lookup() == "FILE_NAME": + attr_data = self.context.object( + fn_object, offset=attr_data_offset, layer_name=layer.name + ) file_name = attr_data.get_full_name() # If we don't have a valid enum, coerce to hex so we can keep the record @@ -112,13 +130,20 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): except ValueError: permissions = hex(attr_data.Flags) - yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(), - mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, - attr_header.AttrType.lookup(), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), file_name) + yield 1, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + mft_flag, + permissions, + attr_header.AttrType.lookup(), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + file_name, + ) # If there's no advancement the loop will never end, so break it now if attr_header.Length == 0: @@ -127,16 +152,18 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length # Get the next attribute - attr_header = self.context.object(header_object, - offset = offset + attr_base_offset, - layer_name = layer.name) + attr_header = self.context.object( + header_object, + offset=offset + attr_base_offset, + layer_name=layer.name, + ) def generate_timeline(self): for row in self._generator(): _depth, row_data = row # Only Output FN Records - if row_data[6] == 'FILE_NAME': + if row_data[6] == "FILE_NAME": filename = row_data[-1] description = f"MFT FILE_NAME entry for {filename}" yield (description, timeliner.TimeLinerType.CREATED, row_data[7]) @@ -145,17 +172,20 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): yield (description, timeliner.TimeLinerType.ACCESSED, row_data[10]) def run(self): - return renderers.TreeGrid([ - ('Offset', format_hints.Hex), - ('Record Type', str), - ('Record Number', int), - ('Link Count', int), - ('MFT Type', str), - ('Permissions', str), - ('Attribute Type', str), - ('Created', datetime.datetime), - ('Modified', datetime.datetime), - ('Updated', datetime.datetime), - ('Accessed', datetime.datetime), - ('Filename', str), - ], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Record Type", str), + ("Record Number", int), + ("Link Count", int), + ("MFT Type", str), + ("Permissions", str), + ("Attribute Type", str), + ("Created", datetime.datetime), + ("Modified", datetime.datetime), + ("Updated", datetime.datetime), + ("Accessed", datetime.datetime), + ("Filename", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index e352c21fe..bbd9a7b4a 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -23,25 +23,35 @@ class ModScan(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'poolscanner', - component = poolscanner.PoolScanner, - version = (1, 0, 0)), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'dlllist', component = dlllist.DllList, version = (2, 0, 0)), - requirements.BooleanRequirement(name = 'dump', - description = "Extract listed modules", - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="poolscanner", component=poolscanner.PoolScanner, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="dlllist", component=dlllist.DllList, version=(2, 0, 0) + ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed modules", + default=False, + optional=True, + ), ] @classmethod - def scan_modules(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def scan_modules( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Scans for modules using the poolscanner module and constraints. Args: @@ -53,19 +63,25 @@ class ModScan(interfaces.plugins.PluginInterface): A list of Driver objects as found from the `layer_name` layer based on Driver pool signatures """ - constraints = poolscanner.PoolScanner.builtin_constraints(symbol_table, [b'MmLd']) + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"MmLd"] + ) - for result in poolscanner.PoolScanner.generate_pool_scan(context, layer_name, symbol_table, constraints): + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): _constraint, mem_object, _header = result yield mem_object @classmethod - def get_session_layers(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - pids: List[int] = None) -> Generator[str, None, None]: + def get_session_layers( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + pids: List[int] = None, + ) -> Generator[str, None, None]: """Build a cache of possible virtual layers, in priority starting with the primary/kernel layer. Then keep one layer per session by cycling through the process list. @@ -82,10 +98,12 @@ class ModScan(interfaces.plugins.PluginInterface): seen_ids: List[interfaces.objects.ObjectInterface] = [] filter_func = pslist.PsList.create_pid_filter(pids or []) - for proc in pslist.PsList.list_processes(context = context, - layer_name = layer_name, - symbol_table = symbol_table, - filter_func = filter_func): + for proc in pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table, + filter_func=filter_func, + ): proc_id = "Unknown" try: proc_id = proc.UniqueProcessId @@ -93,9 +111,11 @@ class ModScan(interfaces.plugins.PluginInterface): # create the session space object in the process' own layer. # not all processes have a valid session pointer. - session_space = context.object(symbol_table + constants.BANG + "_MM_SESSION_SPACE", - layer_name = layer_name, - offset = proc.Session) + session_space = context.object( + symbol_table + constants.BANG + "_MM_SESSION_SPACE", + layer_name=layer_name, + offset=proc.Session, + ) if session_space.SessionId in seen_ids: continue @@ -104,7 +124,9 @@ class ModScan(interfaces.plugins.PluginInterface): vollog.log( constants.LOGLEVEL_VVV, "Process {} does not have a valid Session or a layer could not be constructed for it".format( - proc_id)) + proc_id + ), + ) continue # save the layer if we haven't seen the session yet @@ -112,8 +134,12 @@ class ModScan(interfaces.plugins.PluginInterface): yield proc_layer_name @classmethod - def find_session_layer(cls, context: interfaces.context.ContextInterface, session_layers: Iterable[str], - base_address: int): + def find_session_layer( + cls, + context: interfaces.context.ContextInterface, + session_layers: Iterable[str], + base_address: int, + ): """Given a base address and a list of layer names, find a layer that can access the specified address. @@ -135,16 +161,20 @@ class ModScan(interfaces.plugins.PluginInterface): return None def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - session_layers = list(self.get_session_layers(self.context, kernel.layer_name, kernel.symbol_table_name)) - pe_table_name = intermed.IntermediateSymbolTable.create(self.context, - self.config_path, - "windows", - "pe", - class_types = pe.class_types) + session_layers = list( + self.get_session_layers( + self.context, kernel.layer_name, kernel.symbol_table_name + ) + ) + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "windows", "pe", class_types=pe.class_types + ) - for mod in self.scan_modules(self.context, kernel.layer_name, kernel.symbol_table_name): + for mod in self.scan_modules( + self.context, kernel.layer_name, kernel.symbol_table_name + ): try: BaseDllName = mod.BaseDllName.get_string() @@ -157,23 +187,45 @@ class ModScan(interfaces.plugins.PluginInterface): FullDllName = "" file_output = "Disabled" - if self.config['dump']: + if self.config["dump"]: - session_layer_name = self.find_session_layer(self.context, session_layers, mod.DllBase) + session_layer_name = self.find_session_layer( + self.context, session_layers, mod.DllBase + ) file_output = f"Cannot find a viable session layer for {mod.DllBase:#x}" if session_layer_name: - file_handle = dlllist.DllList.dump_pe(self.context, - pe_table_name, - mod, - self.open, - layer_name = session_layer_name) + file_handle = dlllist.DllList.dump_pe( + self.context, + pe_table_name, + mod, + self.open, + layer_name=session_layer_name, + ) file_output = "Error outputting file" if file_handle: file_output = file_handle.preferred_filename - yield (0, (format_hints.Hex(mod.vol.offset), format_hints.Hex(mod.DllBase), - format_hints.Hex(mod.SizeOfImage), BaseDllName, FullDllName, file_output)) + yield ( + 0, + ( + format_hints.Hex(mod.vol.offset), + format_hints.Hex(mod.DllBase), + format_hints.Hex(mod.SizeOfImage), + BaseDllName, + FullDllName, + file_output, + ), + ) def run(self): - return renderers.TreeGrid([("Offset", format_hints.Hex), ("Base", format_hints.Hex), ("Size", format_hints.Hex), - ("Name", str), ("Path", str), ("File output", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Base", format_hints.Hex), + ("Size", format_hints.Hex), + ("Name", str), + ("Path", str), + ("File output", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 7b488a8eb..eba6d1ce7 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -25,25 +25,34 @@ class Modules(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'dlllist', component = dlllist.DllList, version = (2, 0, 0)), - requirements.BooleanRequirement(name = 'dump', - description = "Extract listed modules", - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="dlllist", component=dlllist.DllList, version=(2, 0, 0) + ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed modules", + default=False, + optional=True, + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] - pe_table_name = intermed.IntermediateSymbolTable.create(self.context, - self.config_path, - "windows", - "pe", - class_types = pe.class_types) + kernel = self.context.modules[self.config["kernel"]] + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "windows", "pe", class_types=pe.class_types + ) - for mod in self.list_modules(self.context, kernel.layer_name, kernel.symbol_table_name): + for mod in self.list_modules( + self.context, kernel.layer_name, kernel.symbol_table_name + ): try: BaseDllName = mod.BaseDllName.get_string() @@ -56,22 +65,35 @@ class Modules(interfaces.plugins.PluginInterface): FullDllName = "" file_output = "Disabled" - if self.config['dump']: - file_handle = dlllist.DllList.dump_pe(self.context, pe_table_name, mod, self.open) + if self.config["dump"]: + file_handle = dlllist.DllList.dump_pe( + self.context, pe_table_name, mod, self.open + ) file_output = "Error outputting file" if file_handle: file_handle.close() file_output = file_handle.preferred_filename - yield (0, (format_hints.Hex(mod.vol.offset), format_hints.Hex(mod.DllBase), - format_hints.Hex(mod.SizeOfImage), BaseDllName, FullDllName, file_output)) + yield ( + 0, + ( + format_hints.Hex(mod.vol.offset), + format_hints.Hex(mod.DllBase), + format_hints.Hex(mod.SizeOfImage), + BaseDllName, + FullDllName, + file_output, + ), + ) @classmethod - def get_session_layers(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - pids: List[int] = None) -> Generator[str, None, None]: + def get_session_layers( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + pids: List[int] = None, + ) -> Generator[str, None, None]: """Build a cache of possible virtual layers, in priority starting with the primary/kernel layer. Then keep one layer per session by cycling through the process list. @@ -88,10 +110,12 @@ class Modules(interfaces.plugins.PluginInterface): seen_ids: List[interfaces.objects.ObjectInterface] = [] filter_func = pslist.PsList.create_pid_filter(pids or []) - for proc in pslist.PsList.list_processes(context = context, - layer_name = layer_name, - symbol_table = symbol_table, - filter_func = filter_func): + for proc in pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table, + filter_func=filter_func, + ): proc_id = "Unknown" try: proc_id = proc.UniqueProcessId @@ -99,9 +123,11 @@ class Modules(interfaces.plugins.PluginInterface): # create the session space object in the process' own layer. # not all processes have a valid session pointer. - session_space = context.object(symbol_table + constants.BANG + "_MM_SESSION_SPACE", - layer_name = layer_name, - offset = proc.Session) + session_space = context.object( + symbol_table + constants.BANG + "_MM_SESSION_SPACE", + layer_name=layer_name, + offset=proc.Session, + ) if session_space.SessionId in seen_ids: continue @@ -110,7 +136,9 @@ class Modules(interfaces.plugins.PluginInterface): vollog.log( constants.LOGLEVEL_VVV, "Process {} does not have a valid Session or a layer could not be constructed for it".format( - proc_id)) + proc_id + ), + ) continue # save the layer if we haven't seen the session yet @@ -118,8 +146,12 @@ class Modules(interfaces.plugins.PluginInterface): yield proc_layer_name @classmethod - def find_session_layer(cls, context: interfaces.context.ContextInterface, session_layers: Iterable[str], - base_address: int): + def find_session_layer( + cls, + context: interfaces.context.ContextInterface, + session_layers: Iterable[str], + base_address: int, + ): """Given a base address and a list of layer names, find a layer that can access the specified address. @@ -141,8 +173,12 @@ class Modules(interfaces.plugins.PluginInterface): return None @classmethod - def list_modules(cls, context: interfaces.context.ContextInterface, layer_name: str, - symbol_table: str) -> Iterable[interfaces.objects.ObjectInterface]: + def list_modules( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the modules in the primary layer. Args: @@ -154,8 +190,8 @@ class Modules(interfaces.plugins.PluginInterface): A list of Modules as retrieved from PsLoadedModuleList """ - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) try: # use this type if its available (starting with windows 10) @@ -166,13 +202,24 @@ class Modules(interfaces.plugins.PluginInterface): type_name = ldr_entry_type.type_name.split(constants.BANG)[1] list_head = ntkrnlmp.get_symbol("PsLoadedModuleList").address - list_entry = ntkrnlmp.object(object_type = "_LIST_ENTRY", offset = list_head) + list_entry = ntkrnlmp.object(object_type="_LIST_ENTRY", offset=list_head) reloff = ldr_entry_type.relative_child_offset("InLoadOrderLinks") - module = ntkrnlmp.object(object_type = type_name, offset = list_entry.vol.offset - reloff, absolute = True) + module = ntkrnlmp.object( + object_type=type_name, offset=list_entry.vol.offset - reloff, absolute=True + ) for mod in module.InLoadOrderLinks: yield mod def run(self): - return renderers.TreeGrid([("Offset", format_hints.Hex), ("Base", format_hints.Hex), ("Size", format_hints.Hex), - ("Name", str), ("Path", str), ("File output", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Base", format_hints.Hex), + ("Size", format_hints.Hex), + ("Name", str), + ("Path", str), + ("File output", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/mutantscan.py b/volatility3/framework/plugins/windows/mutantscan.py index 29e27c9b1..ad6e024d1 100644 --- a/volatility3/framework/plugins/windows/mutantscan.py +++ b/volatility3/framework/plugins/windows/mutantscan.py @@ -18,17 +18,23 @@ class MutantScan(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'poolscanner', plugin = poolscanner.PoolScanner, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), ] @classmethod - def scan_mutants(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def scan_mutants( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Scans for mutants using the poolscanner module and constraints. Args: @@ -40,17 +46,23 @@ class MutantScan(interfaces.plugins.PluginInterface): A list of Mutant objects found by scanning memory for the Mutant pool signatures """ - constraints = poolscanner.PoolScanner.builtin_constraints(symbol_table, [b'Mut\xe1', b'Muta']) + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"Mut\xe1", b"Muta"] + ) - for result in poolscanner.PoolScanner.generate_pool_scan(context, layer_name, symbol_table, constraints): + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): _constraint, mem_object, _header = result yield mem_object def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for mutant in self.scan_mutants(self.context, kernel.layer_name, kernel.symbol_table_name): + for mutant in self.scan_mutants( + self.context, kernel.layer_name, kernel.symbol_table_name + ): try: name = mutant.get_name() @@ -60,7 +72,10 @@ class MutantScan(interfaces.plugins.PluginInterface): yield (0, (format_hints.Hex(mutant.vol.offset), name)) def run(self): - return renderers.TreeGrid([ - ("Offset", format_hints.Hex), - ("Name", str), - ], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Name", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 16301bdd8..5c866bfeb 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -28,24 +28,32 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'poolscanner', - component = poolscanner.PoolScanner, - version = (1, 0, 0)), - requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), - requirements.VersionRequirement(name = 'verinfo', component = verinfo.VerInfo, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="poolscanner", component=poolscanner.PoolScanner, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="info", component=info.Info, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="verinfo", component=verinfo.VerInfo, version=(1, 0, 0) + ), requirements.BooleanRequirement( - name = 'include-corrupt', - description = - "Radically eases result validation. This will show partially overwritten data. WARNING: the results are likely to include garbage and/or corrupt data. Be cautious!", - default = False, - optional = True), + name="include-corrupt", + description="Radically eases result validation. This will show partially overwritten data. WARNING: the results are likely to include garbage and/or corrupt data. Be cautious!", + default=False, + optional=True, + ), ] @staticmethod - def create_netscan_constraints(context: interfaces.context.ContextInterface, - symbol_table: str) -> List[poolscanner.PoolConstraint]: + def create_netscan_constraints( + context: interfaces.context.ContextInterface, symbol_table: str + ) -> List[poolscanner.PoolConstraint]: """Creates a list of Pool Tag Constraints for network objects. Args: @@ -56,33 +64,49 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The list containing the built constraints. """ - tcpl_size = context.symbol_space.get_type(symbol_table + constants.BANG + "_TCP_LISTENER").size - tcpe_size = context.symbol_space.get_type(symbol_table + constants.BANG + "_TCP_ENDPOINT").size - udpa_size = context.symbol_space.get_type(symbol_table + constants.BANG + "_UDP_ENDPOINT").size + tcpl_size = context.symbol_space.get_type( + symbol_table + constants.BANG + "_TCP_LISTENER" + ).size + tcpe_size = context.symbol_space.get_type( + symbol_table + constants.BANG + "_TCP_ENDPOINT" + ).size + udpa_size = context.symbol_space.get_type( + symbol_table + constants.BANG + "_UDP_ENDPOINT" + ).size # ~ vollog.debug("Using pool size constraints: TcpL {}, TcpE {}, UdpA {}".format(tcpl_size, tcpe_size, udpa_size)) return [ # TCP listener - poolscanner.PoolConstraint(b'TcpL', - type_name = symbol_table + constants.BANG + "_TCP_LISTENER", - size = (tcpl_size, None), - page_type = poolscanner.PoolType.NONPAGED | poolscanner.PoolType.FREE), + poolscanner.PoolConstraint( + b"TcpL", + type_name=symbol_table + constants.BANG + "_TCP_LISTENER", + size=(tcpl_size, None), + page_type=poolscanner.PoolType.NONPAGED | poolscanner.PoolType.FREE, + ), # TCP Endpoint - poolscanner.PoolConstraint(b'TcpE', - type_name = symbol_table + constants.BANG + "_TCP_ENDPOINT", - size = (tcpe_size, None), - page_type = poolscanner.PoolType.NONPAGED | poolscanner.PoolType.FREE), + poolscanner.PoolConstraint( + b"TcpE", + type_name=symbol_table + constants.BANG + "_TCP_ENDPOINT", + size=(tcpe_size, None), + page_type=poolscanner.PoolType.NONPAGED | poolscanner.PoolType.FREE, + ), # UDP Endpoint - poolscanner.PoolConstraint(b'UdpA', - type_name = symbol_table + constants.BANG + "_UDP_ENDPOINT", - size = (udpa_size, None), - page_type = poolscanner.PoolType.NONPAGED | poolscanner.PoolType.FREE) + poolscanner.PoolConstraint( + b"UdpA", + type_name=symbol_table + constants.BANG + "_UDP_ENDPOINT", + size=(udpa_size, None), + page_type=poolscanner.PoolType.NONPAGED | poolscanner.PoolType.FREE, + ), ] @classmethod - def determine_tcpip_version(cls, context: interfaces.context.ContextInterface, layer_name: str, - nt_symbol_table: str) -> Tuple[str, Type]: + def determine_tcpip_version( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + nt_symbol_table: str, + ) -> Tuple[str, Type]: """Tries to determine which symbol filename to use for the image's tcpip driver. The logic is partially taken from the info plugin. Args: @@ -102,7 +126,9 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): is_64bit = symbols.symbol_table_is_64bit(context, nt_symbol_table) - is_18363_or_later = versions.is_win10_18363_or_later(context = context, symbol_table = nt_symbol_table) + is_18363_or_later = versions.is_win10_18363_or_later( + context=context, symbol_table=nt_symbol_table + ) if is_64bit: arch = "x64" @@ -119,15 +145,24 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): nt_minor_version = int(kuser.NtMinorVersion) except ValueError: # vers struct exists, but is not an int anymore? - raise NotImplementedError("Kernel Debug Structure version format not supported!") + raise NotImplementedError( + "Kernel Debug Structure version format not supported!" + ) except: # unsure what to raise here. Also, it might be useful to add some kind of fallback, # either to a user-provided version or to another method to determine tcpip.sys's version raise exceptions.VolatilityException( - "Kernel Debug Structure missing VERSION/KUSER structure, unable to determine Windows version!") + "Kernel Debug Structure missing VERSION/KUSER structure, unable to determine Windows version!" + ) - vollog.debug("Determined OS Version: {}.{} {}.{}".format(kuser.NtMajorVersion, kuser.NtMinorVersion, - vers.MajorVersion, vers.MinorVersion)) + vollog.debug( + "Determined OS Version: {}.{} {}.{}".format( + kuser.NtMajorVersion, + kuser.NtMinorVersion, + vers.MajorVersion, + vers.MinorVersion, + ) + ) if nt_major_version == 10 and arch == "x64": # win10 x64 has an additional class type we have to include. @@ -159,7 +194,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): (10, 0, 17134, 0): "netscan-win10-17134-x86", (10, 0, 17763, 0): "netscan-win10-17134-x86", (10, 0, 18362, 0): "netscan-win10-17134-x86", - (10, 0, 18363, 0): "netscan-win10-17134-x86" + (10, 0, 18363, 0): "netscan-win10-17134-x86", } else: version_dict = { @@ -182,11 +217,11 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): (10, 0, 17763, 0): "netscan-win10-17763-x64", (10, 0, 18362, 0): "netscan-win10-18362-x64", (10, 0, 18363, 0): "netscan-win10-18363-x64", - (10, 0, 19041, 0): "netscan-win10-19041-x64" + (10, 0, 19041, 0): "netscan-win10-19041-x64", } # we do not need to check for tcpip's specific FileVersion in every case - tcpip_mod_version = 0 # keep it 0 as a default + tcpip_mod_version = 0 # keep it 0 as a default # special use cases @@ -195,25 +230,44 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # "10.0.18362.1198" with the last part being incremented. However, we can use # os_distinguisher to differentiate between 18362 and 18363 if vers_minor_version == 18362 and is_18363_or_later: - vollog.debug("Detected 18363 data structures: working with 18363 symbol table.") + vollog.debug( + "Detected 18363 data structures: working with 18363 symbol table." + ) vers_minor_version = 18363 # we need to define additional version numbers (which are then found via tcpip.sys's FileVersion header) in case there is # ambiguity _within_ an OS version. If such a version number (last number of the tuple) is defined for the current OS # we need to inspect tcpip.sys's headers to see if we can grab the precise version - if [ (a,b,c,d) for a, b, c, d in version_dict if (a,b,c) == (nt_major_version, nt_minor_version, vers_minor_version) and d != 0]: - vollog.debug("Requiring further version inspection due to OS version by checking tcpip.sys's FileVersion header") + if [ + (a, b, c, d) + for a, b, c, d in version_dict + if (a, b, c) == (nt_major_version, nt_minor_version, vers_minor_version) + and d != 0 + ]: + vollog.debug( + "Requiring further version inspection due to OS version by checking tcpip.sys's FileVersion header" + ) # the following is IntelLayer specific and might need to be adapted to other architectures. - physical_layer_name = context.layers[layer_name].config.get('memory_layer', None) + physical_layer_name = context.layers[layer_name].config.get( + "memory_layer", None + ) if physical_layer_name: - ver = verinfo.VerInfo.find_version_info(context, physical_layer_name, "tcpip.sys") + ver = verinfo.VerInfo.find_version_info( + context, physical_layer_name, "tcpip.sys" + ) if ver: tcpip_mod_version = ver[3] - vollog.debug("Determined tcpip.sys's FileVersion: {}".format(tcpip_mod_version)) + vollog.debug( + "Determined tcpip.sys's FileVersion: {}".format( + tcpip_mod_version + ) + ) else: vollog.debug("Could not determine tcpip.sys's FileVersion.") else: - vollog.debug("Unable to retrieve physical memory layer, skipping FileVersion check.") + vollog.debug( + "Unable to retrieve physical memory layer, skipping FileVersion check." + ) # when determining the symbol file we have to consider the following cases: # the determined version's symbol file is found by intermed.create -> proceed @@ -221,13 +275,19 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # the determined version has no mapped symbol file -> if win10 use latest, otherwise throw exc # windows version cannot be determined -> throw exc - filename = version_dict.get((nt_major_version, nt_minor_version, vers_minor_version, tcpip_mod_version)) + filename = version_dict.get( + (nt_major_version, nt_minor_version, vers_minor_version, tcpip_mod_version) + ) if not filename: # no match on filename means that we possibly have a version newer than those listed here. # try to grab the latest supported version of the current image NT version. If that symbol # version does not work, support has to be added manually. current_versions = [ - (nt_maj, nt_min, vers_min, tcpip_ver) for nt_maj, nt_min, vers_min, tcpip_ver in version_dict if nt_maj == nt_major_version and nt_min == nt_minor_version and tcpip_ver <= tcpip_mod_version + (nt_maj, nt_min, vers_min, tcpip_ver) + for nt_maj, nt_min, vers_min, tcpip_ver in version_dict + if nt_maj == nt_major_version + and nt_min == nt_minor_version + and tcpip_ver <= tcpip_mod_version ] current_versions.sort() @@ -236,19 +296,32 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): filename = version_dict.get(latest_version) - vollog.debug(f"Unable to find exact matching symbol file, going with latest: {filename}") + vollog.debug( + f"Unable to find exact matching symbol file, going with latest: {filename}" + ) else: - raise NotImplementedError("This version of Windows is not supported: {}.{} {}.{}!".format( - nt_major_version, nt_minor_version, vers.MajorVersion, vers_minor_version)) + raise NotImplementedError( + "This version of Windows is not supported: {}.{} {}.{}!".format( + nt_major_version, + nt_minor_version, + vers.MajorVersion, + vers_minor_version, + ) + ) vollog.debug(f"Determined symbol filename: {filename}") return filename, class_types @classmethod - def create_netscan_symbol_table(cls, context: interfaces.context.ContextInterface, layer_name: str, - nt_symbol_table: str, config_path: str) -> str: + def create_netscan_symbol_table( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + nt_symbol_table: str, + config_path: str, + ) -> str: """Creates a symbol table for TCP Listeners and TCP/UDP Endpoints. Args: @@ -268,20 +341,23 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): nt_symbol_table, ) - return intermed.IntermediateSymbolTable.create(context, - config_path, - os.path.join("windows", "netscan"), - symbol_filename, - class_types = class_types, - table_mapping = table_mapping) + return intermed.IntermediateSymbolTable.create( + context, + config_path, + os.path.join("windows", "netscan"), + symbol_filename, + class_types=class_types, + table_mapping=table_mapping, + ) @classmethod - def scan(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - nt_symbol_table: str, - netscan_symbol_table: str) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def scan( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + nt_symbol_table: str, + netscan_symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Scans for network objects using the poolscanner module and constraints. Args: @@ -296,23 +372,32 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): constraints = cls.create_netscan_constraints(context, netscan_symbol_table) - for result in poolscanner.PoolScanner.generate_pool_scan(context, layer_name, nt_symbol_table, constraints): + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, nt_symbol_table, constraints + ): _constraint, mem_object, _header = result yield mem_object def _generator(self, show_corrupt_results: Optional[bool] = None): - """ Generates the network objects for use in rendering. """ + """Generates the network objects for use in rendering.""" - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - netscan_symbol_table = self.create_netscan_symbol_table(self.context, kernel.layer_name, - kernel.symbol_table_name, - self.config_path) + netscan_symbol_table = self.create_netscan_symbol_table( + self.context, kernel.layer_name, kernel.symbol_table_name, self.config_path + ) - for netw_obj in self.scan(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table): + for netw_obj in self.scan( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + netscan_symbol_table, + ): - vollog.debug(f"Found netw obj @ 0x{netw_obj.vol.offset:2x} of assumed type {type(netw_obj)}") + vollog.debug( + f"Found netw obj @ 0x{netw_obj.vol.offset:2x} of assumed type {type(netw_obj)}" + ) # objects passed pool header constraints. check for additional constraints if strict flag is set. if not show_corrupt_results and not netw_obj.is_valid(): continue @@ -322,10 +407,22 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # For UdpA, the state is always blank and the remote end is asterisks for ver, laddr, _ in netw_obj.dual_stack_sockets(): - yield (0, (format_hints.Hex(netw_obj.vol.offset), "UDP" + ver, laddr, netw_obj.Port, "*", 0, "", - netw_obj.get_owner_pid() or renderers.UnreadableValue(), netw_obj.get_owner_procname() - or renderers.UnreadableValue(), netw_obj.get_create_time() - or renderers.UnreadableValue())) + yield ( + 0, + ( + format_hints.Hex(netw_obj.vol.offset), + "UDP" + ver, + laddr, + netw_obj.Port, + "*", + 0, + "", + netw_obj.get_owner_pid() or renderers.UnreadableValue(), + netw_obj.get_owner_procname() + or renderers.UnreadableValue(), + netw_obj.get_create_time() or renderers.UnreadableValue(), + ), + ) elif isinstance(netw_obj, network._TCP_ENDPOINT): vollog.debug(f"Found _TCP_ENDPOINT @ 0x{netw_obj.vol.offset:2x}") @@ -341,11 +438,21 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): except ValueError: state = renderers.UnreadableValue() - yield (0, (format_hints.Hex(netw_obj.vol.offset), proto, netw_obj.get_local_address() - or renderers.UnreadableValue(), netw_obj.LocalPort, netw_obj.get_remote_address() - or renderers.UnreadableValue(), netw_obj.RemotePort, state, netw_obj.get_owner_pid() - or renderers.UnreadableValue(), netw_obj.get_owner_procname() or renderers.UnreadableValue(), - netw_obj.get_create_time() or renderers.UnreadableValue())) + yield ( + 0, + ( + format_hints.Hex(netw_obj.vol.offset), + proto, + netw_obj.get_local_address() or renderers.UnreadableValue(), + netw_obj.LocalPort, + netw_obj.get_remote_address() or renderers.UnreadableValue(), + netw_obj.RemotePort, + state, + netw_obj.get_owner_pid() or renderers.UnreadableValue(), + netw_obj.get_owner_procname() or renderers.UnreadableValue(), + netw_obj.get_create_time() or renderers.UnreadableValue(), + ), + ) # check for isinstance of tcp listener last, because all other objects are inherited from here elif isinstance(netw_obj, network._TCP_LISTENER): @@ -353,13 +460,27 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # For TcpL, the state is always listening and the remote port is zero for ver, laddr, raddr in netw_obj.dual_stack_sockets(): - yield (0, (format_hints.Hex(netw_obj.vol.offset), "TCP" + ver, laddr, netw_obj.Port, raddr, 0, - "LISTENING", netw_obj.get_owner_pid() or renderers.UnreadableValue(), - netw_obj.get_owner_procname() or renderers.UnreadableValue(), netw_obj.get_create_time() - or renderers.UnreadableValue())) + yield ( + 0, + ( + format_hints.Hex(netw_obj.vol.offset), + "TCP" + ver, + laddr, + netw_obj.Port, + raddr, + 0, + "LISTENING", + netw_obj.get_owner_pid() or renderers.UnreadableValue(), + netw_obj.get_owner_procname() + or renderers.UnreadableValue(), + netw_obj.get_create_time() or renderers.UnreadableValue(), + ), + ) else: # this should not happen therefore we log it. - vollog.debug(f"Found network object unsure of its type: {netw_obj} of type {type(netw_obj)}") + vollog.debug( + f"Found network object unsure of its type: {netw_obj} of type {type(netw_obj)}" + ) def generate_timeline(self): for row in self._generator(): @@ -368,28 +489,42 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if not isinstance(row_data[9], datetime.datetime): continue row_data = [ - "N/A" if isinstance(i, renderers.UnreadableValue) or isinstance(i, renderers.UnparsableValue) else i + "N/A" + if isinstance(i, renderers.UnreadableValue) + or isinstance(i, renderers.UnparsableValue) + else i for i in row_data ] - description = "Network connection: Process {} {} Local Address {}:{} " \ - "Remote Address {}:{} State {} Protocol {} ".format(row_data[7], row_data[8], - row_data[2], row_data[3], - row_data[4], row_data[5], - row_data[6], row_data[1]) + description = ( + "Network connection: Process {} {} Local Address {}:{} " + "Remote Address {}:{} State {} Protocol {} ".format( + row_data[7], + row_data[8], + row_data[2], + row_data[3], + row_data[4], + row_data[5], + row_data[6], + row_data[1], + ) + ) yield (description, timeliner.TimeLinerType.CREATED, row_data[9]) def run(self): - show_corrupt_results = self.config.get('include-corrupt', None) + show_corrupt_results = self.config.get("include-corrupt", None) - return renderers.TreeGrid([ - ("Offset", format_hints.Hex), - ("Proto", str), - ("LocalAddr", str), - ("LocalPort", int), - ("ForeignAddr", str), - ("ForeignPort", int), - ("State", str), - ("PID", int), - ("Owner", str), - ("Created", datetime.datetime), - ], self._generator(show_corrupt_results = show_corrupt_results)) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Proto", str), + ("LocalAddr", str), + ("LocalPort", int), + ("ForeignAddr", str), + ("ForeignPort", int), + ("State", str), + ("PID", int), + ("Owner", str), + ("Created", datetime.datetime), + ], + self._generator(show_corrupt_results=show_corrupt_results), + ) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 651ca7696..1685f2a21 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -26,19 +26,32 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'netscan', component = netscan.NetScan, version = (1, 0, 0)), - requirements.VersionRequirement(name = 'modules', component = modules.Modules, version = (1, 0, 0)), - requirements.VersionRequirement(name = 'pdbutil', component = pdbutil.PDBUtility, version = (1, 0, 0)), - requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), - requirements.VersionRequirement(name = 'verinfo', component = verinfo.VerInfo, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="netscan", component=netscan.NetScan, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="modules", component=modules.Modules, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="pdbutil", component=pdbutil.PDBUtility, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="info", component=info.Info, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="verinfo", component=verinfo.VerInfo, version=(1, 0, 0) + ), requirements.BooleanRequirement( - name = 'include-corrupt', - description = - "Radically eases result validation. This will show partially overwritten data. WARNING: the results are likely to include garbage and/or corrupt data. Be cautious!", - default = False, - optional = True), + name="include-corrupt", + description="Radically eases result validation. This will show partially overwritten data. WARNING: the results are likely to include garbage and/or corrupt data. Be cautious!", + default=False, + optional=True, + ), ] @classmethod @@ -57,8 +70,13 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): return value @classmethod - def read_pointer(cls, context: interfaces.context.ContextInterface, layer_name: str, offset: int, - length: int) -> int: + def read_pointer( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + offset: int, + length: int, + ) -> int: """Reads a pointer at a given offset and returns the address it points to. Args: @@ -74,8 +92,13 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): return int.from_bytes(context.layers[layer_name].read(offset, length), "little") @classmethod - def parse_bitmap(cls, context: interfaces.context.ContextInterface, layer_name: str, bitmap_offset: int, - bitmap_size_in_byte: int) -> list: + def parse_bitmap( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + bitmap_offset: int, + bitmap_size_in_byte: int, + ) -> list: """Parses a given bitmap and looks for each occurrence of a 1. Args: @@ -97,14 +120,15 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): return ret @classmethod - def enumerate_structures_by_port(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - net_symbol_table: str, - port: int, - port_pool_addr: int, - proto = "tcp") -> \ - Iterable[interfaces.objects.ObjectInterface]: + def enumerate_structures_by_port( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + net_symbol_table: str, + port: int, + port_pool_addr: int, + proto="tcp", + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all UDP Endpoints and TCP Listeners by parsing UdpPortPool and TcpPortPool. Args: @@ -120,10 +144,14 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ if proto == "tcp": obj_name = net_symbol_table + constants.BANG + "_TCP_LISTENER" - ptr_offset = context.symbol_space.get_type(obj_name).relative_child_offset("Next") + ptr_offset = context.symbol_space.get_type(obj_name).relative_child_offset( + "Next" + ) elif proto == "udp": obj_name = net_symbol_table + constants.BANG + "_UDP_ENDPOINT" - ptr_offset = context.symbol_space.get_type(obj_name).relative_child_offset("Next") + ptr_offset = context.symbol_space.get_type(obj_name).relative_child_offset( + "Next" + ) else: # invalid argument. return @@ -131,12 +159,14 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.debug(f"Current Port: {port}") # the given port serves as a shifted index into the port pool lists list_index = port >> 8 - truncated_port = port & 0xff + truncated_port = port & 0xFF # constructing port_pool object here so callers don't have to - port_pool = context.object(net_symbol_table + constants.BANG + "_INET_PORT_POOL", - layer_name = layer_name, - offset = port_pool_addr) + port_pool = context.object( + net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name=layer_name, + offset=port_pool_addr, + ) # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) inpa = port_pool.PortAssignments[list_index] @@ -153,20 +183,28 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if netw_inside: # if the value is valid, calculate the actual object address by subtracting the offset - curr_obj = context.object(obj_name, layer_name = layer_name, offset = netw_inside - ptr_offset) + curr_obj = context.object( + obj_name, layer_name=layer_name, offset=netw_inside - ptr_offset + ) yield curr_obj # if the same port is used on different interfaces multiple objects are created # those can be found by following the pointer within the object's `Next` field until it is empty while curr_obj.Next: - curr_obj = context.object(obj_name, - layer_name = layer_name, - offset = cls._decode_pointer(curr_obj.Next) - ptr_offset) + curr_obj = context.object( + obj_name, + layer_name=layer_name, + offset=cls._decode_pointer(curr_obj.Next) - ptr_offset, + ) yield curr_obj @classmethod - def get_tcpip_module(cls, context: interfaces.context.ContextInterface, layer_name: str, - nt_symbols: str) -> Optional[interfaces.objects.ObjectInterface]: + def get_tcpip_module( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + nt_symbols: str, + ) -> Optional[interfaces.objects.ObjectInterface]: """Uses `windows.modules` to find tcpip.sys in memory. Args: @@ -184,9 +222,15 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): return None @classmethod - def parse_hashtable(cls, context: interfaces.context.ContextInterface, layer_name: str, ht_offset: int, - ht_length: int, alignment: int, - net_symbol_table: str) -> Generator[interfaces.objects.ObjectInterface, None, None]: + def parse_hashtable( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + ht_offset: int, + ht_length: int, + alignment: int, + net_symbol_table: str, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Parses a hashtable quick and dirty. Args: @@ -201,18 +245,25 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # we are looking for entries whose values are not their own address for index in range(ht_length): current_addr = ht_offset + index * alignment - current_pointer = context.object(net_symbol_table + constants.BANG + "pointer", - layer_name = layer_name, - offset = current_addr) + current_pointer = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=current_addr, + ) # check if addr of pointer is equal to the value pointed to if current_pointer.vol.offset == current_pointer: continue yield current_pointer @classmethod - def parse_partitions(cls, context: interfaces.context.ContextInterface, layer_name: str, net_symbol_table: str, - tcpip_symbol_table: str, - tcpip_module_offset: int) -> Iterable[interfaces.objects.ObjectInterface]: + def parse_partitions( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + net_symbol_table: str, + tcpip_symbol_table: str, + tcpip_module_offset: int, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Parses tcpip.sys's PartitionTable containing established TCP connections. The amount of Partition depends on the value of the symbol `PartitionCount` and correlates with the maximum processor count (refer to Art of Memory Forensics, chapter 11). @@ -234,38 +285,67 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): obj_name = net_symbol_table + constants.BANG + "_TCP_ENDPOINT" # part_table_symbol is the offset within tcpip.sys which contains the address of the partition table itself - part_table_symbol = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + - "PartitionTable").address - part_count_symbol = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + - "PartitionCount").address + part_table_symbol = context.symbol_space.get_symbol( + tcpip_symbol_table + constants.BANG + "PartitionTable" + ).address + part_count_symbol = context.symbol_space.get_symbol( + tcpip_symbol_table + constants.BANG + "PartitionCount" + ).address - part_table_addr = context.object(net_symbol_table + constants.BANG + "pointer", - layer_name = layer_name, - offset = tcpip_module_offset + part_table_symbol) + part_table_addr = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=tcpip_module_offset + part_table_symbol, + ) # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects - part_table = context.object(net_symbol_table + constants.BANG + "_PARTITION_TABLE", - layer_name = layer_name, - offset = part_table_addr) - part_count = int.from_bytes(context.layers[layer_name].read(tcpip_module_offset + part_count_symbol, 1), - "little") + part_table = context.object( + net_symbol_table + constants.BANG + "_PARTITION_TABLE", + layer_name=layer_name, + offset=part_table_addr, + ) + part_count = int.from_bytes( + context.layers[layer_name].read(tcpip_module_offset + part_count_symbol, 1), + "little", + ) part_table.Partitions.count = part_count - vollog.debug("Found TCP connection PartitionTable @ 0x{:x} (partition count: {})".format( - part_table_addr, part_count)) - entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset("ListEntry") + vollog.debug( + "Found TCP connection PartitionTable @ 0x{:x} (partition count: {})".format( + part_table_addr, part_count + ) + ) + entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset( + "ListEntry" + ) for ctr, partition in enumerate(part_table.Partitions): vollog.debug(f"Parsing partition {ctr}") if partition.Endpoints.NumEntries > 0: - for endpoint_entry in cls.parse_hashtable(context, layer_name, partition.Endpoints.Directory, - partition.Endpoints.TableSize, alignment, net_symbol_table): + for endpoint_entry in cls.parse_hashtable( + context, + layer_name, + partition.Endpoints.Directory, + partition.Endpoints.TableSize, + alignment, + net_symbol_table, + ): - endpoint = context.object(obj_name, layer_name = layer_name, offset = endpoint_entry - entry_offset) + endpoint = context.object( + obj_name, + layer_name=layer_name, + offset=endpoint_entry - entry_offset, + ) yield endpoint @classmethod - def create_tcpip_symbol_table(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, - tcpip_module_offset: int, tcpip_module_size: int) -> str: + def create_tcpip_symbol_table( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + tcpip_module_offset: int, + tcpip_module_size: int, + ) -> str: """DEPRECATED: Use PDBUtility.symbol_table_from_pdb instead Creates symbol table for the current image's tcpip.sys driver. @@ -286,13 +366,24 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.debug( "Deprecation: This plugin uses netstat.create_tcpip_symbol_table instead of PDBUtility.symbol_table_from_pdb" ) - return pdbutil.PDBUtility.symbol_table_from_pdb(context, - interfaces.configuration.path_join(config_path, 'tcpip'), - layer_name, "tcpip.pdb", tcpip_module_offset, tcpip_module_size) + return pdbutil.PDBUtility.symbol_table_from_pdb( + context, + interfaces.configuration.path_join(config_path, "tcpip"), + layer_name, + "tcpip.pdb", + tcpip_module_offset, + tcpip_module_size, + ) @classmethod - def find_port_pools(cls, context: interfaces.context.ContextInterface, layer_name: str, net_symbol_table: str, - tcpip_symbol_table: str, tcpip_module_offset: int) -> Tuple[int, int]: + def find_port_pools( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + net_symbol_table: str, + tcpip_symbol_table: str, + tcpip_module_offset: int, + ) -> Tuple[int, int]: """Finds the given image's port pools. Older Windows versions (presumably < Win10 build 14251) use driver symbols called `UdpPortPool` and `TcpPortPool` which point towards the pools. Newer Windows versions use `UdpCompartmentSet` and `TcpCompartmentSet`, which we first have to translate into @@ -311,56 +402,79 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if "UdpPortPool" in context.symbol_space[tcpip_symbol_table].symbols: # older Windows versions - upp_symbol = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + "UdpPortPool").address - upp_addr = context.object(net_symbol_table + constants.BANG + "pointer", - layer_name = layer_name, - offset = tcpip_module_offset + upp_symbol) + upp_symbol = context.symbol_space.get_symbol( + tcpip_symbol_table + constants.BANG + "UdpPortPool" + ).address + upp_addr = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=tcpip_module_offset + upp_symbol, + ) - tpp_symbol = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + "TcpPortPool").address - tpp_addr = context.object(net_symbol_table + constants.BANG + "pointer", - layer_name = layer_name, - offset = tcpip_module_offset + tpp_symbol) + tpp_symbol = context.symbol_space.get_symbol( + tcpip_symbol_table + constants.BANG + "TcpPortPool" + ).address + tpp_addr = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=tcpip_module_offset + tpp_symbol, + ) elif "UdpCompartmentSet" in context.symbol_space[tcpip_symbol_table].symbols: # newer Windows versions since 10.14xxx - ucs = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + "UdpCompartmentSet").address - tcs = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + "TcpCompartmentSet").address + ucs = context.symbol_space.get_symbol( + tcpip_symbol_table + constants.BANG + "UdpCompartmentSet" + ).address + tcs = context.symbol_space.get_symbol( + tcpip_symbol_table + constants.BANG + "TcpCompartmentSet" + ).address - ucs_offset = context.object(net_symbol_table + constants.BANG + "pointer", - layer_name = layer_name, - offset = tcpip_module_offset + ucs) - tcs_offset = context.object(net_symbol_table + constants.BANG + "pointer", - layer_name = layer_name, - offset = tcpip_module_offset + tcs) + ucs_offset = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=tcpip_module_offset + ucs, + ) + tcs_offset = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=tcpip_module_offset + tcs, + ) - ucs_obj = context.object(net_symbol_table + constants.BANG + "_INET_COMPARTMENT_SET", - layer_name = layer_name, - offset = ucs_offset) + ucs_obj = context.object( + net_symbol_table + constants.BANG + "_INET_COMPARTMENT_SET", + layer_name=layer_name, + offset=ucs_offset, + ) upp_addr = ucs_obj.InetCompartment.ProtocolCompartment.PortPool - tcs_obj = context.object(net_symbol_table + constants.BANG + "_INET_COMPARTMENT_SET", - layer_name = layer_name, - offset = tcs_offset) + tcs_obj = context.object( + net_symbol_table + constants.BANG + "_INET_COMPARTMENT_SET", + layer_name=layer_name, + offset=tcs_offset, + ) tpp_addr = tcs_obj.InetCompartment.ProtocolCompartment.PortPool else: # this branch should not be reached. raise exceptions.SymbolError( - "UdpPortPool", tcpip_symbol_table, - f"Neither UdpPortPool nor UdpCompartmentSet found in {tcpip_symbol_table} table") + "UdpPortPool", + tcpip_symbol_table, + f"Neither UdpPortPool nor UdpCompartmentSet found in {tcpip_symbol_table} table", + ) vollog.debug(f"Found PortPools @ 0x{upp_addr:x} (UDP) && 0x{tpp_addr:x} (TCP)") return upp_addr, tpp_addr @classmethod - def list_sockets(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - nt_symbols: str, - net_symbol_table: str, - tcpip_module_offset: int, - tcpip_symbol_table: str) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def list_sockets( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + nt_symbols: str, + net_symbol_table: str, + tcpip_module_offset: int, + tcpip_symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all UDP Endpoints, TCP Listeners and TCP Endpoints in the primary layer that are in tcpip.sys's UdpPortPool, TcpPortPool and TCP Endpoint partition table, respectively. @@ -377,27 +491,49 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ # first, TCP endpoints by parsing the partition table - for endpoint in cls.parse_partitions(context, layer_name, net_symbol_table, tcpip_symbol_table, - tcpip_module_offset): + for endpoint in cls.parse_partitions( + context, + layer_name, + net_symbol_table, + tcpip_symbol_table, + tcpip_module_offset, + ): yield endpoint # then, towards the UDP and TCP port pools # first, find their addresses - upp_addr, tpp_addr = cls.find_port_pools(context, layer_name, net_symbol_table, tcpip_symbol_table, - tcpip_module_offset) + upp_addr, tpp_addr = cls.find_port_pools( + context, + layer_name, + net_symbol_table, + tcpip_symbol_table, + tcpip_module_offset, + ) # create port pool objects at the detected address and parse the port bitmap - upp_obj = context.object(net_symbol_table + constants.BANG + "_INET_PORT_POOL", - layer_name = layer_name, - offset = upp_addr) - udpa_ports = cls.parse_bitmap(context, layer_name, upp_obj.PortBitMap.Buffer, - upp_obj.PortBitMap.SizeOfBitMap // 8) + upp_obj = context.object( + net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name=layer_name, + offset=upp_addr, + ) + udpa_ports = cls.parse_bitmap( + context, + layer_name, + upp_obj.PortBitMap.Buffer, + upp_obj.PortBitMap.SizeOfBitMap // 8, + ) - tpp_obj = context.object(net_symbol_table + constants.BANG + "_INET_PORT_POOL", - layer_name = layer_name, - offset = tpp_addr) - tcpl_ports = cls.parse_bitmap(context, layer_name, tpp_obj.PortBitMap.Buffer, - tpp_obj.PortBitMap.SizeOfBitMap // 8) + tpp_obj = context.object( + net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name=layer_name, + offset=tpp_addr, + ) + tcpl_ports = cls.parse_bitmap( + context, + layer_name, + tpp_obj.PortBitMap.Buffer, + tpp_obj.PortBitMap.SizeOfBitMap // 8, + ) vollog.debug(f"Found TCP Ports: {tcpl_ports}") vollog.debug(f"Found UDP Ports: {udpa_ports}") @@ -406,39 +542,55 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # port value can be 0, which we can skip if not port: continue - for obj in cls.enumerate_structures_by_port(context, layer_name, net_symbol_table, port, tpp_addr, "tcp"): + for obj in cls.enumerate_structures_by_port( + context, layer_name, net_symbol_table, port, tpp_addr, "tcp" + ): yield obj for port in udpa_ports: # same as above, skip port 0 if not port: continue - for obj in cls.enumerate_structures_by_port(context, layer_name, net_symbol_table, port, upp_addr, "udp"): + for obj in cls.enumerate_structures_by_port( + context, layer_name, net_symbol_table, port, upp_addr, "udp" + ): yield obj def _generator(self, show_corrupt_results: Optional[bool] = None): - """ Generates the network objects for use in rendering. """ + """Generates the network objects for use in rendering.""" - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - netscan_symbol_table = netscan.NetScan.create_netscan_symbol_table(self.context, - kernel.layer_name, - kernel.symbol_table_name, - self.config_path) + netscan_symbol_table = netscan.NetScan.create_netscan_symbol_table( + self.context, kernel.layer_name, kernel.symbol_table_name, self.config_path + ) - tcpip_module = self.get_tcpip_module(self.context, kernel.layer_name, kernel.symbol_table_name) + tcpip_module = self.get_tcpip_module( + self.context, kernel.layer_name, kernel.symbol_table_name + ) if not tcpip_module: vollog.error("Unable to locate symbols for the memory image's tcpip module") try: tcpip_symbol_table = pdbutil.PDBUtility.symbol_table_from_pdb( - self.context, interfaces.configuration.path_join(self.config_path, 'tcpip'), - kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) + self.context, + interfaces.configuration.path_join(self.config_path, "tcpip"), + kernel.layer_name, + "tcpip.pdb", + tcpip_module.DllBase, + tcpip_module.SizeOfImage, + ) except exceptions.VolatilityException: vollog.error("Unable to locate symbols for the memory image's tcpip module") - for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, - netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): + for netw_obj in self.list_sockets( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + netscan_symbol_table, + tcpip_module.DllBase, + tcpip_symbol_table, + ): # objects passed pool header constraints. check for additional constraints if strict flag is set. if not show_corrupt_results and not netw_obj.is_valid(): @@ -449,10 +601,22 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # For UdpA, the state is always blank and the remote end is asterisks for ver, laddr, _ in netw_obj.dual_stack_sockets(): - yield (0, (format_hints.Hex(netw_obj.vol.offset), "UDP" + ver, laddr, netw_obj.Port, "*", 0, "", - netw_obj.get_owner_pid() or renderers.UnreadableValue(), netw_obj.get_owner_procname() - or renderers.UnreadableValue(), netw_obj.get_create_time() - or renderers.UnreadableValue())) + yield ( + 0, + ( + format_hints.Hex(netw_obj.vol.offset), + "UDP" + ver, + laddr, + netw_obj.Port, + "*", + 0, + "", + netw_obj.get_owner_pid() or renderers.UnreadableValue(), + netw_obj.get_owner_procname() + or renderers.UnreadableValue(), + netw_obj.get_create_time() or renderers.UnreadableValue(), + ), + ) elif isinstance(netw_obj, network._TCP_ENDPOINT): vollog.debug(f"Found _TCP_ENDPOINT @ 0x{netw_obj.vol.offset:2x}") @@ -461,8 +625,11 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): elif netw_obj.get_address_family() == network.AF_INET6: proto = "TCPv6" else: - vollog.debug("TCP Endpoint @ 0x{:2x} has unknown address family 0x{:x}".format( - netw_obj.vol.offset, netw_obj.get_address_family())) + vollog.debug( + "TCP Endpoint @ 0x{:2x} has unknown address family 0x{:x}".format( + netw_obj.vol.offset, netw_obj.get_address_family() + ) + ) proto = "TCPv?" try: @@ -470,11 +637,21 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): except ValueError: state = renderers.UnreadableValue() - yield (0, (format_hints.Hex(netw_obj.vol.offset), proto, netw_obj.get_local_address() - or renderers.UnreadableValue(), netw_obj.LocalPort, netw_obj.get_remote_address() - or renderers.UnreadableValue(), netw_obj.RemotePort, state, netw_obj.get_owner_pid() - or renderers.UnreadableValue(), netw_obj.get_owner_procname() or renderers.UnreadableValue(), - netw_obj.get_create_time() or renderers.UnreadableValue())) + yield ( + 0, + ( + format_hints.Hex(netw_obj.vol.offset), + proto, + netw_obj.get_local_address() or renderers.UnreadableValue(), + netw_obj.LocalPort, + netw_obj.get_remote_address() or renderers.UnreadableValue(), + netw_obj.RemotePort, + state, + netw_obj.get_owner_pid() or renderers.UnreadableValue(), + netw_obj.get_owner_procname() or renderers.UnreadableValue(), + netw_obj.get_create_time() or renderers.UnreadableValue(), + ), + ) # check for isinstance of tcp listener last, because all other objects are inherited from here elif isinstance(netw_obj, network._TCP_LISTENER): @@ -482,47 +659,79 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # For TcpL, the state is always listening and the remote port is zero for ver, laddr, raddr in netw_obj.dual_stack_sockets(): - yield (0, (format_hints.Hex(netw_obj.vol.offset), "TCP" + ver, laddr, netw_obj.Port, raddr, 0, - "LISTENING", netw_obj.get_owner_pid() or renderers.UnreadableValue(), - netw_obj.get_owner_procname() or renderers.UnreadableValue(), netw_obj.get_create_time() - or renderers.UnreadableValue())) + yield ( + 0, + ( + format_hints.Hex(netw_obj.vol.offset), + "TCP" + ver, + laddr, + netw_obj.Port, + raddr, + 0, + "LISTENING", + netw_obj.get_owner_pid() or renderers.UnreadableValue(), + netw_obj.get_owner_procname() + or renderers.UnreadableValue(), + netw_obj.get_create_time() or renderers.UnreadableValue(), + ), + ) else: # this should not happen therefore we log it. - vollog.debug(f"Found network object unsure of its type: {netw_obj} of type {type(netw_obj)}") + vollog.debug( + f"Found network object unsure of its type: {netw_obj} of type {type(netw_obj)}" + ) def generate_timeline(self): for row in self._generator(): _depth, row_data = row row_dict = {} - row_dict["Offset"], row_dict["Proto"], row_dict["LocalAddr"], row_dict["LocalPort"], \ - row_dict["ForeignAddr"], row_dict["ForeignPort"], row_dict["State"], \ - row_dict["PID"], row_dict["Owner"], row_dict["Created"] = row_data + ( + row_dict["Offset"], + row_dict["Proto"], + row_dict["LocalAddr"], + row_dict["LocalPort"], + row_dict["ForeignAddr"], + row_dict["ForeignPort"], + row_dict["State"], + row_dict["PID"], + row_dict["Owner"], + row_dict["Created"], + ) = row_data # Skip network connections without creation time if not isinstance(row_dict["Created"], datetime.datetime): continue - description = "Network connection: Process {} {} Local Address {}:{} " \ - "Remote Address {}:{} State {} Protocol {} ".format(row_dict["PID"], row_dict["Owner"], - row_dict["LocalAddr"], - row_dict["LocalPort"], - row_dict["ForeignAddr"], - row_dict["ForeignPort"], - row_dict["State"], row_dict["Proto"]) + description = ( + "Network connection: Process {} {} Local Address {}:{} " + "Remote Address {}:{} State {} Protocol {} ".format( + row_dict["PID"], + row_dict["Owner"], + row_dict["LocalAddr"], + row_dict["LocalPort"], + row_dict["ForeignAddr"], + row_dict["ForeignPort"], + row_dict["State"], + row_dict["Proto"], + ) + ) yield (description, timeliner.TimeLinerType.CREATED, row_dict["Created"]) def run(self): - show_corrupt_results = self.config.get('include-corrupt', None) + show_corrupt_results = self.config.get("include-corrupt", None) - return renderers.TreeGrid([ - ("Offset", format_hints.Hex), - ("Proto", str), - ("LocalAddr", str), - ("LocalPort", int), - ("ForeignAddr", str), - ("ForeignPort", int), - ("State", str), - ("PID", int), - ("Owner", str), - ("Created", datetime.datetime), - ], self._generator(show_corrupt_results = show_corrupt_results)) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Proto", str), + ("LocalAddr", str), + ("LocalPort", int), + ("ForeignAddr", str), + ("ForeignPort", int), + ("State", str), + ("PID", int), + ("Owner", str), + ("Created", datetime.datetime), + ], + self._generator(show_corrupt_results=show_corrupt_results), + ) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index c5d60ce03..13c611bf8 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -31,16 +31,18 @@ class PoolConstraint: """Class to maintain tag/size/index/type information about Pool header tags.""" - def __init__(self, - tag: bytes, - type_name: str, - object_type: Optional[str] = None, - page_type: Optional[PoolType] = None, - size: Optional[Tuple[Optional[int], Optional[int]]] = None, - index: Optional[Tuple[Optional[int], Optional[int]]] = None, - alignment: Optional[int] = 1, - skip_type_test: bool = False, - additional_structures: Optional[List[str]] = None) -> None: + def __init__( + self, + tag: bytes, + type_name: str, + object_type: Optional[str] = None, + page_type: Optional[PoolType] = None, + size: Optional[Tuple[Optional[int], Optional[int]]] = None, + index: Optional[Tuple[Optional[int], Optional[int]]] = None, + alignment: Optional[int] = 1, + skip_type_test: bool = False, + additional_structures: Optional[List[str]] = None, + ) -> None: self.tag = tag self.type_name = type_name self.object_type = object_type @@ -53,23 +55,30 @@ class PoolConstraint: class PoolHeaderScanner(interfaces.layers.ScannerInterface): - - def __init__(self, module: interfaces.context.ModuleInterface, constraint_lookup: Dict[bytes, PoolConstraint], - alignment: int): + def __init__( + self, + module: interfaces.context.ModuleInterface, + constraint_lookup: Dict[bytes, PoolConstraint], + alignment: int, + ): super().__init__() self._module = module self._constraint_lookup = constraint_lookup self._alignment = alignment - header_type = self._module.get_type('_POOL_HEADER') - self._header_offset = header_type.relative_child_offset('PoolTag') - self._subscanner = scanners.MultiStringScanner([c for c in constraint_lookup.keys()]) + header_type = self._module.get_type("_POOL_HEADER") + self._header_offset = header_type.relative_child_offset("PoolTag") + self._subscanner = scanners.MultiStringScanner( + [c for c in constraint_lookup.keys()] + ) def __call__(self, data: bytes, data_offset: int): for offset, pattern in self._subscanner(data, data_offset): - header = self._module.object(object_type = "_POOL_HEADER", - offset = offset - self._header_offset, - absolute = True) + header = self._module.object( + object_type="_POOL_HEADER", + offset=offset - self._header_offset, + absolute=True, + ) constraint = self._constraint_lookup[pattern] try: # Size check @@ -87,9 +96,13 @@ class PoolHeaderScanner(interfaces.layers.ScannerInterface): if (constraint.page_type & PoolType.FREE) and header.is_free_pool(): checks_pass = True - elif (constraint.page_type & PoolType.NONPAGED) and header.is_nonpaged_pool(): + elif ( + constraint.page_type & PoolType.NONPAGED + ) and header.is_nonpaged_pool(): checks_pass = True - elif (constraint.page_type & PoolType.PAGED) and header.is_paged_pool(): + elif ( + constraint.page_type & PoolType.PAGED + ) and header.is_paged_pool(): checks_pass = True if not checks_pass: @@ -120,38 +133,59 @@ class PoolScanner(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'handles', plugin = handles.Handles, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="handles", plugin=handles.Handles, version=(1, 0, 0) + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] symbol_table = kernel.symbol_table_name constraints = self.builtin_constraints(symbol_table) - for constraint, mem_object, header in self.generate_pool_scan(self.context, kernel.layer_name, - symbol_table, constraints): + for constraint, mem_object, header in self.generate_pool_scan( + self.context, kernel.layer_name, symbol_table, constraints + ): # generate some type-specific info for sanity checking if constraint.object_type == "Process": - name = mem_object.ImageFileName.cast("string", - max_length = mem_object.ImageFileName.vol.count, - errors = "replace") + name = mem_object.ImageFileName.cast( + "string", + max_length=mem_object.ImageFileName.vol.count, + errors="replace", + ) elif constraint.object_type == "File": try: name = mem_object.FileName.String except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, f"Skipping file at {mem_object.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Skipping file at {mem_object.vol.offset:#x}", + ) continue else: name = renderers.NotApplicableValue() - yield (0, (constraint.type_name, format_hints.Hex(header.vol.offset), header.vol.layer_name, name)) + yield ( + 0, + ( + constraint.type_name, + format_hints.Hex(header.vol.offset), + header.vol.layer_name, + name, + ), + ) @staticmethod - def builtin_constraints(symbol_table: str, tags_filter: List[bytes] = None) -> List[PoolConstraint]: + def builtin_constraints( + symbol_table: str, tags_filter: List[bytes] = None + ) -> List[PoolConstraint]: """Get built-in PoolConstraints given a list of pool tags. The tags_filter is a list of pool tags, and the associated @@ -168,83 +202,109 @@ class PoolScanner(plugins.PluginInterface): builtins = [ # atom tables - PoolConstraint(b'AtmT', - type_name = symbol_table + constants.BANG + "_RTL_ATOM_TABLE", - size = (200, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"AtmT", + type_name=symbol_table + constants.BANG + "_RTL_ATOM_TABLE", + size=(200, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # processes on windows before windows 8 - PoolConstraint(b'Pro\xe3', - type_name = symbol_table + constants.BANG + "_EPROCESS", - object_type = "Process", - size = (600, None), - skip_type_test = True, - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"Pro\xe3", + type_name=symbol_table + constants.BANG + "_EPROCESS", + object_type="Process", + size=(600, None), + skip_type_test=True, + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # processes on windows starting with windows 8 - PoolConstraint(b'Proc', - type_name = symbol_table + constants.BANG + "_EPROCESS", - object_type = "Process", - size = (600, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"Proc", + type_name=symbol_table + constants.BANG + "_EPROCESS", + object_type="Process", + size=(600, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # files on windows before windows 8 - PoolConstraint(b'Fil\xe5', - type_name = symbol_table + constants.BANG + "_FILE_OBJECT", - object_type = "File", - size = (150, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"Fil\xe5", + type_name=symbol_table + constants.BANG + "_FILE_OBJECT", + object_type="File", + size=(150, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # files on windows starting with windows 8 - PoolConstraint(b'File', - type_name = symbol_table + constants.BANG + "_FILE_OBJECT", - object_type = "File", - size = (150, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"File", + type_name=symbol_table + constants.BANG + "_FILE_OBJECT", + object_type="File", + size=(150, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # mutants on windows before windows 8 - PoolConstraint(b'Mut\xe1', - type_name = symbol_table + constants.BANG + "_KMUTANT", - object_type = "Mutant", - size = (64, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"Mut\xe1", + type_name=symbol_table + constants.BANG + "_KMUTANT", + object_type="Mutant", + size=(64, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # mutants on windows starting with windows 8 - PoolConstraint(b'Muta', - type_name = symbol_table + constants.BANG + "_KMUTANT", - object_type = "Mutant", - size = (64, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"Muta", + type_name=symbol_table + constants.BANG + "_KMUTANT", + object_type="Mutant", + size=(64, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # drivers on windows before windows 8 - PoolConstraint(b'Dri\xf6', - type_name = symbol_table + constants.BANG + "_DRIVER_OBJECT", - object_type = "Driver", - size = (248, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, - additional_structures = ["_DRIVER_EXTENSION"]), + PoolConstraint( + b"Dri\xf6", + type_name=symbol_table + constants.BANG + "_DRIVER_OBJECT", + object_type="Driver", + size=(248, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + additional_structures=["_DRIVER_EXTENSION"], + ), # drivers on windows starting with windows 8 - PoolConstraint(b'Driv', - type_name = symbol_table + constants.BANG + "_DRIVER_OBJECT", - object_type = "Driver", - size = (248, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"Driv", + type_name=symbol_table + constants.BANG + "_DRIVER_OBJECT", + object_type="Driver", + size=(248, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # kernel modules - PoolConstraint(b'MmLd', - type_name = symbol_table + constants.BANG + "_LDR_DATA_TABLE_ENTRY", - size = (76, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"MmLd", + type_name=symbol_table + constants.BANG + "_LDR_DATA_TABLE_ENTRY", + size=(76, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # symlinks on windows before windows 8 - PoolConstraint(b'Sym\xe2', - type_name = symbol_table + constants.BANG + "_OBJECT_SYMBOLIC_LINK", - object_type = "SymbolicLink", - size = (72, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"Sym\xe2", + type_name=symbol_table + constants.BANG + "_OBJECT_SYMBOLIC_LINK", + object_type="SymbolicLink", + size=(72, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # symlinks on windows starting with windows 8 - PoolConstraint(b'Symb', - type_name = symbol_table + constants.BANG + "_OBJECT_SYMBOLIC_LINK", - object_type = "SymbolicLink", - size = (72, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + PoolConstraint( + b"Symb", + type_name=symbol_table + constants.BANG + "_OBJECT_SYMBOLIC_LINK", + object_type="SymbolicLink", + size=(72, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + ), # registry hives - PoolConstraint(b'CM10', - type_name = symbol_table + constants.BANG + "_CMHIVE", - size = (800, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, - skip_type_test = True), + PoolConstraint( + b"CM10", + type_name=symbol_table + constants.BANG + "_CMHIVE", + size=(800, None), + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + skip_type_test=True, + ), ] if not tags_filter: @@ -253,13 +313,21 @@ class PoolScanner(plugins.PluginInterface): return [constraint for constraint in builtins if constraint.tag in tags_filter] @classmethod - def generate_pool_scan(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - constraints: List[PoolConstraint]) \ - -> Generator[Tuple[ - PoolConstraint, interfaces.objects.ObjectInterface, interfaces.objects.ObjectInterface], None, None]: + def generate_pool_scan( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + constraints: List[PoolConstraint], + ) -> Generator[ + Tuple[ + PoolConstraint, + interfaces.objects.ObjectInterface, + interfaces.objects.ObjectInterface, + ], + None, + None, + ]: """ Args: @@ -273,9 +341,13 @@ class PoolScanner(plugins.PluginInterface): """ # get the object type map - type_map = handles.Handles.get_type_map(context = context, layer_name = layer_name, symbol_table = symbol_table) + type_map = handles.Handles.get_type_map( + context=context, layer_name=layer_name, symbol_table=symbol_table + ) - cookie = handles.Handles.find_cookie(context = context, layer_name = layer_name, symbol_table = symbol_table) + cookie = handles.Handles.find_cookie( + context=context, layer_name=layer_name, symbol_table=symbol_table + ) is_windows_10 = versions.is_windows_10(context, symbol_table) is_windows_8_or_later = versions.is_windows_8_or_later(context, symbol_table) @@ -285,45 +357,62 @@ class PoolScanner(plugins.PluginInterface): # switch to a non-virtual layer if necessary if not is_windows_10: - scan_layer = context.layers[scan_layer].config['memory_layer'] + scan_layer = context.layers[scan_layer].config["memory_layer"] if symbols.symbol_table_is_64bit(context, symbol_table): alignment = 0x10 else: alignment = 8 - for constraint, header in cls.pool_scan(context, scan_layer, symbol_table, constraints, alignment = alignment): + for constraint, header in cls.pool_scan( + context, scan_layer, symbol_table, constraints, alignment=alignment + ): - mem_objects = header.get_object(constraint = constraint, - use_top_down = is_windows_8_or_later, - native_layer_name = layer_name, - kernel_symbol_table = symbol_table) + mem_objects = header.get_object( + constraint=constraint, + use_top_down=is_windows_8_or_later, + native_layer_name=layer_name, + kernel_symbol_table=symbol_table, + ) for mem_object in mem_objects: if mem_object is None: - vollog.log(constants.LOGLEVEL_VVV, f"Cannot create an instance of {constraint.type_name}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Cannot create an instance of {constraint.type_name}", + ) continue if constraint.object_type is not None and not constraint.skip_type_test: try: - if mem_object.get_object_header().get_object_type(type_map, cookie) != constraint.object_type: + if ( + mem_object.get_object_header().get_object_type( + type_map, cookie + ) + != constraint.object_type + ): continue except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot test instance type check for {constraint.type_name}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Cannot test instance type check for {constraint.type_name}", + ) continue yield constraint, mem_object, header @classmethod - def pool_scan(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - pool_constraints: List[PoolConstraint], - alignment: int = 8, - progress_callback: Optional[constants.ProgressCallback] = None) \ - -> Generator[Tuple[PoolConstraint, interfaces.objects.ObjectInterface], None, None]: + def pool_scan( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + pool_constraints: List[PoolConstraint], + alignment: int = 8, + progress_callback: Optional[constants.ProgressCallback] = None, + ) -> Generator[ + Tuple[PoolConstraint, interfaces.objects.ObjectInterface], None, None + ]: """Returns the _POOL_HEADER object (based on the symbol_table template) after scanning through layer_name returning all headers that match any of the constraints provided. Only one constraint can be provided per @@ -344,11 +433,13 @@ class PoolScanner(plugins.PluginInterface): constraint_lookup: Dict[bytes, PoolConstraint] = {} for constraint in pool_constraints: if constraint.tag in constraint_lookup: - raise ValueError(f"Constraint tag is used for more than one constraint: {repr(constraint.tag)}") + raise ValueError( + f"Constraint tag is used for more than one constraint: {repr(constraint.tag)}" + ) constraint_lookup[constraint.tag] = constraint pool_header_table_name = cls.get_pool_header_table(context, symbol_table) - module = context.module(pool_header_table_name, layer_name, offset = 0) + module = context.module(pool_header_table_name, layer_name, offset=0) # Run the scan locating the offsets of a particular tag layer = context.layers[layer_name] @@ -356,7 +447,9 @@ class PoolScanner(plugins.PluginInterface): yield from layer.scan(context, scanner, progress_callback) @classmethod - def get_pool_header_table(cls, context: interfaces.context.ContextInterface, symbol_table: str) -> str: + def get_pool_header_table( + cls, context: interfaces.context.ContextInterface, symbol_table: str + ) -> str: """Returns the appropriate symbol_table containing a _POOL_HEADER type, even if the original symbol table doesn't contain one. @@ -366,7 +459,9 @@ class PoolScanner(plugins.PluginInterface): """ # Setup the pool header and offset differential try: - context.symbol_space.get_type(symbol_table + constants.BANG + "_POOL_HEADER") + context.symbol_space.get_type( + symbol_table + constants.BANG + "_POOL_HEADER" + ) table_name = symbol_table except exceptions.SymbolError: # We have to manually load a symbol table @@ -387,16 +482,20 @@ class PoolScanner(plugins.PluginInterface): else: class_type = extensions.pool.POOL_HEADER - table_name = intermed.IntermediateSymbolTable.create(context = context, - config_path = configuration.path_join( - context.symbol_space[symbol_table].config_path, - "poolheader"), - sub_path = "windows", - filename = pool_header_json_filename, - table_mapping = {'nt_symbols': symbol_table}, - class_types = {'_POOL_HEADER': class_type}) + table_name = intermed.IntermediateSymbolTable.create( + context=context, + config_path=configuration.path_join( + context.symbol_space[symbol_table].config_path, "poolheader" + ), + sub_path="windows", + filename=pool_header_json_filename, + table_mapping={"nt_symbols": symbol_table}, + class_types={"_POOL_HEADER": class_type}, + ) return table_name def run(self) -> renderers.TreeGrid: - return renderers.TreeGrid([("Tag", str), ("Offset", format_hints.Hex), ("Layer", str), ("Name", str)], - self._generator()) + return renderers.TreeGrid( + [("Tag", str), ("Offset", format_hints.Hex), ("Layer", str), ("Name", str)], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/privileges.py b/volatility3/framework/plugins/windows/privileges.py index 2d48a30f7..7a7087c95 100644 --- a/volatility3/framework/plugins/windows/privileges.py +++ b/volatility3/framework/plugins/windows/privileges.py @@ -24,29 +24,45 @@ class Privs(interfaces.plugins.PluginInterface): # Find the sids json path (or raise error if its not in the plugin directory). for plugin_dir in constants.PLUGINS_PATH: - sids_json_file_name = os.path.join(plugin_dir, os.path.join("windows", "sids_and_privileges.json")) + sids_json_file_name = os.path.join( + plugin_dir, os.path.join("windows", "sids_and_privileges.json") + ) if os.path.exists(sids_json_file_name): break else: - vollog.log(constants.LOGLEVEL_VVV, 'sids_and_privileges.json file is missing plugin error') - raise RuntimeError("The sids_and_privileges.json file missed from you plugin directory") + vollog.log( + constants.LOGLEVEL_VVV, + "sids_and_privileges.json file is missing plugin error", + ) + raise RuntimeError( + "The sids_and_privileges.json file missed from you plugin directory" + ) # Get service sids dictionary (we need only the service sids). - with open(sids_json_file_name, 'r') as file_handle: - temp_json = json.load(file_handle)['privileges'] - self.privilege_info = {int(priv_num): temp_json[priv_num] for priv_num in temp_json} + with open(sids_json_file_name, "r") as file_handle: + temp_json = json.load(file_handle)["privileges"] + self.privilege_info = { + int(priv_num): temp_json[priv_num] for priv_num in temp_json + } @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), ] def _generator(self, procs): @@ -55,14 +71,16 @@ class Privs(interfaces.plugins.PluginInterface): try: process_token = task.Token.dereference().cast("_TOKEN") except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, 'Skip invalid token.') + vollog.log(constants.LOGLEVEL_VVV, "Skip invalid token.") continue for value, present, enabled, default in process_token.privileges(): # Skip privileges whose bit positions cannot be # translated to a privilege name if not self.privilege_info.get(int(value)): - vollog.log(constants.LOGLEVEL_VVV, f'Skip invalid privilege ({value}).') + vollog.log( + constants.LOGLEVEL_VVV, f"Skip invalid privilege ({value})." + ) continue name, desc = self.privilege_info.get(int(value)) @@ -76,23 +94,38 @@ class Privs(interfaces.plugins.PluginInterface): if default: attributes.append("Default") - yield (0, [ - int(task.UniqueProcessId), - objects.utility.array_to_string(task.ImageFileName), - int(value), - str(name), ",".join(attributes), - str(desc) - ]) + yield ( + 0, + [ + int(task.UniqueProcessId), + objects.utility.array_to_string(task.ImageFileName), + int(value), + str(name), + ",".join(attributes), + str(desc), + ], + ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] - return renderers.TreeGrid([("PID", int), ("Process", str), ("Value", int), ("Privilege", str), - ("Attributes", str), ("Description", str)], - self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Value", int), + ("Privilege", str), + ("Attributes", str), + ("Description", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 6023d5f04..7a06af36f 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -27,27 +27,40 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.BooleanRequirement(name = 'physical', - description = 'Display physical offsets instead of virtual', - default = cls.PHYSICAL_DEFAULT, - optional = True), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process ID to include (all other processes are excluded)", - optional = True), - requirements.BooleanRequirement(name = 'dump', - description = "Extract listed processes", - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.BooleanRequirement( + name="physical", + description="Display physical offsets instead of virtual", + default=cls.PHYSICAL_DEFAULT, + optional=True, + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process ID to include (all other processes are excluded)", + optional=True, + ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed processes", + default=False, + optional=True, + ), ] @classmethod def process_dump( - cls, context: interfaces.context.ContextInterface, kernel_table_name: str, pe_table_name: str, - proc: interfaces.objects.ObjectInterface, - open_method: Type[interfaces.plugins.FileHandlerInterface]) -> interfaces.plugins.FileHandlerInterface: + cls, + context: interfaces.context.ContextInterface, + kernel_table_name: str, + pe_table_name: str, + proc: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + ) -> interfaces.plugins.FileHandlerInterface: """Extracts the complete data for a process as a FileHandlerInterface Args: @@ -62,18 +75,24 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ file_handle = None - proc_id = 'Invalid process object' + proc_id = "Invalid process object" try: proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() - peb = context.object(kernel_table_name + constants.BANG + "_PEB", - layer_name = proc_layer_name, - offset = proc.Peb) + peb = context.object( + kernel_table_name + constants.BANG + "_PEB", + layer_name=proc_layer_name, + offset=proc.Peb, + ) - dos_header = context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset = peb.ImageBaseAddress, - layer_name = proc_layer_name) - file_handle = open_method(f"pid.{proc.UniqueProcessId}.{peb.ImageBaseAddress:#x}.dmp") + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=peb.ImageBaseAddress, + layer_name=proc_layer_name, + ) + file_handle = open_method( + f"pid.{proc.UniqueProcessId}.{peb.ImageBaseAddress:#x}.dmp" + ) for offset, data in dos_header.reconstruct(): file_handle.seek(offset) file_handle.write(data) @@ -83,8 +102,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): return file_handle @classmethod - def create_pid_filter(cls, pid_list: List[int] = None, exclude: bool = False) -> Callable[ - [interfaces.objects.ObjectInterface], bool]: + def create_pid_filter( + cls, pid_list: List[int] = None, exclude: bool = False + ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process IDs. @@ -107,8 +127,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): return filter_func @classmethod - def create_name_filter(cls, name_list: List[str] = None, exclude: bool = False) -> Callable[ - [interfaces.objects.ObjectInterface], bool]: + def create_name_filter( + cls, name_list: List[str] = None, exclude: bool = False + ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process names. @@ -124,18 +145,26 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): filter_list = [x for x in name_list if x is not None] if filter_list: if exclude: - filter_func = lambda x: utility.array_to_string(x.ImageFileName) in filter_list + filter_func = ( + lambda x: utility.array_to_string(x.ImageFileName) in filter_list + ) else: - filter_func = lambda x: utility.array_to_string(x.ImageFileName) not in filter_list + filter_func = ( + lambda x: utility.array_to_string(x.ImageFileName) + not in filter_list + ) return filter_func @classmethod - def list_processes(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - filter_func: Callable[[interfaces.objects.ObjectInterface], bool] = lambda _: False) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def list_processes( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: False, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the processes in the primary layer that are in the pid config option. @@ -150,11 +179,11 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ # We only use the object factory to demonstrate how to use one - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) ps_aph_offset = ntkrnlmp.get_symbol("PsActiveProcessHead").address - list_entry = ntkrnlmp.object(object_type = "_LIST_ENTRY", offset = ps_aph_offset) + list_entry = ntkrnlmp.object(object_type="_LIST_ENTRY", offset=ps_aph_offset) # This is example code to demonstrate how to use symbol_space directly, rather than through a module: # @@ -167,55 +196,85 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Note: "nt_symbols!_EPROCESS" could have been used, but would rely on the "nt_symbols" symbol table not already # having been present. Strictly, the value of the requirement should be joined with the BANG character # defined in the constants file - reloff = ntkrnlmp.get_type("_EPROCESS").relative_child_offset("ActiveProcessLinks") - eproc = ntkrnlmp.object(object_type = "_EPROCESS", offset = list_entry.vol.offset - reloff, absolute = True) + reloff = ntkrnlmp.get_type("_EPROCESS").relative_child_offset( + "ActiveProcessLinks" + ) + eproc = ntkrnlmp.object( + object_type="_EPROCESS", + offset=list_entry.vol.offset - reloff, + absolute=True, + ) for proc in eproc.ActiveProcessLinks: if not filter_func(proc): yield proc def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - pe_table_name = intermed.IntermediateSymbolTable.create(self.context, - self.config_path, - "windows", - "pe", - class_types = pe.class_types) + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "windows", "pe", class_types=pe.class_types + ) memory = self.context.layers[kernel.layer_name] if not isinstance(memory, layers.intel.Intel): raise TypeError("Primary layer is not an intel layer") - for proc in self.list_processes(self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func = self.create_pid_filter(self.config.get('pid', None))): + for proc in self.list_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=self.create_pid_filter(self.config.get("pid", None)), + ): - if not self.config.get('physical', self.PHYSICAL_DEFAULT): + if not self.config.get("physical", self.PHYSICAL_DEFAULT): offset = proc.vol.offset else: - (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + (_, _, offset, _, _) = list( + memory.mapping(offset=proc.vol.offset, length=0) + )[0] file_output = "Disabled" try: - if self.config['dump']: - file_handle = self.process_dump(self.context, kernel.symbol_table_name, - pe_table_name, proc, self.open) + if self.config["dump"]: + file_handle = self.process_dump( + self.context, + kernel.symbol_table_name, + pe_table_name, + proc, + self.open, + ) file_output = "Error outputting file" if file_handle: file_handle.close() file_output = str(file_handle.preferred_filename) - yield (0, (proc.UniqueProcessId, proc.InheritedFromUniqueProcessId, - proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, - errors = 'replace'), - format_hints.Hex(offset), proc.ActiveThreads, proc.get_handle_count(), proc.get_session_id(), - proc.get_is_wow64(), proc.get_create_time(), proc.get_exit_time(), file_output)) + yield ( + 0, + ( + proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, + proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ), + format_hints.Hex(offset), + proc.ActiveThreads, + proc.get_handle_count(), + proc.get_session_id(), + proc.get_is_wow64(), + proc.get_create_time(), + proc.get_exit_time(), + file_output, + ), + ) except exceptions.InvalidAddressException: - vollog.info(f"Invalid process found at address: {proc.vol.offset:x}. Skipping") + vollog.info( + f"Invalid process found at address: {proc.vol.offset:x}. Skipping" + ) def generate_timeline(self): for row in self._generator(): @@ -225,10 +284,23 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): yield (description, timeliner.TimeLinerType.MODIFIED, row_data[9]) def run(self): - offsettype = "(V)" if not self.config.get('physical', self.PHYSICAL_DEFAULT) else "(P)" + offsettype = ( + "(V)" if not self.config.get("physical", self.PHYSICAL_DEFAULT) else "(P)" + ) - return renderers.TreeGrid([("PID", int), ("PPID", int), ("ImageFileName", str), - (f"Offset{offsettype}", format_hints.Hex), ("Threads", int), - ("Handles", int), ("SessionId", int), ("Wow64", bool), - ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime), - ("File output", str)], self._generator()) + return renderers.TreeGrid( + [ + ("PID", int), + ("PPID", int), + ("ImageFileName", str), + (f"Offset{offsettype}", format_hints.Hex), + ("Threads", int), + ("Handles", int), + ("SessionId", int), + ("Wow64", bool), + ("CreateTime", datetime.datetime), + ("ExitTime", datetime.datetime), + ("File output", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 00f96ff63..427814d22 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -28,31 +28,47 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process ID to include (all other processes are excluded)", - optional = True), - requirements.BooleanRequirement(name = 'dump', - description = "Extract listed processes", - default = False, - optional = True), - requirements.BooleanRequirement(name = 'physical', - description = "Display physical offset instead of virtual", - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="info", component=info.Info, version=(1, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process ID to include (all other processes are excluded)", + optional=True, + ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed processes", + default=False, + optional=True, + ), + requirements.BooleanRequirement( + name="physical", + description="Display physical offset instead of virtual", + default=False, + optional=True, + ), ] @classmethod - def scan_processes(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - filter_func: Callable[[interfaces.objects.ObjectInterface], bool] = lambda _: False) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def scan_processes( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: False, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Scans for processes using the poolscanner module and constraints. Args: @@ -64,22 +80,27 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): A list of processes found by scanning the `layer_name` layer for process pool signatures """ - constraints = poolscanner.PoolScanner.builtin_constraints(symbol_table, [b'Pro\xe3', b'Proc']) + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"Pro\xe3", b"Proc"] + ) - for result in poolscanner.PoolScanner.generate_pool_scan(context, layer_name, symbol_table, constraints): + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): _constraint, mem_object, _header = result if not filter_func(mem_object): yield mem_object @classmethod - def virtual_process_from_physical(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - proc: interfaces.objects.ObjectInterface) -> \ - Optional[interfaces.objects.ObjectInterface]: - """ Returns a virtual process from a physical addressed one + def virtual_process_from_physical( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + proc: interfaces.objects.ObjectInterface, + ) -> Optional[interfaces.objects.ObjectInterface]: + """Returns a virtual process from a physical addressed one Args: context: The context to retrieve required elements (layers, symbol tables) from @@ -96,10 +117,12 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # If it's WinXP->8.1 we have now a physical process address. # We'll use the first thread to bounce back to the virtual process - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) - tleoffset = ntkrnlmp.get_type("_ETHREAD").relative_child_offset("ThreadListEntry") + tleoffset = ntkrnlmp.get_type("_ETHREAD").relative_child_offset( + "ThreadListEntry" + ) # Start out with the member offset offsets = [tleoffset] @@ -111,24 +134,32 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Now we can try to bounce back for ofs in offsets: - ethread = ntkrnlmp.object(object_type = "_ETHREAD", - offset = proc.ThreadListHead.Flink - ofs, - absolute = True) + ethread = ntkrnlmp.object( + object_type="_ETHREAD", + offset=proc.ThreadListHead.Flink - ofs, + absolute=True, + ) # Ask for the thread's process to get an _EPROCESS with a virtual address layer virtual_process = ethread.owning_process() # Sanity check the bounce. # This compares the original offset with the new one (translated from virtual layer) - (_, _, ph_offset, _, _) = list(context.layers[layer_name].mapping(offset = virtual_process.vol.offset, - length = 0))[0] - if virtual_process and \ - proc.vol.offset == ph_offset: + (_, _, ph_offset, _, _) = list( + context.layers[layer_name].mapping( + offset=virtual_process.vol.offset, length=0 + ) + )[0] + if virtual_process and proc.vol.offset == ph_offset: return virtual_process return None @classmethod - def get_osversion(cls, context: interfaces.context.ContextInterface, layer_name: str, - symbol_table: str) -> Tuple[int, int, int]: + def get_osversion( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Tuple[int, int, int]: """Returns the complete OS version (MAJ,MIN,BUILD) Args: @@ -147,50 +178,74 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): return (nt_major_version, nt_minor_version, build) def _generator(self): - kernel = self.context.modules[self.config['kernel']] - pe_table_name = intermed.IntermediateSymbolTable.create(self.context, - self.config_path, - "windows", - "pe", - class_types = pe.class_types) - memory = self.context.layers[kernel.layer_name] + kernel = self.context.modules[self.config["kernel"]] + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "windows", "pe", class_types=pe.class_types + ) + memory = self.context.layers[kernel.layer_name] if not isinstance(memory, layers.intel.Intel): raise TypeError("Primary layer is not an intel layer") - for proc in self.scan_processes(self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None))): + for proc in self.scan_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=pslist.PsList.create_pid_filter(self.config.get("pid", None)), + ): file_output = "Disabled" - if self.config['dump']: + if self.config["dump"]: # windows 10 objects (maybe others in the future) are already in virtual memory if proc.vol.layer_name == kernel.layer_name: vproc = proc else: - vproc = self.virtual_process_from_physical(self.context, kernel.layer_name, - kernel.symbol_table_name, proc) + vproc = self.virtual_process_from_physical( + self.context, kernel.layer_name, kernel.symbol_table_name, proc + ) - file_handle = pslist.PsList.process_dump(self.context, kernel.symbol_table_name, - pe_table_name, vproc, - self.open) + file_handle = pslist.PsList.process_dump( + self.context, + kernel.symbol_table_name, + pe_table_name, + vproc, + self.open, + ) file_output = "Error outputting file" if file_handle: file_output = file_handle.preferred_filename - if not self.config['physical']: + if not self.config["physical"]: offset = proc.vol.offset else: - (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + (_, _, offset, _, _) = list( + memory.mapping(offset=proc.vol.offset, length=0) + )[0] try: - yield (0, (proc.UniqueProcessId, proc.InheritedFromUniqueProcessId, - proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, - errors = 'replace'), format_hints.Hex(offset), - proc.ActiveThreads, proc.get_handle_count(), proc.get_session_id(), proc.get_is_wow64(), - proc.get_create_time(), proc.get_exit_time(), file_output)) + yield ( + 0, + ( + proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, + proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ), + format_hints.Hex(offset), + proc.ActiveThreads, + proc.get_handle_count(), + proc.get_session_id(), + proc.get_is_wow64(), + proc.get_create_time(), + proc.get_exit_time(), + file_output, + ), + ) except exceptions.InvalidAddressException: - vollog.info(f"Invalid process found at address: {proc.vol.offset:x}. Skipping") + vollog.info( + f"Invalid process found at address: {proc.vol.offset:x}. Skipping" + ) def generate_timeline(self): for row in self._generator(): @@ -200,9 +255,20 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): yield (description, timeliner.TimeLinerType.MODIFIED, row_data[9]) def run(self): - offsettype = "(V)" if not self.config['physical'] else "(P)" - return renderers.TreeGrid([("PID", int), ("PPID", int), ("ImageFileName", str), - (f"Offset{offsettype}", format_hints.Hex), ("Threads", int), - ("Handles", int), ("SessionId", int), ("Wow64", bool), - ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime), - ("File output", str)], self._generator()) + offsettype = "(V)" if not self.config["physical"] else "(P)" + return renderers.TreeGrid( + [ + ("PID", int), + ("PPID", int), + ("ImageFileName", str), + (f"Offset{offsettype}", format_hints.Hex), + ("Threads", int), + ("Handles", int), + ("SessionId", int), + ("Wow64", bool), + ("CreateTime", datetime.datetime), + ("ExitTime", datetime.datetime), + ("File output", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/pstree.py b/volatility3/framework/plugins/windows/pstree.py index fbb883839..88a3697da 100644 --- a/volatility3/framework/plugins/windows/pstree.py +++ b/volatility3/framework/plugins/windows/pstree.py @@ -12,6 +12,7 @@ from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) + class PsTree(interfaces.plugins.PluginInterface): """Plugin for listing processes in a tree based on their parent process ID.""" @@ -27,17 +28,26 @@ class PsTree(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.BooleanRequirement(name = 'physical', - description = 'Display physical offsets instead of virtual', - default = pslist.PsList.PHYSICAL_DEFAULT, - optional = True), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process ID to include (all other processes are excluded)", - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.BooleanRequirement( + name="physical", + description="Display physical offsets instead of virtual", + default=pslist.PsList.PHYSICAL_DEFAULT, + optional=True, + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process ID to include (all other processes are excluded)", + optional=True, + ), ] def find_level(self, pid: objects.Pointer) -> None: @@ -51,22 +61,27 @@ class PsTree(interfaces.plugins.PluginInterface): child_list.add(proc.UniqueProcessId) self._children[proc.InheritedFromUniqueProcessId] = child_list seen.add(proc.InheritedFromUniqueProcessId) - proc, _ = self._processes.get(proc.InheritedFromUniqueProcessId, (None, None)) + proc, _ = self._processes.get( + proc.InheritedFromUniqueProcessId, (None, None) + ) level += 1 self._levels[pid] = level def _generator(self): """Generates the Tree of processes.""" - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, - kernel.symbol_table_name): - if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT): + for proc in pslist.PsList.list_processes( + self.context, kernel.layer_name, kernel.symbol_table_name + ): + if not self.config.get("physical", pslist.PsList.PHYSICAL_DEFAULT): offset = proc.vol.offset else: layer_name = kernel.layer_name memory = self.context.layers[layer_name] - (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + (_, _, offset, _, _) = list( + memory.mapping(offset=proc.vol.offset, length=0) + )[0] self._processes[proc.UniqueProcessId] = proc, offset @@ -75,16 +90,27 @@ class PsTree(interfaces.plugins.PluginInterface): self.find_level(pid) process_pids = set([]) + def yield_processes(pid): if pid in process_pids: vollog.debug(f"Pid cycle: already processed pid {pid}") return process_pids.add(pid) proc, offset = self._processes[pid] - row = (proc.UniqueProcessId, proc.InheritedFromUniqueProcessId, - proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, errors = 'replace'), - format_hints.Hex(offset), proc.ActiveThreads, proc.get_handle_count(), proc.get_session_id(), - proc.get_is_wow64(), proc.get_create_time(), proc.get_exit_time()) + row = ( + proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, + proc.ImageFileName.cast( + "string", max_length=proc.ImageFileName.vol.count, errors="replace" + ), + format_hints.Hex(offset), + proc.ActiveThreads, + proc.get_handle_count(), + proc.get_session_id(), + proc.get_is_wow64(), + proc.get_create_time(), + proc.get_exit_time(), + ) yield (self._levels[pid] - 1, row) for child_pid in self._children.get(pid, []): @@ -95,10 +121,24 @@ class PsTree(interfaces.plugins.PluginInterface): yield from yield_processes(pid) def run(self): - offsettype = "(V)" if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT) else "(P)" + offsettype = ( + "(V)" + if not self.config.get("physical", pslist.PsList.PHYSICAL_DEFAULT) + else "(P)" + ) - return renderers.TreeGrid([("PID", int), ("PPID", int), ("ImageFileName", str), - (f"Offset{offsettype}", format_hints.Hex), ("Threads", int), - ("Handles", int), ("SessionId", int), ("Wow64", bool), - ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime)], - self._generator()) + return renderers.TreeGrid( + [ + ("PID", int), + ("PPID", int), + ("ImageFileName", str), + (f"Offset{offsettype}", format_hints.Hex), + ("Threads", int), + ("Handles", int), + ("SessionId", int), + ("Wow64", bool), + ("CreateTime", datetime.datetime), + ("ExitTime", datetime.datetime), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/registry/hivelist.py b/volatility3/framework/plugins/windows/registry/hivelist.py index ac30560c6..4abcd2f15 100644 --- a/volatility3/framework/plugins/windows/registry/hivelist.py +++ b/volatility3/framework/plugins/windows/registry/hivelist.py @@ -19,13 +19,15 @@ class HiveGenerator: _required_framework_version = (2, 0, 0) - def __init__(self, cmhive, forward = True): + def __init__(self, cmhive, forward=True): self._cmhive = cmhive self._forward = forward self._invalid = None def __iter__(self): - for hive in self._cmhive.HiveList.to_list(self._cmhive.vol.type_name, "HiveList", forward = self._forward): + for hive in self._cmhive.HiveList.to_list( + self._cmhive.vol.type_name, "HiveList", forward=self._forward + ): if not hive.is_valid(): self._invalid = hive.vol.offset return @@ -45,69 +47,102 @@ class HiveList(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.StringRequirement(name = 'filter', - description = "String to filter hive names returned", - optional = True, - default = None), - requirements.PluginRequirement(name = 'hivescan', plugin = hivescan.HiveScan, version = (1, 0, 0)), - requirements.BooleanRequirement(name = 'dump', - description = "Extract listed registry hives", - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.StringRequirement( + name="filter", + description="String to filter hive names returned", + optional=True, + default=None, + ), + requirements.PluginRequirement( + name="hivescan", plugin=hivescan.HiveScan, version=(1, 0, 0) + ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed registry hives", + default=False, + optional=True, + ), ] def _sanitize_hive_name(self, name: str) -> str: - return name.split('\\')[-1].replace(' ', '_').replace('.', '').replace('[', '').replace(']', '') + return ( + name.split("\\")[-1] + .replace(" ", "_") + .replace(".", "") + .replace("[", "") + .replace("]", "") + ) def _generator(self) -> Iterator[Tuple[int, Tuple[int, str]]]: chunk_size = 0x500000 - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for hive_object in self.list_hive_objects(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_string = self.config.get('filter', None)): + for hive_object in self.list_hive_objects( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_string=self.config.get("filter", None), + ): file_output = "Disabled" - if self.config['dump']: + if self.config["dump"]: # Construct the hive hive = next( - self.list_hives(self.context, - self.config_path, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - hive_offsets = [hive_object.vol.offset])) + self.list_hives( + self.context, + self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + hive_offsets=[hive_object.vol.offset], + ) + ) maxaddr = hive.hive.Storage[0].Length hive_name = self._sanitize_hive_name(hive.get_name()) - file_handle = self.open(f'registry.{hive_name}.{hex(hive.hive_offset)}.hive') + file_handle = self.open( + f"registry.{hive_name}.{hex(hive.hive_offset)}.hive" + ) with file_handle as file_data: if hive._base_block: - hive_data = self.context.layers[hive.dependencies[0]].read(hive.hive.BaseBlock, 1 << 12) + hive_data = self.context.layers[hive.dependencies[0]].read( + hive.hive.BaseBlock, 1 << 12 + ) else: - hive_data = '\x00' * (1 << 12) + hive_data = "\x00" * (1 << 12) file_data.write(hive_data) for i in range(0, maxaddr, chunk_size): current_chunk_size = min(chunk_size, maxaddr - i) - data = hive.read(i, current_chunk_size, pad = True) + data = hive.read(i, current_chunk_size, pad=True) file_data.write(data) # if self._progress_callback: # self._progress_callback((i / maxaddr) * 100, 'Writing layer {}'.format(hive_name)) file_output = file_handle.preferred_filename - yield (0, (format_hints.Hex(hive_object.vol.offset), hive_object.get_name() or "", file_output)) + yield ( + 0, + ( + format_hints.Hex(hive_object.vol.offset), + hive_object.get_name() or "", + file_output, + ), + ) @classmethod - def list_hives(cls, - context: interfaces.context.ContextInterface, - base_config_path: str, - layer_name: str, - symbol_table: str, - filter_string: Optional[str] = None, - hive_offsets: List[int] = None) -> Iterable[registry.RegistryHive]: + def list_hives( + cls, + context: interfaces.context.ContextInterface, + base_config_path: str, + layer_name: str, + symbol_table: str, + filter_string: Optional[str] = None, + hive_offsets: List[int] = None, + ) -> Iterable[registry.RegistryHive]: """Walks through a registry, hive by hive returning the constructed registry layer name. @@ -125,34 +160,49 @@ class HiveList(interfaces.plugins.PluginInterface): if hive_offsets is None: try: hive_offsets = [ - hive.vol.offset for hive in cls.list_hive_objects(context, layer_name, symbol_table, filter_string) + hive.vol.offset + for hive in cls.list_hive_objects( + context, layer_name, symbol_table, filter_string + ) ] except ImportError: - vollog.warning("Unable to import windows.hivelist plugin, please provide a hive offset") - raise ValueError("Unable to import windows.hivelist plugin, please provide a hive offset") + vollog.warning( + "Unable to import windows.hivelist plugin, please provide a hive offset" + ) + raise ValueError( + "Unable to import windows.hivelist plugin, please provide a hive offset" + ) for hive_offset in hive_offsets: # Construct the hive - reg_config_path = cls.make_subconfig(context = context, - base_config_path = base_config_path, - hive_offset = hive_offset, - base_layer = layer_name, - nt_symbols = symbol_table) + reg_config_path = cls.make_subconfig( + context=context, + base_config_path=base_config_path, + hive_offset=hive_offset, + base_layer=layer_name, + nt_symbols=symbol_table, + ) try: - hive = registry.RegistryHive(context, reg_config_path, name = 'hive' + hex(hive_offset)) + hive = registry.RegistryHive( + context, reg_config_path, name="hive" + hex(hive_offset) + ) except exceptions.InvalidAddressException: - vollog.warning(f"Couldn't create RegistryHive layer at offset {hex(hive_offset)}, skipping") + vollog.warning( + f"Couldn't create RegistryHive layer at offset {hex(hive_offset)}, skipping" + ) continue context.layers.add_layer(hive) yield hive @classmethod - def list_hive_objects(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - filter_string: str = None) -> Iterator[interfaces.objects.ObjectInterface]: + def list_hive_objects( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + filter_string: str = None, + ) -> Iterator[interfaces.objects.ObjectInterface]: """Lists all the hives in the primary layer. Args: @@ -166,40 +216,59 @@ class HiveList(interfaces.plugins.PluginInterface): """ # We only use the object factory to demonstrate how to use one - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) list_head = ntkrnlmp.get_symbol("CmpHiveListHead").address - list_entry = ntkrnlmp.object(object_type = "_LIST_ENTRY", offset = list_head) + list_entry = ntkrnlmp.object(object_type="_LIST_ENTRY", offset=list_head) reloff = ntkrnlmp.get_type("_CMHIVE").relative_child_offset("HiveList") - cmhive = ntkrnlmp.object(object_type = "_CMHIVE", offset = list_entry.vol.offset - reloff, absolute = True) + cmhive = ntkrnlmp.object( + object_type="_CMHIVE", offset=list_entry.vol.offset - reloff, absolute=True + ) # Run through the list forwards seen = set() - hg = HiveGenerator(cmhive, forward = True) + hg = HiveGenerator(cmhive, forward=True) for hive in hg: if hive.vol.offset in seen: - vollog.debug("Hivelist found an already seen offset {} while " \ - "traversing forwards, this should not occur".format(hex(hive.vol.offset))) + vollog.debug( + "Hivelist found an already seen offset {} while " + "traversing forwards, this should not occur".format( + hex(hive.vol.offset) + ) + ) break seen.add(hive.vol.offset) - if filter_string is None or filter_string.lower() in str(hive.get_name() or "").lower(): + if ( + filter_string is None + or filter_string.lower() in str(hive.get_name() or "").lower() + ): if context.layers[layer_name].is_valid(hive.vol.offset): yield hive forward_invalid = hg.invalid if forward_invalid: - vollog.debug("Hivelist failed traversing the list forwards at {}, traversing backwards".format( - hex(forward_invalid))) - hg = HiveGenerator(cmhive, forward = False) + vollog.debug( + "Hivelist failed traversing the list forwards at {}, traversing backwards".format( + hex(forward_invalid) + ) + ) + hg = HiveGenerator(cmhive, forward=False) for hive in hg: if hive.vol.offset in seen: - vollog.debug("Hivelist found an already seen offset {} while " \ - "traversing backwards, list walking met in the middle".format(hex(hive.vol.offset))) + vollog.debug( + "Hivelist found an already seen offset {} while " + "traversing backwards, list walking met in the middle".format( + hex(hive.vol.offset) + ) + ) break seen.add(hive.vol.offset) - if filter_string is None or filter_string.lower() in str(hive.get_name() or "").lower(): + if ( + filter_string is None + or filter_string.lower() in str(hive.get_name() or "").lower() + ): if context.layers[layer_name].is_valid(hive.vol.offset): yield hive @@ -212,30 +281,53 @@ class HiveList(interfaces.plugins.PluginInterface): # therefore, there must be more 2 or more invalid hives, so the middle of the list is not reachable # by walking the list, so revert to scanning, and walk the list forwards and backwards from each # found hive - vollog.debug("Hivelist failed traversing backwards at {}, a different " \ - "location from forwards, revert to scanning".format(hex(backward_invalid))) - for hive in hivescan.HiveScan.scan_hives(context, layer_name, symbol_table): + vollog.debug( + "Hivelist failed traversing backwards at {}, a different " + "location from forwards, revert to scanning".format( + hex(backward_invalid) + ) + ) + for hive in hivescan.HiveScan.scan_hives( + context, layer_name, symbol_table + ): try: if hive.HiveList.Flink: start_hive_offset = hive.HiveList.Flink - reloff ## Now instantiate the first hive in virtual address space as normal - start_hive = ntkrnlmp.object(object_type = "_CMHIVE", - offset = start_hive_offset, - absolute = True) + start_hive = ntkrnlmp.object( + object_type="_CMHIVE", + offset=start_hive_offset, + absolute=True, + ) for forward in (True, False): - for linked_hive in start_hive.HiveList.to_list(hive.vol.type_name, "HiveList", forward): - if not linked_hive.is_valid() or linked_hive.vol.offset in seen: + for linked_hive in start_hive.HiveList.to_list( + hive.vol.type_name, "HiveList", forward + ): + if ( + not linked_hive.is_valid() + or linked_hive.vol.offset in seen + ): continue seen.add(linked_hive.vol.offset) - if filter_string is None or filter_string.lower() in str(linked_hive.get_name() - or "").lower(): - if context.layers[layer_name].is_valid(linked_hive.vol.offset): + if ( + filter_string is None + or filter_string.lower() + in str(linked_hive.get_name() or "").lower() + ): + if context.layers[layer_name].is_valid( + linked_hive.vol.offset + ): yield linked_hive except exceptions.InvalidAddressException: - vollog.debug("InvalidAddressException when traversing hive {} found from scan, skipping".format( - hex(hive.vol.offset))) + vollog.debug( + "InvalidAddressException when traversing hive {} found from scan, skipping".format( + hex(hive.vol.offset) + ) + ) def run(self) -> renderers.TreeGrid: - return renderers.TreeGrid([("Offset", format_hints.Hex), ("FileFullPath", str), ("File output", str)], - self._generator()) + return renderers.TreeGrid( + [("Offset", format_hints.Hex), ("FileFullPath", str), ("File output", str)], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/registry/hivescan.py b/volatility3/framework/plugins/windows/registry/hivescan.py index ab15e56ac..c3a52e303 100644 --- a/volatility3/framework/plugins/windows/registry/hivescan.py +++ b/volatility3/framework/plugins/windows/registry/hivescan.py @@ -21,18 +21,26 @@ class HiveScan(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'poolscanner', plugin = poolscanner.PoolScanner, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'bigpools', plugin = bigpools.BigPools, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="bigpools", plugin=bigpools.BigPools, version=(1, 0, 0) + ), ] @classmethod - def scan_hives(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def scan_hives( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Scans for hives using the poolscanner module and constraints or bigpools module with tag. Args: @@ -45,32 +53,41 @@ class HiveScan(interfaces.plugins.PluginInterface): """ is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) - is_windows_8_1_or_later = versions.is_windows_8_1_or_later(context = context, symbol_table = symbol_table) + is_windows_8_1_or_later = versions.is_windows_8_1_or_later( + context=context, symbol_table=symbol_table + ) if is_windows_8_1_or_later and is_64bit: - kvo = context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) - for pool in bigpools.BigPools.list_big_pools(context, - layer_name = layer_name, - symbol_table = symbol_table, - tags = ["CM10"]): - cmhive = ntkrnlmp.object(object_type = "_CMHIVE", offset = pool.Va, absolute = True) + for pool in bigpools.BigPools.list_big_pools( + context, layer_name=layer_name, symbol_table=symbol_table, tags=["CM10"] + ): + cmhive = ntkrnlmp.object( + object_type="_CMHIVE", offset=pool.Va, absolute=True + ) yield cmhive else: - constraints = poolscanner.PoolScanner.builtin_constraints(symbol_table, [b'CM10']) + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"CM10"] + ) - for result in poolscanner.PoolScanner.generate_pool_scan(context, layer_name, symbol_table, constraints): + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): _constraint, mem_object, _header = result yield mem_object def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for hive in self.scan_hives(self.context, kernel.layer_name, kernel.symbol_table_name): + for hive in self.scan_hives( + self.context, kernel.layer_name, kernel.symbol_table_name + ): - yield (0, (format_hints.Hex(hive.vol.offset), )) + yield (0, (format_hints.Hex(hive.vol.offset),)) def run(self): return renderers.TreeGrid([("Offset", format_hints.Hex)], self._generator()) diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 2082c339e..19527321e 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -25,18 +25,26 @@ class PrintKey(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), - requirements.IntRequirement(name = 'offset', description = "Hive Offset", default = None, optional = True), - requirements.StringRequirement(name = 'key', - description = "Key to start from", - default = None, - optional = True), - requirements.BooleanRequirement(name = 'recurse', - description = 'Recurses through keys', - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), + requirements.IntRequirement( + name="offset", description="Hive Offset", default=None, optional=True + ), + requirements.StringRequirement( + name="key", description="Key to start from", default=None, optional=True + ), + requirements.BooleanRequirement( + name="recurse", + description="Recurses through keys", + default=False, + optional=True, + ), ] @classmethod @@ -44,8 +52,12 @@ class PrintKey(interfaces.plugins.PluginInterface): cls, hive: RegistryHive, node_path: Sequence[objects.StructType] = None, - recurse: bool = False - ) -> Iterable[Tuple[int, bool, datetime.datetime, str, bool, interfaces.objects.ObjectInterface]]: + recurse: bool = False, + ) -> Iterable[ + Tuple[ + int, bool, datetime.datetime, str, bool, interfaces.objects.ObjectInterface + ] + ]: """Walks through a set of nodes from a given node (last one in node_path). Avoids loops by not traversing into nodes already present in the node_path. @@ -65,13 +77,22 @@ class PrintKey(interfaces.plugins.PluginInterface): return node = node_path[-1] key_path_items = [hive] + node_path[1:] - key_path = '\\'.join([k.get_name() for k in key_path_items]) - if node.vol.type_name.endswith(constants.BANG + '_CELL_DATA'): - raise RegistryFormatException(hive.name, "Encountered _CELL_DATA instead of _CM_KEY_NODE") + key_path = "\\".join([k.get_name() for k in key_path_items]) + if node.vol.type_name.endswith(constants.BANG + "_CELL_DATA"): + raise RegistryFormatException( + hive.name, "Encountered _CELL_DATA instead of _CM_KEY_NODE" + ) last_write_time = conversion.wintime_to_datetime(node.LastWriteTime.QuadPart) for key_node in node.get_subkeys(): - result = (len(node_path), True, last_write_time, key_path, key_node.get_volatile(), key_node) + result = ( + len(node_path), + True, + last_write_time, + key_path, + key_node.get_volatile(), + key_node, + ) yield result if recurse: @@ -82,16 +103,27 @@ class PrintKey(interfaces.plugins.PluginInterface): vollog.debug(excp) continue - yield from cls.key_iterator(hive, node_path + [key_node], recurse = recurse) + yield from cls.key_iterator( + hive, node_path + [key_node], recurse=recurse + ) for value_node in node.get_values(): - result = (len(node_path), False, last_write_time, key_path, node.get_volatile(), value_node) + result = ( + len(node_path), + False, + last_write_time, + key_path, + node.get_volatile(), + value_node, + ) yield result - def _printkey_iterator(self, - hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, - recurse: bool = False): + def _printkey_iterator( + self, + hive: RegistryHive, + node_path: Sequence[objects.StructType] = None, + recurse: bool = False, + ): """Method that wraps the more generic key_iterator, to provide output for printkey specifically. @@ -103,96 +135,177 @@ class PrintKey(interfaces.plugins.PluginInterface): Yields: The depth, and a tuple of results (last write time, hive offset, type, path, name, data and volatile) """ - for depth, is_key, last_write_time, key_path, volatile, node in self.key_iterator(hive, node_path, recurse): + for ( + depth, + is_key, + last_write_time, + key_path, + volatile, + node, + ) in self.key_iterator(hive, node_path, recurse): if is_key: try: key_node_name = node.get_name() - except (exceptions.InvalidAddressException, RegistryFormatException) as excp: + except ( + exceptions.InvalidAddressException, + RegistryFormatException, + ) as excp: vollog.debug(excp) key_node_name = renderers.UnreadableValue() - yield (depth, (last_write_time, renderers.format_hints.Hex(hive.hive_offset), "Key", key_path, - key_node_name, renderers.NotApplicableValue(), volatile)) + yield ( + depth, + ( + last_write_time, + renderers.format_hints.Hex(hive.hive_offset), + "Key", + key_path, + key_node_name, + renderers.NotApplicableValue(), + volatile, + ), + ) else: try: value_node_name = node.get_name() or "(Default)" - except (exceptions.InvalidAddressException, RegistryFormatException) as excp: + except ( + exceptions.InvalidAddressException, + RegistryFormatException, + ) as excp: vollog.debug(excp) value_node_name = renderers.UnreadableValue() try: value_type = RegValueTypes(node.Type).name - except (exceptions.InvalidAddressException, RegistryFormatException) as excp: + except ( + exceptions.InvalidAddressException, + RegistryFormatException, + ) as excp: vollog.debug(excp) value_type = renderers.UnreadableValue() if isinstance(value_type, renderers.UnreadableValue): - vollog.debug("Couldn't read registry value type, so data is unreadable") - value_data: Union[interfaces.renderers.BaseAbsentValue, bytes] = renderers.UnreadableValue() + vollog.debug( + "Couldn't read registry value type, so data is unreadable" + ) + value_data: Union[ + interfaces.renderers.BaseAbsentValue, bytes + ] = renderers.UnreadableValue() else: try: value_data = node.decode_data() if isinstance(value_data, int): - value_data = format_hints.MultiTypeData(value_data, encoding = 'utf-8') + value_data = format_hints.MultiTypeData( + value_data, encoding="utf-8" + ) elif RegValueTypes(node.Type) == RegValueTypes.REG_BINARY: - value_data = format_hints.MultiTypeData(value_data, show_hex = True) + value_data = format_hints.MultiTypeData( + value_data, show_hex=True + ) elif RegValueTypes(node.Type) == RegValueTypes.REG_MULTI_SZ: - value_data = format_hints.MultiTypeData(value_data, - encoding = 'utf-16-le', - split_nulls = True) + value_data = format_hints.MultiTypeData( + value_data, encoding="utf-16-le", split_nulls=True + ) else: - value_data = format_hints.MultiTypeData(value_data, encoding = 'utf-16-le') - except (ValueError, exceptions.InvalidAddressException, RegistryFormatException) as excp: + value_data = format_hints.MultiTypeData( + value_data, encoding="utf-16-le" + ) + except ( + ValueError, + exceptions.InvalidAddressException, + RegistryFormatException, + ) as excp: vollog.debug(excp) value_data = renderers.UnreadableValue() - result = (depth, (last_write_time, renderers.format_hints.Hex(hive.hive_offset), value_type, key_path, - value_node_name, value_data, volatile)) + result = ( + depth, + ( + last_write_time, + renderers.format_hints.Hex(hive.hive_offset), + value_type, + key_path, + value_node_name, + value_data, + volatile, + ), + ) yield result - def _registry_walker(self, - layer_name: str, - symbol_table: str, - hive_offsets: List[int] = None, - key: str = None, - recurse: bool = False): + def _registry_walker( + self, + layer_name: str, + symbol_table: str, + hive_offsets: List[int] = None, + key: str = None, + recurse: bool = False, + ): - for hive in hivelist.HiveList.list_hives(self.context, - self.config_path, - layer_name = layer_name, - symbol_table = symbol_table, - hive_offsets = hive_offsets): + for hive in hivelist.HiveList.list_hives( + self.context, + self.config_path, + layer_name=layer_name, + symbol_table=symbol_table, + hive_offsets=hive_offsets, + ): try: # Walk it if key is not None: - node_path = hive.get_key(key, return_list = True) + node_path = hive.get_key(key, return_list=True) else: node_path = [hive.get_node(hive.root_cell_offset)] - for (x, y) in self._printkey_iterator(hive, node_path, recurse = recurse): + for (x, y) in self._printkey_iterator(hive, node_path, recurse=recurse): yield (x - len(node_path), y) - except (exceptions.InvalidAddressException, KeyError, RegistryFormatException) as excp: + except ( + exceptions.InvalidAddressException, + KeyError, + RegistryFormatException, + ) as excp: if isinstance(excp, KeyError): - vollog.debug(f"Key '{key}' not found in Hive at offset {hex(hive.hive_offset)}.") + vollog.debug( + f"Key '{key}' not found in Hive at offset {hex(hive.hive_offset)}." + ) elif isinstance(excp, RegistryFormatException): vollog.debug(excp) elif isinstance(excp, exceptions.InvalidAddressException): - vollog.debug(f"Invalid address identified in Hive: {hex(excp.invalid_address)}") - result = (0, (renderers.UnreadableValue(), format_hints.Hex(hive.hive_offset), "Key", - '?\\' + (key or ''), renderers.UnreadableValue(), renderers.UnreadableValue(), - renderers.UnreadableValue())) + vollog.debug( + f"Invalid address identified in Hive: {hex(excp.invalid_address)}" + ) + result = ( + 0, + ( + renderers.UnreadableValue(), + format_hints.Hex(hive.hive_offset), + "Key", + "?\\" + (key or ""), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + ), + ) yield result def run(self): - offset = self.config.get('offset', None) - kernel = self.context.modules[self.config['kernel']] + offset = self.config.get("offset", None) + kernel = self.context.modules[self.config["kernel"]] - return TreeGrid(columns = [('Last Write Time', datetime.datetime), ('Hive Offset', format_hints.Hex), - ('Type', str), ('Key', str), ('Name', str), ('Data', format_hints.MultiTypeData), - ('Volatile', bool)], - generator = self._registry_walker(kernel.layer_name, - kernel.symbol_table_name, - hive_offsets = None if offset is None else [offset], - key = self.config.get('key', None), - recurse = self.config.get('recurse', None))) + return TreeGrid( + columns=[ + ("Last Write Time", datetime.datetime), + ("Hive Offset", format_hints.Hex), + ("Type", str), + ("Key", str), + ("Name", str), + ("Data", format_hints.MultiTypeData), + ("Volatile", bool), + ], + generator=self._registry_walker( + kernel.layer_name, + kernel.symbol_table_name, + hive_offsets=None if offset is None else [offset], + key=self.config.get("key", None), + recurse=self.config.get("recurse", None), + ), + ) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index f31b7832e..f64a130fa 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -34,7 +34,9 @@ class UserAssist(interfaces.plugins.PluginInterface): self._win7 = None # taken from http://msdn.microsoft.com/en-us/library/dd378457%28v=vs.85%29.aspx try: - with open(os.path.join(os.path.dirname(__file__), "userassist.json"), "rb") as fp: + with open( + os.path.join(os.path.dirname(__file__), "userassist.json"), "rb" + ) as fp: self._folder_guids = json.load(fp) except IOError: vollog.error("Usersassist data file not found") @@ -42,10 +44,17 @@ class UserAssist(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.IntRequirement(name = 'offset', description = "Hive Offset", default = None, optional = True), - requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.IntRequirement( + name="offset", description="Hive Offset", default=None, optional=True + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), ] def parse_userassist_data(self, reg_val): @@ -76,30 +85,44 @@ class UserAssist(interfaces.plugins.PluginInterface): return item userassist_layer_name = self.context.layers.free_layer_name("userassist_buffer") - buffer = BufferDataLayer(self.context, self._config_path, userassist_layer_name, userassist_data) + buffer = BufferDataLayer( + self.context, self._config_path, userassist_layer_name, userassist_data + ) self.context.add_layer(buffer) userassist_obj = self.context.object( - object_type = self._reg_table_name + constants.BANG + self._userassist_type_name, - layer_name = userassist_layer_name, - offset = 0) + object_type=self._reg_table_name + + constants.BANG + + self._userassist_type_name, + layer_name=userassist_layer_name, + offset=0, + ) if self._win7: item["id"] = renderers.NotApplicableValue() item["count"] = int(userassist_obj.Count) seconds = (userassist_obj.FocusTime + 500) / 1000.0 - time = datetime.timedelta(seconds = seconds) if seconds > 0 else userassist_obj.FocusTime + time = ( + datetime.timedelta(seconds=seconds) + if seconds > 0 + else userassist_obj.FocusTime + ) item["focus"] = int(userassist_obj.FocusCount) item["time"] = str(time) else: item["id"] = int(userassist_obj.ID) - item["count"] = int(userassist_obj.CountStartingAtFive - if userassist_obj.CountStartingAtFive < 5 else userassist_obj.CountStartingAtFive - 5) + item["count"] = int( + userassist_obj.CountStartingAtFive + if userassist_obj.CountStartingAtFive < 5 + else userassist_obj.CountStartingAtFive - 5 + ) item["focus"] = renderers.NotApplicableValue() item["time"] = renderers.NotApplicableValue() - item["lastupdated"] = conversion.wintime_to_datetime(userassist_obj.LastUpdated.QuadPart) + item["lastupdated"] = conversion.wintime_to_datetime( + userassist_obj.LastUpdated.QuadPart + ) return item @@ -112,23 +135,29 @@ class UserAssist(interfaces.plugins.PluginInterface): elif self._win7 is False: self._userassist_type_name = "_VOL_USERASSIST_TYPES_XP" - self._userassist_size = self.context.symbol_space.get_type(self._reg_table_name + constants.BANG + - self._userassist_type_name).size + self._userassist_size = self.context.symbol_space.get_type( + self._reg_table_name + constants.BANG + self._userassist_type_name + ).size def _win7_or_later(self) -> bool: # TODO: change this if there is a better way of determining the OS version # _KUSER_SHARED_DATA.CookiePad is in Windows 6.1 (Win7) and later - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - return self.context.symbol_space.get_type(kernel.symbol_table_name + constants.BANG + - "_KUSER_SHARED_DATA").has_member('CookiePad') + return self.context.symbol_space.get_type( + kernel.symbol_table_name + constants.BANG + "_KUSER_SHARED_DATA" + ).has_member("CookiePad") - def list_userassist(self, hive: RegistryHive) -> Generator[Tuple[int, Tuple], None, None]: + def list_userassist( + self, hive: RegistryHive + ) -> Generator[Tuple[int, Tuple], None, None]: """Generate userassist data for a registry hive.""" - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - hive_name = hive.hive.cast(kernel.symbol_table_name + constants.BANG + "_CMHIVE").get_name() + hive_name = hive.hive.cast( + kernel.symbol_table_name + constants.BANG + "_CMHIVE" + ).get_name() if self._win7 is None: with contextlib.suppress(exceptions.SymbolError): @@ -137,8 +166,10 @@ class UserAssist(interfaces.plugins.PluginInterface): self._determine_userassist_type() - userassist_node_path = hive.get_key("software\\microsoft\\windows\\currentversion\\explorer\\userassist", - return_list = True) + userassist_node_path = hive.get_key( + "software\\microsoft\\windows\\currentversion\\explorer\\userassist", + return_list=True, + ) if not userassist_node_path: vollog.warning("list_userassist did not find a valid node_path (or None)") @@ -153,34 +184,66 @@ class UserAssist(interfaces.plugins.PluginInterface): # each guid key should have a Count key in it for countkey in guidkey.get_subkeys(): countkey_path = countkey.get_key_path() - countkey_last_write_time = conversion.wintime_to_datetime(countkey.LastWriteTime.QuadPart) + countkey_last_write_time = conversion.wintime_to_datetime( + countkey.LastWriteTime.QuadPart + ) # output the parent Count key - result: Tuple[int, Tuple[format_hints.Hex, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] = ( - 0, (renderers.format_hints.Hex(hive.hive_offset), hive_name, countkey_path, - countkey_last_write_time, "Key", renderers.NotApplicableValue(), renderers.NotApplicableValue(), - renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), - renderers.NotApplicableValue(), renderers.NotApplicableValue()) + result: Tuple[ + int, + Tuple[ + format_hints.Hex, + Any, + Any, + Any, + Any, + Any, + Any, + Any, + Any, + Any, + Any, + Any, + ], + ] = ( + 0, + ( + renderers.format_hints.Hex(hive.hive_offset), + hive_name, + countkey_path, + countkey_last_write_time, + "Key", + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + ), ) yield result # output any subkeys under Count for subkey in countkey.get_subkeys(): subkey_name = subkey.get_name() - result = (1, ( - renderers.format_hints.Hex(hive.hive_offset), - hive_name, - countkey_path, - countkey_last_write_time, - "Subkey", - subkey_name, - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - )) + result = ( + 1, + ( + renderers.format_hints.Hex(hive.hive_offset), + hive_name, + countkey_path, + countkey_last_write_time, + "Subkey", + subkey_name, + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + ), + ) yield result # output any values under Count @@ -193,65 +256,106 @@ class UserAssist(interfaces.plugins.PluginInterface): if self._win7: guid = value_name.split("\\")[0] if guid in self._folder_guids: - value_name = value_name.replace(guid, self._folder_guids[guid]) + value_name = value_name.replace( + guid, self._folder_guids[guid] + ) userassist_data_dict = self.parse_userassist_data(value) - result = (1, ( - renderers.format_hints.Hex(hive.hive_offset), - hive_name, - countkey_path, - countkey_last_write_time, - "Value", - value_name, - userassist_data_dict["id"], - userassist_data_dict["count"], - userassist_data_dict["focus"], - userassist_data_dict["time"], - userassist_data_dict["lastupdated"], - format_hints.HexBytes(userassist_data_dict["rawdata"]), - )) + result = ( + 1, + ( + renderers.format_hints.Hex(hive.hive_offset), + hive_name, + countkey_path, + countkey_last_write_time, + "Value", + value_name, + userassist_data_dict["id"], + userassist_data_dict["count"], + userassist_data_dict["focus"], + userassist_data_dict["time"], + userassist_data_dict["lastupdated"], + format_hints.HexBytes(userassist_data_dict["rawdata"]), + ), + ) yield result def _generator(self): hive_offsets = None - if self.config.get('offset', None) is not None: - hive_offsets = [self.config.get('offset', None)] - kernel = self.context.modules[self.config['kernel']] + if self.config.get("offset", None) is not None: + hive_offsets = [self.config.get("offset", None)] + kernel = self.context.modules[self.config["kernel"]] # get all the user hive offsets or use the one specified - for hive in hivelist.HiveList.list_hives(context = self.context, - base_config_path = self.config_path, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_string = 'ntuser.dat', - hive_offsets = hive_offsets): + for hive in hivelist.HiveList.list_hives( + context=self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_string="ntuser.dat", + hive_offsets=hive_offsets, + ): try: yield from self.list_userassist(hive) continue except exceptions.PagedInvalidAddressException as excp: - vollog.debug(f"Invalid address identified in Hive: {hex(excp.invalid_address)}") + vollog.debug( + f"Invalid address identified in Hive: {hex(excp.invalid_address)}" + ) except exceptions.InvalidAddressException as excp: - vollog.debug("Invalid address identified in lower layer {}: {}".format( - excp.layer_name, excp.invalid_address)) + vollog.debug( + "Invalid address identified in lower layer {}: {}".format( + excp.layer_name, excp.invalid_address + ) + ) except KeyError: - vollog.debug("Key '{}' not found in Hive at offset {}.".format( - "software\\microsoft\\windows\\currentversion\\explorer\\userassist", hex(hive.hive_offset))) + vollog.debug( + "Key '{}' not found in Hive at offset {}.".format( + "software\\microsoft\\windows\\currentversion\\explorer\\userassist", + hex(hive.hive_offset), + ) + ) # yield UnreadableValues when an exception occurs for a given hive_offset - result = (0, (renderers.format_hints.Hex(hive.hive_offset), - hive.name if hive.name else renderers.UnreadableValue(), renderers.UnreadableValue(), - renderers.UnreadableValue(), renderers.UnreadableValue(), renderers.UnreadableValue(), - renderers.UnreadableValue(), renderers.UnreadableValue(), renderers.UnreadableValue(), - renderers.UnreadableValue(), renderers.UnreadableValue(), renderers.UnreadableValue())) + result = ( + 0, + ( + renderers.format_hints.Hex(hive.hive_offset), + hive.name if hive.name else renderers.UnreadableValue(), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + renderers.UnreadableValue(), + ), + ) yield result def run(self): - self._reg_table_name = intermed.IntermediateSymbolTable.create(self.context, self._config_path, 'windows', - 'registry') + self._reg_table_name = intermed.IntermediateSymbolTable.create( + self.context, self._config_path, "windows", "registry" + ) - return renderers.TreeGrid([("Hive Offset", renderers.format_hints.Hex), ("Hive Name", str), ("Path", str), - ("Last Write Time", datetime.datetime), ("Type", str), ("Name", str), ("ID", int), - ("Count", int), ("Focus Count", int), ("Time Focused", str), - ("Last Updated", datetime.datetime), ("Raw Data", format_hints.HexBytes)], - self._generator()) + return renderers.TreeGrid( + [ + ("Hive Offset", renderers.format_hints.Hex), + ("Hive Name", str), + ("Path", str), + ("Last Write Time", datetime.datetime), + ("Type", str), + ("Name", str), + ("ID", int), + ("Count", int), + ("Focus Count", int), + ("Time Focused", str), + ("Last Updated", datetime.datetime), + ("Raw Data", format_hints.HexBytes), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py index 6745e95ec..3e15878bd 100644 --- a/volatility3/framework/plugins/windows/sessions.py +++ b/volatility3/framework/plugins/windows/sessions.py @@ -22,27 +22,35 @@ class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', - description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config["kernel"]] + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) # Collect all the values as we will want to group them later sessions = {} - for proc in pslist.PsList.list_processes(self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func = filter_func): + for proc in pslist.PsList.list_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=filter_func, + ): session_id = proc.get_session_id() @@ -50,20 +58,20 @@ class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) session_type = renderers.NotAvailableValue() # Construct Username from Process Env - user_domain = '' - user_name = '' + user_domain = "" + user_name = "" for var, val in proc.environment_variables(): - if var.lower() == 'username': + if var.lower() == "username": user_name = val - elif var.lower() == 'userdomain': + elif var.lower() == "userdomain": user_domain = val - if var.lower() == 'sessionname': + if var.lower() == "sessionname": session_type = val # Concat Domain and User - full_user = f'{user_domain}/{user_name}' - if full_user == '/': + full_user = f"{user_domain}/{user_name}" + if full_user == "/": full_user = renderers.NotAvailableValue() # Collect all the values in to a row we can yield after sorting. @@ -73,7 +81,7 @@ class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) "process_name": utility.array_to_string(proc.ImageFileName), "user_name": full_user, "process_start": proc.get_create_time(), - "session_type": session_type + "session_type": session_type, } # Add row to correct session so we can sort it later @@ -85,8 +93,14 @@ class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # Group and yield each row for rows in sessions.values(): for row in rows: - yield 0, (row.get('session_id'), row.get('session_type'), row.get('process_id'), - row.get('process_name'), row.get('user_name'), row.get('process_start')) + yield 0, ( + row.get("session_id"), + row.get("session_type"), + row.get("process_id"), + row.get("process_name"), + row.get("user_name"), + row.get("process_start"), + ) def generate_timeline(self): for row in self._generator(): @@ -99,5 +113,14 @@ class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) def run(self): - return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), - ("User Name", str), ("Create Time", datetime.datetime)], self._generator()) + return renderers.TreeGrid( + [ + ("Session ID", int), + ("Session Type", str), + ("Process ID", int), + ("Process", str), + ("User Name", str), + ("Create Time", datetime.datetime), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index f6f41864a..e7a1820e4 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -39,7 +39,7 @@ vollog = logging.getLogger(__name__) class Skeleton_Key_Check(interfaces.plugins.PluginInterface): - """ Looks for signs of Skeleton Key malware """ + """Looks for signs of Skeleton Key malware""" _required_framework_version = (2, 4, 0) @@ -47,14 +47,25 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): def get_requirements(cls): # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'pdbutil', component = pdbutil.PDBUtility, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pdbutil", component=pdbutil.PDBUtility, version=(1, 0, 0) + ), ] - def _get_pefile_obj(self, pe_table_name: str, layer_name: str, base_address: int) -> pefile.PE: + def _get_pefile_obj( + self, pe_table_name: str, layer_name: str, base_address: int + ) -> pefile.PE: """ Attempts to pefile object from the bytes of the PE file @@ -69,15 +80,17 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): pe_data = io.BytesIO() try: - dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset = base_address, - layer_name = layer_name) + dos_header = self.context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base_address, + layer_name=layer_name, + ) for offset, data in dos_header.reconstruct(): pe_data.seek(offset) pe_data.write(data) - pe_ret = pefile.PE(data = pe_data.getvalue(), fast_load = True) + pe_ret = pefile.PE(data=pe_data.getvalue(), fast_load=True) except exceptions.InvalidAddressException: vollog.debug("Unable to reconstruct cryptdll.dll in memory") @@ -85,9 +98,12 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): return pe_ret - def _check_for_skeleton_key_vad(self, csystem: interfaces.objects.ObjectInterface, - cryptdll_base: int, - cryptdll_size: int) -> bool: + def _check_for_skeleton_key_vad( + self, + csystem: interfaces.objects.ObjectInterface, + cryptdll_base: int, + cryptdll_size: int, + ) -> bool: """ Checks if Initialize and/or Decrypt is hooked by determining if these function pointers reference addresses inside of the cryptdll VAD @@ -99,12 +115,17 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): Returns: bool: if a skeleton key hook is present """ - return not ((cryptdll_base <= csystem.Initialize <= cryptdll_base + cryptdll_size) and \ - (cryptdll_base <= csystem.Decrypt <= cryptdll_base + cryptdll_size)) + return not ( + (cryptdll_base <= csystem.Initialize <= cryptdll_base + cryptdll_size) + and (cryptdll_base <= csystem.Decrypt <= cryptdll_base + cryptdll_size) + ) - def _check_for_skeleton_key_symbols(self, csystem: interfaces.objects.ObjectInterface, - rc4HmacInitialize: int, - rc4HmacDecrypt: int) -> bool: + def _check_for_skeleton_key_symbols( + self, + csystem: interfaces.objects.ObjectInterface, + rc4HmacInitialize: int, + rc4HmacDecrypt: int, + ) -> bool: """ Uses the PDB information to specifically check if the csystem for RC4HMAC has an initialization pointer to rc4HmacInitialize and a decryption pointer @@ -118,10 +139,16 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): Returns: bool: if a skeleton key hook was found """ - return csystem.Initialize != rc4HmacInitialize or csystem.Decrypt != rc4HmacDecrypt + return ( + csystem.Initialize != rc4HmacInitialize or csystem.Decrypt != rc4HmacDecrypt + ) - def _construct_ecrypt_array(self, array_start: int, count: int, \ - cryptdll_types: interfaces.context.ModuleInterface) -> interfaces.context.ModuleInterface: + def _construct_ecrypt_array( + self, + array_start: int, + count: int, + cryptdll_types: interfaces.context.ModuleInterface, + ) -> interfaces.context.ModuleInterface: """ Attempts to construct an array of _KERB_ECRYPT structures @@ -135,22 +162,31 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): """ try: - array = cryptdll_types.object(object_type = "array", - offset = array_start, - subtype = cryptdll_types.get_type("_KERB_ECRYPT"), - count = count, - absolute = True) + array = cryptdll_types.object( + object_type="array", + offset=array_start, + subtype=cryptdll_types.get_type("_KERB_ECRYPT"), + count=count, + absolute=True, + ) except exceptions.InvalidAddressException: - vollog.debug("Unable to construct cSystems array at given offset: {:x}".format(array_start)) + vollog.debug( + "Unable to construct cSystems array at given offset: {:x}".format( + array_start + ) + ) array = None return array - def _find_array_with_pdb_symbols(self, cryptdll_symbols: str, - cryptdll_types: interfaces.context.ModuleInterface, - proc_layer_name: str, - cryptdll_base: int) -> Tuple[interfaces.objects.ObjectInterface, int, int, int]: + def _find_array_with_pdb_symbols( + self, + cryptdll_symbols: str, + cryptdll_types: interfaces.context.ModuleInterface, + proc_layer_name: str, + cryptdll_base: int, + ) -> Tuple[interfaces.objects.ObjectInterface, int, int, int]: """ Finds the CSystems array through use of PDB symbols @@ -167,9 +203,13 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): rc4HmacInitialize: The runtime address of the expected initialization function rc4HmacDecrypt: The runtime address of the expected decryption function """ - cryptdll_module = self.context.module(cryptdll_symbols, layer_name = proc_layer_name, offset = cryptdll_base) + cryptdll_module = self.context.module( + cryptdll_symbols, layer_name=proc_layer_name, offset=cryptdll_base + ) - rc4HmacInitialize = cryptdll_module.get_absolute_symbol_address("rc4HmacInitialize") + rc4HmacInitialize = cryptdll_module.get_absolute_symbol_address( + "rc4HmacInitialize" + ) rc4HmacDecrypt = cryptdll_module.get_absolute_symbol_address("rc4HmacDecrypt") @@ -178,7 +218,9 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): # we do not want to fail just because the count is not in memory # 16 was the size on samples I tested, so I chose it as the default try: - count = cryptdll_types.object(object_type = "unsigned long", offset = count_address) + count = cryptdll_types.object( + object_type="unsigned long", offset=count_address + ) except exceptions.InvalidAddressException: count = 16 @@ -187,15 +229,20 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): array = self._construct_ecrypt_array(array_start, count, cryptdll_types) if array is None: - vollog.debug("The CSystem array is not present in memory. Stopping PDB based analysis.") + vollog.debug( + "The CSystem array is not present in memory. Stopping PDB based analysis." + ) return array, rc4HmacInitialize, rc4HmacDecrypt - def _get_cryptdll_types(self, context: interfaces.context.ContextInterface, - config, - config_path: str, - proc_layer_name: str, - cryptdll_base: int): + def _get_cryptdll_types( + self, + context: interfaces.context.ContextInterface, + config, + config_path: str, + proc_layer_name: str, + cryptdll_base: int, + ): """ Builds a symbol table from the cryptdll types generated after binary analysis @@ -206,19 +253,24 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): proc_layer_name: name of the lsass.exe process layer cryptdll_base: base address of cryptdll.dll inside of lsass.exe """ - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] table_mapping = {"nt_symbols": kernel.symbol_table_name} - cryptdll_symbol_table = intermed.IntermediateSymbolTable.create(context = context, - config_path = config_path, - sub_path = "windows", - filename = "kerb_ecrypt", - table_mapping = table_mapping) + cryptdll_symbol_table = intermed.IntermediateSymbolTable.create( + context=context, + config_path=config_path, + sub_path="windows", + filename="kerb_ecrypt", + table_mapping=table_mapping, + ) - return context.module(cryptdll_symbol_table, proc_layer_name, offset = cryptdll_base) + return context.module( + cryptdll_symbol_table, proc_layer_name, offset=cryptdll_base + ) - def _find_lsass_proc(self, proc_list: Iterable) -> \ - Tuple[interfaces.context.ContextInterface, str]: + def _find_lsass_proc( + self, proc_list: Iterable + ) -> Tuple[interfaces.context.ContextInterface, str]: """ Walks the process list and returns the first valid lsass instances. There should be only one lsass process, but malware will often use the @@ -239,13 +291,17 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): return proc, proc_layer_name except exceptions.InvalidAddressException as excp: - vollog.debug("Process {}: invalid address {} in layer {}".format(proc_id, excp.invalid_address, - excp.layer_name)) + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) return None, None - def _find_cryptdll(self, lsass_proc: interfaces.context.ContextInterface) -> \ - Tuple[int, int]: + def _find_cryptdll( + self, lsass_proc: interfaces.context.ContextInterface + ) -> Tuple[int, int]: """ Finds the base address of cryptdll.dll inside of lsass.exe @@ -266,11 +322,13 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): return None, None - def _find_csystems_with_symbols(self, proc_layer_name: str, - cryptdll_types: interfaces.context.ModuleInterface, - cryptdll_base: int, - cryptdll_size: int) -> \ - Tuple[interfaces.objects.ObjectInterface, int, int]: + def _find_csystems_with_symbols( + self, + proc_layer_name: str, + cryptdll_types: interfaces.context.ModuleInterface, + cryptdll_base: int, + cryptdll_size: int, + ) -> Tuple[interfaces.objects.ObjectInterface, int, int]: """ Attempts to find CSystems and the expected address of the handlers. Relies on downloading and parsing of the cryptdll PDB file. @@ -288,22 +346,28 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): rc4HmacDecrypt: The expected address of the csystem Decryption function """ try: - cryptdll_symbols = pdbutil.PDBUtility.symbol_table_from_pdb(self.context, - interfaces.configuration.path_join( - self.config_path, 'cryptdll'), - proc_layer_name, - "cryptdll.pdb", - cryptdll_base, - cryptdll_size) + cryptdll_symbols = pdbutil.PDBUtility.symbol_table_from_pdb( + self.context, + interfaces.configuration.path_join(self.config_path, "cryptdll"), + proc_layer_name, + "cryptdll.pdb", + cryptdll_base, + cryptdll_size, + ) except exceptions.VolatilityException: - vollog.debug("Unable to use the cryptdll PDB. Stopping PDB symbols based analysis.") + vollog.debug( + "Unable to use the cryptdll PDB. Stopping PDB symbols based analysis." + ) return None, None, None - array, rc4HmacInitialize, rc4HmacDecrypt = \ - self._find_array_with_pdb_symbols(cryptdll_symbols, cryptdll_types, proc_layer_name, cryptdll_base) + array, rc4HmacInitialize, rc4HmacDecrypt = self._find_array_with_pdb_symbols( + cryptdll_symbols, cryptdll_types, proc_layer_name, cryptdll_base + ) if array is None: - vollog.debug("The CSystem array is not present in memory. Stopping PDB symbols based analysis.") + vollog.debug( + "The CSystem array is not present in memory. Stopping PDB symbols based analysis." + ) return array, rc4HmacInitialize, rc4HmacDecrypt @@ -333,10 +397,13 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): return inst.address + inst.size + opnd.mem.disp - def _analyze_cdlocatecsystem(self, function_bytes: bytes, - function_start: int, - cryptdll_types: interfaces.context.ModuleInterface, - proc_layer_name: str) -> Optional[interfaces.objects.ObjectInterface]: + def _analyze_cdlocatecsystem( + self, + function_bytes: bytes, + function_start: int, + cryptdll_types: interfaces.context.ModuleInterface, + proc_layer_name: str, + ) -> Optional[interfaces.objects.ObjectInterface]: """ Performs static analysis on CDLocateCSystem to find the instructions that reference CSystems as well as cCsystems @@ -370,7 +437,12 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): # we do not want to fail just because the count is not in memory # 16 was the size on samples I tested, so I chose it as the default if target_address: - count = int.from_bytes(self.context.layers[proc_layer_name].read(target_address, 4), "little") + count = int.from_bytes( + self.context.layers[proc_layer_name].read( + target_address, 4 + ), + "little", + ) else: count = 16 @@ -392,10 +464,13 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): return array - def _find_csystems_with_export(self, proc_layer_name: str, - cryptdll_types: interfaces.context.ModuleInterface, - cryptdll_base: int, - _) -> Optional[interfaces.objects.ObjectInterface]: + def _find_csystems_with_export( + self, + proc_layer_name: str, + cryptdll_types: interfaces.context.ModuleInterface, + cryptdll_base: int, + _, + ) -> Optional[interfaces.objects.ObjectInterface]: """ Uses export table analysis to locate CDLocateCsystem This function references CSystems and cCsystems @@ -410,23 +485,27 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): """ if not has_capstone: - vollog.debug("capstone is not installed so cannot fall back to export table analysis.") + vollog.debug( + "capstone is not installed so cannot fall back to export table analysis." + ) return None - vollog.debug("Unable to perform analysis using PDB symbols, falling back to export table analysis.") + vollog.debug( + "Unable to perform analysis using PDB symbols, falling back to export table analysis." + ) - pe_table_name = intermed.IntermediateSymbolTable.create(self.context, - self.config_path, - "windows", - "pe", - class_types = pe.class_types) + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "windows", "pe", class_types=pe.class_types + ) cryptdll = self._get_pefile_obj(pe_table_name, proc_layer_name, cryptdll_base) if not cryptdll: return None - cryptdll.parse_data_directories(directories = [pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_EXPORT"]]) - if not hasattr(cryptdll, 'DIRECTORY_ENTRY_EXPORT'): + cryptdll.parse_data_directories( + directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_EXPORT"]] + ) + if not hasattr(cryptdll, "DIRECTORY_ENTRY_EXPORT"): return None # find the location of CDLocateCSystem and then perform static analysis @@ -437,24 +516,34 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): function_start = cryptdll_base + export.address try: - function_bytes = self.context.layers[proc_layer_name].read(function_start, 0x50) + function_bytes = self.context.layers[proc_layer_name].read( + function_start, 0x50 + ) except exceptions.InvalidAddressException: vollog.debug( - "The CDLocateCSystem function is not present in the lsass address space. Stopping export based analysis.") + "The CDLocateCSystem function is not present in the lsass address space. Stopping export based analysis." + ) break - array = self._analyze_cdlocatecsystem(function_bytes, function_start, cryptdll_types, proc_layer_name) + array = self._analyze_cdlocatecsystem( + function_bytes, function_start, cryptdll_types, proc_layer_name + ) if array is None: - vollog.debug("The CSystem array is not present in memory. Stopping export based analysis.") + vollog.debug( + "The CSystem array is not present in memory. Stopping export based analysis." + ) return array return None - def _find_csystems_with_scanning(self, proc_layer_name: str, - cryptdll_types: interfaces.context.ModuleInterface, - cryptdll_base: int, - cryptdll_size: int) -> List[interfaces.context.ModuleInterface]: + def _find_csystems_with_scanning( + self, + proc_layer_name: str, + cryptdll_types: interfaces.context.ModuleInterface, + cryptdll_base: int, + cryptdll_size: int, + ) -> List[interfaces.context.ModuleInterface]: """ Performs scanning to find potential RC4 HMAC csystem instances @@ -480,22 +569,23 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): # scan for potential instances of RC4 HMAC # the signature is based on the type being 0x17 # and the block size member being 1 in all test samples - for address in proc_layer.scan(self.context, - scanners.BytesScanner(b"\x17\x00\x00\x00\x01\x00\x00\x00"), - sections = [(cryptdll_base, cryptdll_size)]): + for address in proc_layer.scan( + self.context, + scanners.BytesScanner(b"\x17\x00\x00\x00\x01\x00\x00\x00"), + sections=[(cryptdll_base, cryptdll_size)], + ): # this occurs across page boundaries if not proc_layer.is_valid(address, ecrypt_size): continue - kerb = cryptdll_types.object("_KERB_ECRYPT", - offset = address, - absolute = True) + kerb = cryptdll_types.object("_KERB_ECRYPT", offset=address, absolute=True) # ensure the Encrypt and Finish pointers are inside the VAD # these are not manipulated in the attack - if (cryptdll_base < kerb.Encrypt < cryptdll_end) and \ - (cryptdll_base < kerb.Finish < cryptdll_end): + if (cryptdll_base < kerb.Encrypt < cryptdll_end) and ( + cryptdll_base < kerb.Finish < cryptdll_end + ): csystems.append(kerb) return csystems @@ -509,7 +599,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): Args: procs: the process list filtered to lsass.exe instances """ - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name): vollog.info("This plugin only supports 64bit Windows memory samples") @@ -518,51 +608,55 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): lsass_proc, proc_layer_name = self._find_lsass_proc(procs) if not lsass_proc: vollog.info( - "Unable to find a valid lsass.exe process in the process list. This should never happen. Analysis cannot proceed.") + "Unable to find a valid lsass.exe process in the process list. This should never happen. Analysis cannot proceed." + ) return cryptdll_base, cryptdll_size = self._find_cryptdll(lsass_proc) if not cryptdll_base: - vollog.info("Unable to find the location of cryptdll.dll inside of lsass.exe. Analysis cannot proceed.") + vollog.info( + "Unable to find the location of cryptdll.dll inside of lsass.exe. Analysis cannot proceed." + ) return # the custom type information from binary analysis - cryptdll_types = self._get_cryptdll_types(self.context, - self.config, - self.config_path, - proc_layer_name, - cryptdll_base) + cryptdll_types = self._get_cryptdll_types( + self.context, self.config, self.config_path, proc_layer_name, cryptdll_base + ) # attempt to find the array and symbols directly from the PDB - csystems, rc4HmacInitialize, rc4HmacDecrypt = \ - self._find_csystems_with_symbols(proc_layer_name, - cryptdll_types, - cryptdll_base, - cryptdll_size) + csystems, rc4HmacInitialize, rc4HmacDecrypt = self._find_csystems_with_symbols( + proc_layer_name, cryptdll_types, cryptdll_base, cryptdll_size + ) # if we can't find cSystems through the PDB then # we fall back to export analysis and scanning # we keep the address of the rc4 functions from the PDB # though as its our only source to get them if csystems is None: - fallback_sources = [self._find_csystems_with_export, - self._find_csystems_with_scanning] + fallback_sources = [ + self._find_csystems_with_export, + self._find_csystems_with_scanning, + ] for source in fallback_sources: - csystems = source(proc_layer_name, - cryptdll_types, - cryptdll_base, - cryptdll_size) + csystems = source( + proc_layer_name, cryptdll_types, cryptdll_base, cryptdll_size + ) if csystems is not None: break if csystems is None: - vollog.info("Unable to find CSystems inside of cryptdll.dll. Analysis cannot proceed.") + vollog.info( + "Unable to find CSystems inside of cryptdll.dll. Analysis cannot proceed." + ) return for csystem in csystems: - if not self.context.layers[proc_layer_name].is_valid(csystem.vol.offset, csystem.vol.size): + if not self.context.layers[proc_layer_name].is_valid( + csystem.vol.offset, csystem.vol.size + ): continue # filter for RC4 HMAC @@ -571,12 +665,21 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): # use the specific symbols if present, otherwise use the vad start and size if rc4HmacInitialize and rc4HmacDecrypt: - skeleton_key_present = self._check_for_skeleton_key_symbols(csystem, rc4HmacInitialize, rc4HmacDecrypt) + skeleton_key_present = self._check_for_skeleton_key_symbols( + csystem, rc4HmacInitialize, rc4HmacDecrypt + ) else: - skeleton_key_present = self._check_for_skeleton_key_vad(csystem, cryptdll_base, cryptdll_size) + skeleton_key_present = self._check_for_skeleton_key_vad( + csystem, cryptdll_base, cryptdll_size + ) - yield 0, (lsass_proc.UniqueProcessId, "lsass.exe", skeleton_key_present, \ - format_hints.Hex(csystem.Initialize), format_hints.Hex(csystem.Decrypt)) + yield 0, ( + lsass_proc.UniqueProcessId, + "lsass.exe", + skeleton_key_present, + format_hints.Hex(csystem.Initialize), + format_hints.Hex(csystem.Decrypt), + ) def _lsass_proc_filter(self, proc): """ @@ -590,13 +693,22 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): return process_name != "lsass.exe" def run(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] return renderers.TreeGrid( - [("PID", int), ("Process", str), ("Skeleton Key Found", bool), ("rc4HmacInitialize", format_hints.Hex), - ("rc4HmacDecrypt", format_hints.Hex)], + [ + ("PID", int), + ("Process", str), + ("Skeleton Key Found", bool), + ("rc4HmacInitialize", format_hints.Hex), + ("rc4HmacDecrypt", format_hints.Hex), + ], self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = self._lsass_proc_filter))) + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=self._lsass_proc_filter, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/ssdt.py b/volatility3/framework/plugins/windows/ssdt.py index 78fd72630..184d8388c 100644 --- a/volatility3/framework/plugins/windows/ssdt.py +++ b/volatility3/framework/plugins/windows/ssdt.py @@ -24,14 +24,23 @@ class SSDT(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'modules', plugin = modules.Modules, version = (1, 0, 0)), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="modules", plugin=modules.Modules, version=(1, 0, 0) + ), ] @classmethod - def build_module_collection(cls, context: interfaces.context.ContextInterface, layer_name: str, - symbol_table: str) -> contexts.ModuleCollection: + def build_module_collection( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> contexts.ModuleCollection: """Builds a collection of modules. Args: @@ -60,12 +69,14 @@ class SSDT(plugins.PluginInterface): if module_name in constants.windows.KERNEL_MODULE_NAMES: symbol_table_name = symbol_table - context_module = contexts.SizedModule.create(context = context, - module_name = module_name, - layer_name = layer_name, - offset = mod.DllBase, - size = mod.SizeOfImage, - symbol_table_name = symbol_table_name) + context_module = contexts.SizedModule.create( + context=context, + module_name=module_name, + layer_name=layer_name, + offset=mod.DllBase, + size=mod.SizeOfImage, + symbol_table_name=symbol_table_name, + ) context_modules.append(context_module) @@ -73,25 +84,31 @@ class SSDT(plugins.PluginInterface): def _generator(self) -> Iterator[Tuple[int, Tuple[int, int, Any, Any]]]: - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name - collection = self.build_module_collection(self.context, layer_name, kernel.symbol_table_name) + collection = self.build_module_collection( + self.context, layer_name, kernel.symbol_table_name + ) - kvo = self.context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = self.context.module(kernel.symbol_table_name, layer_name = layer_name, offset = kvo) + kvo = self.context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = self.context.module( + kernel.symbol_table_name, layer_name=layer_name, offset=kvo + ) # this is just one way to enumerate the native (NT) service table. # to do the same thing for the Win32K service table, we would need Win32K.sys symbol support ## we could also find nt!KeServiceDescriptorTable (NT) and KeServiceDescriptorTableShadow (NT, Win32K) service_table_address = ntkrnlmp.get_symbol("KiServiceTable").address service_limit_address = ntkrnlmp.get_symbol("KiServiceLimit").address - service_limit = ntkrnlmp.object(object_type = "int", offset = service_limit_address) + service_limit = ntkrnlmp.object(object_type="int", offset=service_limit_address) # on 32-bit systems the table indexes are 32-bits and contain pointers (unsigned) # on 64-bit systems the indexes are also 32-bits but they're offsets from the # base address of the table and can be negative, so we need a signed data type - is_kernel_64 = symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) + is_kernel_64 = symbols.symbol_table_is_64bit( + self.context, kernel.symbol_table_name + ) if is_kernel_64: array_subtype = "long" @@ -107,26 +124,53 @@ class SSDT(plugins.PluginInterface): find_address = passthrough - functions = ntkrnlmp.object(object_type = "array", - offset = service_table_address, - subtype = ntkrnlmp.get_type(array_subtype), - count = service_limit) + functions = ntkrnlmp.object( + object_type="array", + offset=service_table_address, + subtype=ntkrnlmp.get_type(array_subtype), + count=service_limit, + ) for idx, function_obj in enumerate(functions): function = find_address(function_obj) - module_symbols = collection.get_module_symbols_by_absolute_location(function) + module_symbols = collection.get_module_symbols_by_absolute_location( + function + ) for module_name, symbol_generator in module_symbols: symbols_found = False for symbol in symbol_generator: symbols_found = True - yield (0, (idx, format_hints.Hex(function), module_name, symbol.split(constants.BANG)[1])) + yield ( + 0, + ( + idx, + format_hints.Hex(function), + module_name, + symbol.split(constants.BANG)[1], + ), + ) if not symbols_found: - yield (0, (idx, format_hints.Hex(function), module_name, renderers.NotAvailableValue())) + yield ( + 0, + ( + idx, + format_hints.Hex(function), + module_name, + renderers.NotAvailableValue(), + ), + ) def run(self) -> renderers.TreeGrid: - return renderers.TreeGrid([("Index", int), ("Address", format_hints.Hex), ("Module", str), ("Symbol", str)], - self._generator()) + return renderers.TreeGrid( + [ + ("Index", int), + ("Address", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index e79f64741..32f3df4c8 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -25,28 +25,39 @@ class Strings(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process ID to include (all other processes are excluded)", - optional = True), - requirements.URIRequirement(name = "strings_file", description = "Strings file") + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process ID to include (all other processes are excluded)", + optional=True, + ), + requirements.URIRequirement( + name="strings_file", description="Strings file" + ), ] # TODO: Make URLRequirement that can accept a file address which the framework can open def run(self): - return renderers.TreeGrid([("String", str), ("Physical Address", format_hints.Hex), ("Result", str)], - self._generator()) + return renderers.TreeGrid( + [("String", str), ("Physical Address", format_hints.Hex), ("Result", str)], + self._generator(), + ) def _generator(self) -> Generator[Tuple, None, None]: """Generates results from a strings file.""" - string_list: List[Tuple[int,bytes]] = [] + string_list: List[Tuple[int, bytes]] = [] # Test strings file format is accurate accessor = resources.ResourceAccessor() - strings_fp = accessor.open(self.config['strings_file'], "rb") + strings_fp = accessor.open(self.config["strings_file"], "rb") line = strings_fp.readline() count: float = 0 while line: @@ -57,24 +68,35 @@ class Strings(interfaces.plugins.PluginInterface): except ValueError: vollog.error(f"Line in unrecognized format: line {count}") line = strings_fp.readline() - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - revmap = self.generate_mapping(self.context, - kernel.layer_name, - kernel.symbol_table_name, - progress_callback = self._progress_callback, - pid_list = self.config['pid']) + revmap = self.generate_mapping( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + progress_callback=self._progress_callback, + pid_list=self.config["pid"], + ) last_prog: float = 0 - line_count: float = 0 + line_count: float = 0 num_strings = len(string_list) for offset, string in string_list: line_count += 1 try: - revmap_list = [name + ":" + hex(offset) for (name, offset) in revmap[offset >> 12]] + revmap_list = [ + name + ":" + hex(offset) for (name, offset) in revmap[offset >> 12] + ] except (IndexError, KeyError): revmap_list = ["FREE MEMORY"] - yield (0, (str(string, 'latin-1'), format_hints.Hex(offset), ", ".join(revmap_list))) + yield ( + 0, + ( + str(string, "latin-1"), + format_hints.Hex(offset), + ", ".join(revmap_list), + ), + ) prog = line_count / num_strings * 100 if round(prog, 1) > last_prog: last_prog = round(prog, 1) @@ -97,12 +119,14 @@ class Strings(interfaces.plugins.PluginInterface): return int(offset), string @classmethod - def generate_mapping(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - progress_callback: constants.ProgressCallback = None, - pid_list: Optional[List[int]] = None) -> Dict[int, Set[Tuple[str, int]]]: + def generate_mapping( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + progress_callback: constants.ProgressCallback = None, + pid_list: Optional[List[int]] = None, + ) -> Dict[int, Set[Tuple[str, int]]]: """Creates a reverse mapping between virtual addresses and physical addresses. @@ -122,39 +146,55 @@ class Strings(interfaces.plugins.PluginInterface): reverse_map: Dict[int, Set[Tuple[str, int]]] = dict() if isinstance(layer, intel.Intel): # We don't care about errors, we just wanted chunks that map correctly - for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors = True): + for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors=True): offset, _, mapped_offset, mapped_size, maplayer = mapval for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): cur_set = reverse_map.get(mapped_offset >> 12, set()) cur_set.add(("kernel", offset)) reverse_map[mapped_offset >> 12] = cur_set if progress_callback: - progress_callback((offset * 100) / layer.maximum_address, "Creating reverse kernel map") + progress_callback( + (offset * 100) / layer.maximum_address, + "Creating reverse kernel map", + ) # TODO: Include kernel modules - for process in pslist.PsList.list_processes(context, layer_name, symbol_table): + for process in pslist.PsList.list_processes( + context, layer_name, symbol_table + ): if not filter(process): proc_id = "Unknown" try: proc_id = process.UniqueProcessId proc_layer_name = process.add_process_layer() except exceptions.InvalidAddressException as excp: - vollog.debug("Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name)) + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) continue proc_layer = context.layers[proc_layer_name] if isinstance(proc_layer, linear.LinearlyMappedLayer): - for mapval in proc_layer.mapping(0x0, proc_layer.maximum_address, ignore_errors = True): + for mapval in proc_layer.mapping( + 0x0, proc_layer.maximum_address, ignore_errors=True + ): mapped_offset, _, offset, mapped_size, maplayer = mapval - for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): + for val in range( + mapped_offset, mapped_offset + mapped_size, 0x1000 + ): cur_set = reverse_map.get(mapped_offset >> 12, set()) - cur_set.add((f"Process {process.UniqueProcessId}", offset)) + cur_set.add( + (f"Process {process.UniqueProcessId}", offset) + ) reverse_map[mapped_offset >> 12] = cur_set # FIXME: make the progress for all processes, rather than per-process if progress_callback: - progress_callback((offset * 100) / layer.maximum_address, - f"Creating mapping for task {process.UniqueProcessId}") + progress_callback( + (offset * 100) / layer.maximum_address, + f"Creating mapping for task {process.UniqueProcessId}", + ) return reverse_map diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index b6adbb0b0..e6c1829e9 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -28,21 +28,42 @@ class SvcScan(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.PluginRequirement(name = 'poolscanner', plugin = poolscanner.PoolScanner, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'vadyarascan', plugin = vadyarascan.VadYaraScan, version = (1, 0, 0)) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) + ), ] @staticmethod def get_record_tuple(service_record: interfaces.objects.ObjectInterface): - return (format_hints.Hex(service_record.vol.offset), service_record.Order, service_record.get_pid(), - service_record.Start.description, service_record.State.description, service_record.get_type(), - service_record.get_name(), service_record.get_display(), service_record.get_binary()) + return ( + format_hints.Hex(service_record.vol.offset), + service_record.Order, + service_record.get_pid(), + service_record.Start.description, + service_record.State.description, + service_record.get_type(), + service_record.get_name(), + service_record.get_display(), + service_record.get_binary(), + ) @staticmethod - def create_service_table(context: interfaces.context.ContextInterface, symbol_table: str, config_path: str) -> str: + def create_service_table( + context: interfaces.context.ContextInterface, + symbol_table: str, + config_path: str, + ) -> str: """Constructs a symbol table containing the symbols for services depending upon the operating system in use. @@ -57,53 +78,94 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types = context.symbol_space[symbol_table].natives is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) - if versions.is_windows_xp(context = context, symbol_table = symbol_table) and not is_64bit: + if ( + versions.is_windows_xp(context=context, symbol_table=symbol_table) + and not is_64bit + ): symbol_filename = "services-xp-x86" - elif versions.is_xp_or_2003(context = context, symbol_table = symbol_table) and is_64bit: + elif ( + versions.is_xp_or_2003(context=context, symbol_table=symbol_table) + and is_64bit + ): symbol_filename = "services-xp-2003-x64" - elif versions.is_win10_16299_or_later(context = context, symbol_table = symbol_table) and is_64bit: + elif ( + versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): symbol_filename = "services-win10-16299-x64" - elif versions.is_win10_16299_or_later(context = context, symbol_table = symbol_table) and not is_64bit: + elif ( + versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): symbol_filename = "services-win10-16299-x86" - elif versions.is_win10_up_to_15063(context = context, symbol_table = symbol_table) and is_64bit: + elif ( + versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) + and is_64bit + ): symbol_filename = "services-win8-x64" - elif versions.is_win10_up_to_15063(context = context, symbol_table = symbol_table) and not is_64bit: + elif ( + versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) + and not is_64bit + ): symbol_filename = "services-win8-x86" - elif versions.is_win10_15063(context = context, symbol_table = symbol_table) and is_64bit: + elif ( + versions.is_win10_15063(context=context, symbol_table=symbol_table) + and is_64bit + ): symbol_filename = "services-win10-15063-x64" - elif versions.is_win10_15063(context = context, symbol_table = symbol_table) and not is_64bit: + elif ( + versions.is_win10_15063(context=context, symbol_table=symbol_table) + and not is_64bit + ): symbol_filename = "services-win10-15063-x86" - elif versions.is_windows_8_or_later(context = context, symbol_table = symbol_table) and is_64bit: + elif ( + versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): symbol_filename = "services-win8-x64" - elif versions.is_windows_8_or_later(context = context, symbol_table = symbol_table) and not is_64bit: + elif ( + versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): symbol_filename = "services-win8-x86" - elif versions.is_vista_or_later(context = context, symbol_table = symbol_table) and is_64bit: + elif ( + versions.is_vista_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): symbol_filename = "services-vista-x64" - elif versions.is_vista_or_later(context = context, symbol_table = symbol_table) and not is_64bit: + elif ( + versions.is_vista_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): symbol_filename = "services-vista-x86" else: raise NotImplementedError("This version of Windows is not supported!") - return intermed.IntermediateSymbolTable.create(context, - config_path, - os.path.join("windows", "services"), - symbol_filename, - class_types = services.class_types, - native_types = native_types) + return intermed.IntermediateSymbolTable.create( + context, + config_path, + os.path.join("windows", "services"), + symbol_filename, + class_types=services.class_types, + native_types=native_types, + ) def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - service_table_name = self.create_service_table(self.context, kernel.symbol_table_name, - self.config_path) + service_table_name = self.create_service_table( + self.context, kernel.symbol_table_name, self.config_path + ) - relative_tag_offset = self.context.symbol_space.get_type(service_table_name + constants.BANG + - "_SERVICE_RECORD").relative_child_offset("Tag") + relative_tag_offset = self.context.symbol_space.get_type( + service_table_name + constants.BANG + "_SERVICE_RECORD" + ).relative_child_offset("Tag") filter_func = pslist.PsList.create_name_filter(["services.exe"]) - is_vista_or_later = versions.is_vista_or_later(context = self.context, - symbol_table = kernel.symbol_table_name) + is_vista_or_later = versions.is_vista_or_later( + context=self.context, symbol_table=kernel.symbol_table_name + ) if is_vista_or_later: service_tag = b"serH" @@ -112,39 +174,50 @@ class SvcScan(interfaces.plugins.PluginInterface): seen = [] - for task in pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func): + for task in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ): proc_id = "Unknown" try: proc_id = task.UniqueProcessId proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: - vollog.debug("Process {}: invalid address {} in layer {}".format(proc_id, excp.invalid_address, - excp.layer_name)) + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) continue layer = self.context.layers[proc_layer_name] - for offset in layer.scan(context = self.context, - scanner = scanners.BytesScanner(needle = service_tag), - sections = vadyarascan.VadYaraScan.get_vad_maps(task)): + for offset in layer.scan( + context=self.context, + scanner=scanners.BytesScanner(needle=service_tag), + sections=vadyarascan.VadYaraScan.get_vad_maps(task), + ): if not is_vista_or_later: - service_record = self.context.object(service_table_name + constants.BANG + "_SERVICE_RECORD", - offset = offset - relative_tag_offset, - layer_name = proc_layer_name) + service_record = self.context.object( + service_table_name + constants.BANG + "_SERVICE_RECORD", + offset=offset - relative_tag_offset, + layer_name=proc_layer_name, + ) if not service_record.is_valid(): continue yield (0, self.get_record_tuple(service_record)) else: - service_header = self.context.object(service_table_name + constants.BANG + "_SERVICE_HEADER", - offset = offset, - layer_name = proc_layer_name) + service_header = self.context.object( + service_table_name + constants.BANG + "_SERVICE_HEADER", + offset=offset, + layer_name=proc_layer_name, + ) if not service_header.is_valid(): continue @@ -159,14 +232,17 @@ class SvcScan(interfaces.plugins.PluginInterface): yield (0, self.get_record_tuple(service_record)) def run(self): - return renderers.TreeGrid([ - ('Offset', format_hints.Hex), - ('Order', int), - ('PID', int), - ('Start', str), - ('State', str), - ('Type', str), - ('Name', str), - ('Display', str), - ('Binary', str), - ], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Order", int), + ("PID", int), + ("Start", str), + ("State", str), + ("Type", str), + ("Name", str), + ("Display", str), + ("Binary", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/symlinkscan.py b/volatility3/framework/plugins/windows/symlinkscan.py index ef970b296..78c2c6931 100644 --- a/volatility3/framework/plugins/windows/symlinkscan.py +++ b/volatility3/framework/plugins/windows/symlinkscan.py @@ -20,16 +20,20 @@ class SymlinkScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfa @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), ] @classmethod - def scan_symlinks(cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str) -> \ - Iterable[interfaces.objects.ObjectInterface]: + def scan_symlinks( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Scans for links using the poolscanner module and constraints. Args: @@ -41,17 +45,23 @@ class SymlinkScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfa A list of symlink objects found by scanning memory for the Symlink pool signatures """ - constraints = poolscanner.PoolScanner.builtin_constraints(symbol_table, [b'Sym\xe2', b'Symb']) + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"Sym\xe2", b"Symb"] + ) - for result in poolscanner.PoolScanner.generate_pool_scan(context, layer_name, symbol_table, constraints): + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): _constraint, mem_object, _header = result yield mem_object def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for link in self.scan_symlinks(self.context, kernel.layer_name, kernel.symbol_table_name): + for link in self.scan_symlinks( + self.context, kernel.layer_name, kernel.symbol_table_name + ): try: from_name = link.get_link_name() @@ -63,7 +73,15 @@ class SymlinkScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfa except exceptions.InvalidAddressException: continue - yield (0, (format_hints.Hex(link.vol.offset), link.get_create_time(), from_name, to_name)) + yield ( + 0, + ( + format_hints.Hex(link.vol.offset), + link.get_create_time(), + from_name, + to_name, + ), + ) def generate_timeline(self): for row in self._generator(): @@ -72,9 +90,12 @@ class SymlinkScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfa yield (description, timeliner.TimeLinerType.CREATED, row_data[1]) def run(self): - return renderers.TreeGrid([ - ("Offset", format_hints.Hex), - ("CreateTime", datetime.datetime), - ("From Name", str), - ("To Name", str), - ], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("CreateTime", datetime.datetime), + ("From Name", str), + ("To Name", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index d3997c8c8..4403e7c2c 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -44,33 +44,51 @@ class VadInfo(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements - return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - # TODO: Convert this to a ListRequirement so that people can filter on sets of ranges - requirements.IntRequirement(name = 'address', - description = "Process virtual memory address to include " \ - "(all other address ranges are excluded). This must be " \ - "a base address, not an address within the desired range.", - optional = True), - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.BooleanRequirement(name = 'dump', - description = "Extract listed memory ranges", - default = False, - optional = True), - requirements.IntRequirement(name = 'maxsize', - description = "Maximum size for dumped VAD sections " \ - "(all the bigger sections will be ignored)", - default = cls.MAXSIZE_DEFAULT, - optional = True), - ] + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + # TODO: Convert this to a ListRequirement so that people can filter on sets of ranges + requirements.IntRequirement( + name="address", + description="Process virtual memory address to include " + "(all other address ranges are excluded). This must be " + "a base address, not an address within the desired range.", + optional=True, + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed memory ranges", + default=False, + optional=True, + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size for dumped VAD sections " + "(all the bigger sections will be ignored)", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), + ] @classmethod - def protect_values(cls, context: interfaces.context.ContextInterface, layer_name: str, - symbol_table: str) -> Iterable[int]: + def protect_values( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[int]: """Look up the array of memory protection constants from the memory sample. These don't change often, but if they do in the future, then finding them dynamically versus hard-coding here will ensure we parse @@ -83,15 +101,21 @@ class VadInfo(interfaces.plugins.PluginInterface): """ kvo = context.layers[layer_name].config["kernel_virtual_offset"] - ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) addr = ntkrnlmp.get_symbol("MmProtectToValue").address - values = ntkrnlmp.object(object_type = "array", offset = addr, subtype = ntkrnlmp.get_type("int"), count = 32) + values = ntkrnlmp.object( + object_type="array", offset=addr, subtype=ntkrnlmp.get_type("int"), count=32 + ) return values # type: ignore @classmethod - def list_vads(cls, proc: interfaces.objects.ObjectInterface, - filter_func: Callable[[interfaces.objects.ObjectInterface], bool] = lambda _: False) -> \ - Generator[interfaces.objects.ObjectInterface, None, None]: + def list_vads( + cls, + proc: interfaces.objects.ObjectInterface, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: False, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Lists the Virtual Address Descriptors of a specific process. Args: @@ -106,12 +130,14 @@ class VadInfo(interfaces.plugins.PluginInterface): yield vad @classmethod - def vad_dump(cls, - context: interfaces.context.ContextInterface, - proc: interfaces.objects.ObjectInterface, - vad: interfaces.objects.ObjectInterface, - open_method: Type[interfaces.plugins.FileHandlerInterface], - maxsize: int = MAXSIZE_DEFAULT) -> Optional[interfaces.plugins.FileHandlerInterface]: + def vad_dump( + cls, + context: interfaces.context.ContextInterface, + proc: interfaces.objects.ObjectInterface, + vad: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + maxsize: int = MAXSIZE_DEFAULT, + ) -> Optional[interfaces.plugins.FileHandlerInterface]: """Extracts the complete data for Vad as a FileInterface. Args: @@ -133,7 +159,9 @@ class VadInfo(interfaces.plugins.PluginInterface): return None if 0 < maxsize < vad.get_size(): - vollog.debug(f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit") + vollog.debug( + f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit" + ) return None proc_id = "Unknown" @@ -141,8 +169,11 @@ class VadInfo(interfaces.plugins.PluginInterface): proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: - vollog.debug("Process {}: invalid address {} in layer {}".format(proc_id, excp.invalid_address, - excp.layer_name)) + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) return None proc_layer = context.layers[proc_layer_name] @@ -154,7 +185,7 @@ class VadInfo(interfaces.plugins.PluginInterface): vad_size = vad.get_size() while offset < vad_start + vad_size: to_read = min(chunk_size, vad_start + vad_size - offset) - data = proc_layer.read(offset, to_read, pad = True) + data = proc_layer.read(offset, to_read, pad=True) if not data: break file_handle.write(data) @@ -167,50 +198,85 @@ class VadInfo(interfaces.plugins.PluginInterface): return file_handle def _generator(self, procs): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] def passthrough(_: interfaces.objects.ObjectInterface) -> bool: return False filter_func = passthrough - if self.config.get('address', None) is not None: + if self.config.get("address", None) is not None: def filter_function(x: interfaces.objects.ObjectInterface) -> bool: - return x.get_start() not in [self.config['address']] + return x.get_start() not in [self.config["address"]] filter_func = filter_function for proc in procs: process_name = utility.array_to_string(proc.ImageFileName) - for vad in self.list_vads(proc, filter_func = filter_func): + for vad in self.list_vads(proc, filter_func=filter_func): file_output = "Disabled" - if self.config['dump']: - file_handle = self.vad_dump(self.context, proc, vad, self.open, self.config['maxsize']) + if self.config["dump"]: + file_handle = self.vad_dump( + self.context, proc, vad, self.open, self.config["maxsize"] + ) file_output = "Error outputting file" if file_handle: file_handle.close() file_output = file_handle.preferred_filename - yield (0, (proc.UniqueProcessId, process_name, format_hints.Hex(vad.vol.offset), - format_hints.Hex(vad.get_start()), format_hints.Hex(vad.get_end()), vad.get_tag(), - vad.get_protection( - self.protect_values(self.context, kernel.layer_name, kernel.symbol_table_name), - winnt_protections), vad.get_commit_charge(), vad.get_private_memory(), - format_hints.Hex(vad.get_parent()), vad.get_file_name(), file_output)) + yield ( + 0, + ( + proc.UniqueProcessId, + process_name, + format_hints.Hex(vad.vol.offset), + format_hints.Hex(vad.get_start()), + format_hints.Hex(vad.get_end()), + vad.get_tag(), + vad.get_protection( + self.protect_values( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + ), + winnt_protections, + ), + vad.get_commit_charge(), + vad.get_private_memory(), + format_hints.Hex(vad.get_parent()), + vad.get_file_name(), + file_output, + ), + ) def run(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - return renderers.TreeGrid([("PID", int), ("Process", str), ("Offset", format_hints.Hex), - ("Start VPN", format_hints.Hex), ("End VPN", format_hints.Hex), ("Tag", str), - ("Protection", str), ("CommitCharge", int), ("PrivateMemory", int), - ("Parent", format_hints.Hex), ("File", str), ("File output", str)], - self._generator( - pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Offset", format_hints.Hex), + ("Start VPN", format_hints.Hex), + ("End VPN", format_hints.Hex), + ("Tag", str), + ("Protection", str), + ("CommitCharge", int), + ("PrivateMemory", int), + ("Parent", format_hints.Hex), + ("File", str), + ("File output", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/vadwalk.py b/volatility3/framework/plugins/windows/vadwalk.py index 1090ed554..930388b3a 100644 --- a/volatility3/framework/plugins/windows/vadwalk.py +++ b/volatility3/framework/plugins/windows/vadwalk.py @@ -23,46 +23,71 @@ class VadWalk(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.PluginRequirement(name = 'vadinfo', plugin = vadinfo.VadInfo, version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="vadinfo", plugin=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), ] - def _generator(self, procs: Generator[interfaces.objects.ObjectInterface, None, None]) -> Iterator[Tuple]: + def _generator( + self, procs: Generator[interfaces.objects.ObjectInterface, None, None] + ) -> Iterator[Tuple]: for proc in procs: for vad in vadinfo.VadInfo.list_vads(proc): - if(vad): - yield(0, (proc.UniqueProcessId, - utility.array_to_string(proc.ImageFileName), - format_hints.Hex(vad.vol.offset), - format_hints.Hex(vad.get_parent() & self.context.layers[vad.vol.layer_name].address_mask), - format_hints.Hex(vad.get_left_child()), - format_hints.Hex(vad.get_right_child()), - format_hints.Hex(vad.get_start()), - format_hints.Hex(vad.get_end()), - vad.get_tag())) + if vad: + yield ( + 0, + ( + proc.UniqueProcessId, + utility.array_to_string(proc.ImageFileName), + format_hints.Hex(vad.vol.offset), + format_hints.Hex( + vad.get_parent() + & self.context.layers[vad.vol.layer_name].address_mask + ), + format_hints.Hex(vad.get_left_child()), + format_hints.Hex(vad.get_right_child()), + format_hints.Hex(vad.get_start()), + format_hints.Hex(vad.get_end()), + vad.get_tag(), + ), + ) def run(self) -> renderers.TreeGrid: - kernel = self.context.modules[self.config['kernel']] - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config["kernel"]] + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - return renderers.TreeGrid([('PID', int), - ('Process', str), - ('Offset', format_hints.Hex), - ('Parent', format_hints.Hex), - ('Left', format_hints.Hex), - ('Right', format_hints.Hex), - ('Start', format_hints.Hex), - ('End', format_hints.Hex), - ('Tag', str)], - self._generator( - pslist.PsList.list_processes( - context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Offset", format_hints.Hex), + ("Parent", format_hints.Hex), + ("Left", format_hints.Hex), + ("Right", format_hints.Hex), + ("Start", format_hints.Hex), + ("End", format_hints.Hex), + ("Tag", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index b71e2f605..4b30a9d8b 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -23,54 +23,82 @@ class VadYaraScan(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.BooleanRequirement(name = "wide", - description = "Match wide (unicode) strings", - default = False, - optional = True), - requirements.StringRequirement(name = "yara_rules", - description = "Yara rules (as a string)", - optional = True), - requirements.URIRequirement(name = "yara_file", description = "Yara rules (as a file)", optional = True), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.BooleanRequirement( + name="wide", + description="Match wide (unicode) strings", + default=False, + optional=True, + ), + requirements.StringRequirement( + name="yara_rules", description="Yara rules (as a string)", optional=True + ), + requirements.URIRequirement( + name="yara_file", description="Yara rules (as a file)", optional=True + ), # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later - requirements.URIRequirement(name = "yara_compiled_file", - description = "Yara compiled rules (as a file)", - optional = True), - requirements.IntRequirement(name = "max_size", - default = 0x40000000, - description = "Set the maximum size (default is 1GB)", - optional = True), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, - version = (2, 0, 0)), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True) + requirements.URIRequirement( + name="yara_compiled_file", + description="Yara compiled rules (as a file)", + optional=True, + ), + requirements.IntRequirement( + name="max_size", + default=0x40000000, + description="Set the maximum size (default is 1GB)", + optional=True, + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), ] def _generator(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] rules = yarascan.YaraScan.process_yara_options(dict(self.config)) - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - for task in pslist.PsList.list_processes(context = self.context, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name, - filter_func = filter_func): + for task in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] - for offset, rule_name, name, value in layer.scan(context = self.context, - scanner = yarascan.YaraScanner(rules = rules), - sections = self.get_vad_maps(task)): - yield 0, (format_hints.Hex(offset), task.UniqueProcessId, rule_name, name, value) + for offset, rule_name, name, value in layer.scan( + context=self.context, + scanner=yarascan.YaraScanner(rules=rules), + sections=self.get_vad_maps(task), + ): + yield 0, ( + format_hints.Hex(offset), + task.UniqueProcessId, + rule_name, + name, + value, + ) @staticmethod - def get_vad_maps(task: interfaces.objects.ObjectInterface) -> Iterable[Tuple[int, int]]: + def get_vad_maps( + task: interfaces.objects.ObjectInterface, + ) -> Iterable[Tuple[int, int]]: """Creates a map of start/end addresses within a virtual address descriptor tree. @@ -85,5 +113,13 @@ class VadYaraScan(interfaces.plugins.PluginInterface): yield (vad.get_start(), vad.get_size()) def run(self): - return renderers.TreeGrid([('Offset', format_hints.Hex), ('PID', int), ('Rule', str), ('Component', str), - ('Value', bytes)], self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("PID", int), + ("Rule", str), + ("Component", str), + ("Value", bytes), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 70d473d2e..fea4a0f80 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -20,7 +20,9 @@ vollog = logging.getLogger(__name__) try: import pefile except ImportError: - vollog.info("Python pefile module not found, plugin (and dependent plugins) not available") + vollog.info( + "Python pefile module not found, plugin (and dependent plugins) not available" + ) raise @@ -35,40 +37,65 @@ class VerInfo(interfaces.plugins.PluginInterface): ## TODO: we might add a regex option on the name later, but otherwise we're good ## TODO: and we don't want any CLI options from pslist, modules, or moddump return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.PluginRequirement(name = 'modules', plugin = modules.Modules, version = (1, 0, 0)), - requirements.VersionRequirement(name = 'dlllist', component = dlllist.DllList, version = (2, 0, 0)), - requirements.BooleanRequirement(name = "extensive", - description = "Search physical layer for version information", - optional = True, - default = False), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="modules", plugin=modules.Modules, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="dlllist", component=dlllist.DllList, version=(2, 0, 0) + ), + requirements.BooleanRequirement( + name="extensive", + description="Search physical layer for version information", + optional=True, + default=False, + ), ] @classmethod - def find_version_info(cls, context: interfaces.context.ContextInterface, layer_name: str, - filename: str) -> Optional[Tuple[int, int, int, int]]: + def find_version_info( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + filename: str, + ) -> Optional[Tuple[int, int, int, int]]: """Searches for an original filename, then tracks back to find the VS_VERSION_INFO and read the fixed version information structure""" premable_max_distance = 0x500 filename = "OriginalFilename\x00" + filename - iterator = context.layers[layer_name].scan(context = context, - scanner = scanners.BytesScanner(bytes(filename, 'utf-16be'))) + iterator = context.layers[layer_name].scan( + context=context, scanner=scanners.BytesScanner(bytes(filename, "utf-16be")) + ) for offset in iterator: - data = context.layers[layer_name].read(offset - premable_max_distance, premable_max_distance) + data = context.layers[layer_name].read( + offset - premable_max_distance, premable_max_distance + ) vs_ver_info = b"\xbd\x04\xef\xfe" verinfo_offset = data.find(vs_ver_info) + len(vs_ver_info) if verinfo_offset >= 0: - structure = ' Tuple[int, int, int, int]: + def get_version_information( + cls, + context: interfaces.context.ContextInterface, + pe_table_name: str, + layer_name: str, + base_address: int, + ) -> Tuple[int, int, int, int]: """Get File and Product version information from PE files. Args: @@ -83,16 +110,20 @@ class VerInfo(interfaces.plugins.PluginInterface): pe_data = io.BytesIO() - dos_header = context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset = base_address, - layer_name = layer_name) + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base_address, + layer_name=layer_name, + ) for offset, data in dos_header.reconstruct(): pe_data.seek(offset) pe_data.write(data) - pe = pefile.PE(data = pe_data.getvalue(), fast_load = True) - pe.parse_data_directories([pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]]) + pe = pefile.PE(data=pe_data.getvalue(), fast_load=True) + pe.parse_data_directories( + [pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]] + ) if isinstance(pe.VS_FIXEDFILEINFO, list): # pefile >= 2018.8.8 (estimated) @@ -110,9 +141,12 @@ class VerInfo(interfaces.plugins.PluginInterface): return major, minor, product, build - def _generator(self, procs: Generator[interfaces.objects.ObjectInterface, None, None], - mods: Generator[interfaces.objects.ObjectInterface, None, None], session_layers: Generator[str, None, - None]): + def _generator( + self, + procs: Generator[interfaces.objects.ObjectInterface, None, None], + mods: Generator[interfaces.objects.ObjectInterface, None, None], + session_layers: Generator[str, None, None], + ): """Generates a list of PE file version info for processes, dlls, and modules. @@ -121,16 +155,16 @@ class VerInfo(interfaces.plugins.PluginInterface): mods: of modules session_layers: of layers in the session to be checked """ - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - pe_table_name = intermed.IntermediateSymbolTable.create(self.context, - self.config_path, - "windows", - "pe", - class_types = pe.class_types) + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "windows", "pe", class_types=pe.class_types + ) # TODO: Fix this so it works with more than just intel layers - physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) + physical_layer_name = self.context.layers[kernel.layer_name].config.get( + "memory_layer", None + ) for mod in mods: try: @@ -138,21 +172,40 @@ class VerInfo(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: BaseDllName = renderers.UnreadableValue() - session_layer_name = modules.Modules.find_session_layer(self.context, session_layers, mod.DllBase) + session_layer_name = modules.Modules.find_session_layer( + self.context, session_layers, mod.DllBase + ) try: - (major, minor, product, build) = self.get_version_information(self._context, pe_table_name, - session_layer_name, mod.DllBase) + (major, minor, product, build) = self.get_version_information( + self._context, pe_table_name, session_layer_name, mod.DllBase + ) except (exceptions.InvalidAddressException, TypeError, AttributeError): (major, minor, product, build) = [renderers.UnreadableValue()] * 4 - if (not isinstance(BaseDllName, renderers.UnreadableValue) and physical_layer_name is not None - and self.config['extensive']): - result = self.find_version_info(self._context, physical_layer_name, BaseDllName) + if ( + not isinstance(BaseDllName, renderers.UnreadableValue) + and physical_layer_name is not None + and self.config["extensive"] + ): + result = self.find_version_info( + self._context, physical_layer_name, BaseDllName + ) if result is not None: (major, minor, product, build) = result # the pid and process are not applicable for kernel modules - yield (0, (renderers.NotApplicableValue(), renderers.NotApplicableValue(), format_hints.Hex(mod.DllBase), - BaseDllName, major, minor, product, build)) + yield ( + 0, + ( + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + format_hints.Hex(mod.DllBase), + BaseDllName, + major, + minor, + product, + build, + ), + ) # now go through the process and dll lists for proc in procs: @@ -161,8 +214,11 @@ class VerInfo(interfaces.plugins.PluginInterface): proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: - vollog.debug("Process {}: invalid address {} in layer {}".format(proc_id, excp.invalid_address, - excp.layer_name)) + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) continue for entry in proc.load_order_modules(): @@ -178,28 +234,56 @@ class VerInfo(interfaces.plugins.PluginInterface): DllBase = renderers.UnreadableValue() try: - (major, minor, product, build) = self.get_version_information(self._context, pe_table_name, - proc_layer_name, entry.DllBase) + (major, minor, product, build) = self.get_version_information( + self._context, pe_table_name, proc_layer_name, entry.DllBase + ) except (exceptions.InvalidAddressException, ValueError, AttributeError): (major, minor, product, build) = [renderers.UnreadableValue()] * 4 - yield (0, (proc.UniqueProcessId, - proc.ImageFileName.cast("string", - max_length = proc.ImageFileName.vol.count, - errors = "replace"), DllBase, BaseDllName, major, minor, product, - build)) + yield ( + 0, + ( + proc.UniqueProcessId, + proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ), + DllBase, + BaseDllName, + major, + minor, + product, + build, + ), + ) def run(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - procs = pslist.PsList.list_processes(self.context, kernel.layer_name, kernel.symbol_table_name) + procs = pslist.PsList.list_processes( + self.context, kernel.layer_name, kernel.symbol_table_name + ) - mods = modules.Modules.list_modules(self.context, kernel.layer_name, kernel.symbol_table_name) + mods = modules.Modules.list_modules( + self.context, kernel.layer_name, kernel.symbol_table_name + ) # populate the session layers for kernel modules - session_layers = modules.Modules.get_session_layers(self.context, kernel.layer_name, kernel.symbol_table_name) - - return renderers.TreeGrid([("PID", int), ("Process", str), ("Base", format_hints.Hex), ("Name", str), - ("Major", int), ("Minor", int), ("Product", int), ("Build", int)], - self._generator(procs, mods, session_layers)) + session_layers = modules.Modules.get_session_layers( + self.context, kernel.layer_name, kernel.symbol_table_name + ) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Base", format_hints.Hex), + ("Name", str), + ("Major", int), + ("Minor", int), + ("Product", int), + ("Build", int), + ], + self._generator(procs, mods, session_layers), + ) diff --git a/volatility3/framework/plugins/windows/virtmap.py b/volatility3/framework/plugins/windows/virtmap.py index 9a241d3d8..6fbf13932 100644 --- a/volatility3/framework/plugins/windows/virtmap.py +++ b/volatility3/framework/plugins/windows/virtmap.py @@ -22,8 +22,11 @@ class VirtMap(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ) ] def _generator(self, map): @@ -32,58 +35,85 @@ class VirtMap(interfaces.plugins.PluginInterface): yield (0, (entry, format_hints.Hex(start), format_hints.Hex(end))) @classmethod - def determine_map(cls, module: interfaces.context.ModuleInterface) -> \ - Dict[str, List[Tuple[int, int]]]: + def determine_map( + cls, module: interfaces.context.ModuleInterface + ) -> Dict[str, List[Tuple[int, int]]]: """Returns the virtual map from a windows kernel module.""" layer = module.context.layers[module.layer_name] if not isinstance(layer, intel.Intel): raise result: Dict[str, List[Tuple[int, int]]] = {} - system_va_type = module.get_enumeration('_MI_SYSTEM_VA_TYPE') - large_page_size = (layer.page_size ** 2) // module.get_type("_MMPTE").size + system_va_type = module.get_enumeration("_MI_SYSTEM_VA_TYPE") + large_page_size = (layer.page_size**2) // module.get_type("_MMPTE").size - if module.has_symbol('MiVisibleState'): - symbol = module.get_symbol('MiVisibleState') - visible_state = module.object(object_type = 'pointer', - offset = symbol.address, - subtype = module.get_type('_MI_VISIBLE_STATE')).dereference() - if hasattr(visible_state, 'SystemVaRegions'): + if module.has_symbol("MiVisibleState"): + symbol = module.get_symbol("MiVisibleState") + visible_state = module.object( + object_type="pointer", + offset=symbol.address, + subtype=module.get_type("_MI_VISIBLE_STATE"), + ).dereference() + if hasattr(visible_state, "SystemVaRegions"): for i in range(visible_state.SystemVaRegions.count): lookup = system_va_type.lookup(i) region_range = result.get(lookup, []) region_range.append( - (visible_state.SystemVaRegions[i].BaseAddress, visible_state.SystemVaRegions[i].NumberOfBytes)) + ( + visible_state.SystemVaRegions[i].BaseAddress, + visible_state.SystemVaRegions[i].NumberOfBytes, + ) + ) result[lookup] = region_range - elif hasattr(visible_state, 'SystemVaType'): - system_range_start = module.object(object_type = "pointer", - offset = module.get_symbol("MmSystemRangeStart").address) - result = cls._enumerate_system_va_type(large_page_size, system_range_start, module, - visible_state.SystemVaType) + elif hasattr(visible_state, "SystemVaType"): + system_range_start = module.object( + object_type="pointer", + offset=module.get_symbol("MmSystemRangeStart").address, + ) + result = cls._enumerate_system_va_type( + large_page_size, + system_range_start, + module, + visible_state.SystemVaType, + ) else: - raise exceptions.SymbolError(None, module.name, "Required structures not found") - elif module.has_symbol('MiSystemVaType'): - system_range_start = module.object(object_type = "pointer", - offset = module.get_symbol("MmSystemRangeStart").address) - symbol = module.get_symbol('MiSystemVaType') + raise exceptions.SymbolError( + None, module.name, "Required structures not found" + ) + elif module.has_symbol("MiSystemVaType"): + system_range_start = module.object( + object_type="pointer", + offset=module.get_symbol("MmSystemRangeStart").address, + ) + symbol = module.get_symbol("MiSystemVaType") array_count = (0xFFFFFFFF + 1 - system_range_start) // large_page_size - type_array = module.object(object_type = 'array', - offset = symbol.address, - count = array_count, - subtype = module.get_type('char')) + type_array = module.object( + object_type="array", + offset=symbol.address, + count=array_count, + subtype=module.get_type("char"), + ) - result = cls._enumerate_system_va_type(large_page_size, system_range_start, module, type_array) + result = cls._enumerate_system_va_type( + large_page_size, system_range_start, module, type_array + ) else: - raise exceptions.SymbolError(None, module.name, "Required structures not found") + raise exceptions.SymbolError( + None, module.name, "Required structures not found" + ) return result @classmethod - def _enumerate_system_va_type(cls, large_page_size: int, system_range_start: int, - module: interfaces.context.ModuleInterface, - type_array: interfaces.objects.ObjectInterface) -> Dict[str, List[Tuple[int, int]]]: + def _enumerate_system_va_type( + cls, + large_page_size: int, + system_range_start: int, + module: interfaces.context.ModuleInterface, + type_array: interfaces.objects.ObjectInterface, + ) -> Dict[str, List[Tuple[int, int]]]: result: Dict[str, List[Tuple[int, int]]] = {} - system_va_type = module.get_enumeration('_MI_SYSTEM_VA_TYPE') + system_va_type = module.get_enumeration("_MI_SYSTEM_VA_TYPE") start = system_range_start prev_entry = -1 cur_size = large_page_size @@ -102,21 +132,30 @@ class VirtMap(interfaces.plugins.PluginInterface): return result @classmethod - def scannable_sections(cls, module: interfaces.context.ModuleInterface) -> Generator[Tuple[int, int], None, None]: + def scannable_sections( + cls, module: interfaces.context.ModuleInterface + ) -> Generator[Tuple[int, int], None, None]: mapping = cls.determine_map(module) for entry in mapping: - if 'Unused' not in entry: + if "Unused" not in entry: for value in mapping[entry]: yield value def run(self): - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] layer = self.context.layers[kernel.layer_name] - module = self.context.module(kernel.symbol_table_name, - layer_name = layer.name, - offset = layer.config['kernel_virtual_offset']) + module = self.context.module( + kernel.symbol_table_name, + layer_name=layer.name, + offset=layer.config["kernel_virtual_offset"], + ) - return renderers.TreeGrid([("Region", str), ("Start offset", format_hints.Hex), - ("End offset", format_hints.Hex)], - self._generator(self.determine_map(module = module))) + return renderers.TreeGrid( + [ + ("Region", str), + ("Start offset", format_hints.Hex), + ("End offset", format_hints.Hex), + ], + self._generator(self.determine_map(module=module)), + ) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 0ef55ff4b..1c548e5a6 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -16,10 +16,12 @@ vollog = logging.getLogger(__name__) try: import yara - if tuple([int(x) for x in yara.__version__.split('.')]) < (3, 8): + if tuple([int(x) for x in yara.__version__.split(".")]) < (3, 8): raise ImportError except ImportError: - vollog.info("Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available") + vollog.info( + "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available" + ) raise @@ -33,8 +35,10 @@ class YaraScanner(interfaces.layers.ScannerInterface): raise ValueError("No rules provided to YaraScanner") self._rules = rules - def __call__(self, data: bytes, data_offset: int) -> Iterable[Tuple[int, str, str, bytes]]: - for match in self._rules.match(data = data): + def __call__( + self, data: bytes, data_offset: int + ) -> Iterable[Tuple[int, str, str, bytes]]: + for match in self._rules.match(data=data): for offset, name, value in match.strings: yield (offset + data_offset, match.rule, name, value) @@ -51,50 +55,70 @@ class YaraScan(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = "Memory layer for the kernel", - architectures = ["Intel32", "Intel64"]), - requirements.BooleanRequirement(name = "insensitive", - description = "Makes the search case insensitive", - default = False, - optional = True), - requirements.BooleanRequirement(name = "wide", - description = "Match wide (unicode) strings", - default = False, - optional = True), - requirements.StringRequirement(name = "yara_rules", - description = "Yara rules (as a string)", - optional = True), - requirements.URIRequirement(name = "yara_file", description = "Yara rules (as a file)", optional = True), + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.BooleanRequirement( + name="insensitive", + description="Makes the search case insensitive", + default=False, + optional=True, + ), + requirements.BooleanRequirement( + name="wide", + description="Match wide (unicode) strings", + default=False, + optional=True, + ), + requirements.StringRequirement( + name="yara_rules", description="Yara rules (as a string)", optional=True + ), + requirements.URIRequirement( + name="yara_file", description="Yara rules (as a file)", optional=True + ), # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later - requirements.URIRequirement(name = "yara_compiled_file", - description = "Yara compiled rules (as a file)", - optional = True), - requirements.IntRequirement(name = "max_size", - default = 0x40000000, - description = "Set the maximum size (default is 1GB)", - optional = True) + requirements.URIRequirement( + name="yara_compiled_file", + description="Yara compiled rules (as a file)", + optional=True, + ), + requirements.IntRequirement( + name="max_size", + default=0x40000000, + description="Set the maximum size (default is 1GB)", + optional=True, + ), ] @classmethod def process_yara_options(cls, config: Dict[str, Any]): rules = None - if config.get('yara_rules', None) is not None: - rule = config['yara_rules'] + if config.get("yara_rules", None) is not None: + rule = config["yara_rules"] if rule[0] not in ["{", "/"]: rule = f'"{rule}"' - if config.get('case', False): + if config.get("case", False): rule += " nocase" - if config.get('wide', False): + if config.get("wide", False): rule += " wide ascii" - rules = yara.compile(sources = {'n': f'rule r1 {{strings: $a = {rule} condition: $a}}'}) - elif config.get('yara_source', None) is not None: - rules = yara.compile(source = config['yara_source']) - elif config.get('yara_file', None) is not None: - rules = yara.compile(file = resources.ResourceAccessor().open(config['yara_file'], "rb")) - elif config.get('yara_compiled_file', None) is not None: - rules = yara.load(file = resources.ResourceAccessor().open(config['yara_compiled_file'], "rb")) + rules = yara.compile( + sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} + ) + elif config.get("yara_source", None) is not None: + rules = yara.compile(source=config["yara_source"]) + elif config.get("yara_file", None) is not None: + rules = yara.compile( + file=resources.ResourceAccessor().open(config["yara_file"], "rb") + ) + elif config.get("yara_compiled_file", None) is not None: + rules = yara.load( + file=resources.ResourceAccessor().open( + config["yara_compiled_file"], "rb" + ) + ) else: vollog.error("No yara rules, nor yara rules file were specified") return rules @@ -102,10 +126,19 @@ class YaraScan(plugins.PluginInterface): def _generator(self): rules = self.process_yara_options(dict(self.config)) - layer = self.context.layers[self.config['primary']] - for offset, rule_name, name, value in layer.scan(context = self.context, scanner = YaraScanner(rules = rules)): + layer = self.context.layers[self.config["primary"]] + for offset, rule_name, name, value in layer.scan( + context=self.context, scanner=YaraScanner(rules=rules) + ): yield 0, (format_hints.Hex(offset), rule_name, name, value) def run(self): - return renderers.TreeGrid([('Offset', format_hints.Hex), ('Rule', str), ('Component', str), ('Value', bytes)], - self._generator()) + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Rule", str), + ("Component", str), + ("Value", bytes), + ], + self._generator(), + ) diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 5773861d9..ee87b3b85 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -49,8 +49,13 @@ class NotAvailableValue(interfaces.renderers.BaseAbsentValue): class TreeNode(interfaces.renderers.TreeNode): """Class representing a particular node in a tree grid.""" - def __init__(self, path: str, treegrid: 'TreeGrid', parent: Optional[interfaces.renderers.TreeNode], - values: List[interfaces.renderers.BaseTypes]) -> None: + def __init__( + self, + path: str, + treegrid: "TreeGrid", + parent: Optional[interfaces.renderers.TreeNode], + values: List[interfaces.renderers.BaseTypes], + ) -> None: if not isinstance(treegrid, TreeGrid): raise TypeError("Treegrid must be an instance of TreeGrid") self._treegrid = treegrid @@ -71,16 +76,22 @@ class TreeNode(interfaces.renderers.TreeNode): def _validate_values(self, values: List[interfaces.renderers.BaseTypes]) -> None: """A function for raising exceptions if a given set of values is invalid according to the column properties.""" - if not (isinstance(values, collections.abc.Sequence) and len(values) == len(self._treegrid.columns)): + if not ( + isinstance(values, collections.abc.Sequence) + and len(values) == len(self._treegrid.columns) + ): raise TypeError( - "Values must be a list of objects made up of simple types and number the same as the columns") + "Values must be a list of objects made up of simple types and number the same as the columns" + ) for index in range(len(self._treegrid.columns)): column = self._treegrid.columns[index] val = values[index] if not isinstance(val, (column.type, interfaces.renderers.BaseAbsentValue)): raise TypeError( "Values item with index {} is the wrong type for column {} (got {} but expected {})".format( - index, column.name, type(val), column.type)) + index, column.name, type(val), column.type + ) + ) # TODO: Consider how to deal with timezone naive/aware datetimes (and alert plugin uses to be precise) # if isinstance(val, datetime.datetime): # tznaive = val.tzinfo is None or val.tzinfo.utcoffset(val) is None @@ -122,12 +133,16 @@ class TreeNode(interfaces.renderers.TreeNode): changed = path.split(TreeGrid.path_sep) changed_index = len(changed) - 1 if int(components[changed_index]) >= int(changed[-1]): - components[changed_index] = str(int(components[changed_index]) + (1 if added else -1)) + components[changed_index] = str( + int(components[changed_index]) + (1 if added else -1) + ) self._path = TreeGrid.path_sep.join(components) def RowStructureConstructor(names: List[str]): - return collections.namedtuple("RowStructure", [TreeGrid.sanitize_name(name) for name in names]) + return collections.namedtuple( + "RowStructure", [TreeGrid.sanitize_name(name) for name in names] + ) class TreeGrid(interfaces.renderers.TreeGrid): @@ -145,8 +160,11 @@ class TreeGrid(interfaces.renderers.TreeGrid): path_sep = "|" - def __init__(self, columns: List[Tuple[str, interfaces.renderers.BaseTypes]], - generator: Optional[Iterable[Tuple[int, Tuple]]]) -> None: + def __init__( + self, + columns: List[Tuple[str, interfaces.renderers.BaseTypes]], + generator: Optional[Iterable[Tuple[int, Tuple]]], + ) -> None: """Constructs a TreeGrid object using a specific set of columns. The TreeGrid itself is a root element, that can have children but no values. @@ -166,10 +184,15 @@ class TreeGrid(interfaces.renderers.TreeGrid): for (name, column_type) in columns: is_simple_type = issubclass(column_type, self.base_types) if not is_simple_type: - raise TypeError("Column {}'s type is not a simple type: {}".format(name, - column_type.__class__.__name__)) + raise TypeError( + "Column {}'s type is not a simple type: {}".format( + name, column_type.__class__.__name__ + ) + ) converted_columns.append(interfaces.renderers.Column(name, column_type)) - self.RowStructure = RowStructureConstructor([column.name for column in converted_columns]) + self.RowStructure = RowStructureConstructor( + [column.name for column in converted_columns] + ) self._columns = converted_columns if generator is None: generator = [] @@ -181,14 +204,20 @@ class TreeGrid(interfaces.renderers.TreeGrid): def sanitize_name(text: str) -> str: output = "" for letter in text.lower(): - if letter != ' ': - output += (letter if letter in 'abcdefghiljklmnopqrstuvwxyz_0123456789' else '_') + if letter != " ": + output += ( + letter + if letter in "abcdefghiljklmnopqrstuvwxyz_0123456789" + else "_" + ) return output - def populate(self, - function: interfaces.renderers.VisitorSignature = None, - initial_accumulator: Any = None, - fail_on_errors: bool = True) -> Optional[Exception]: + def populate( + self, + function: interfaces.renderers.VisitorSignature = None, + initial_accumulator: Any = None, + fail_on_errors: bool = True, + ) -> Optional[Exception]: """Populates the tree by consuming the TreeGrid's construction generator Func is called on every node, so can be used to create output on demand. @@ -242,7 +271,9 @@ class TreeGrid(interfaces.renderers.TreeGrid): """Returns the number of rows populated.""" return self._row_count - def children(self, node: Optional[interfaces.renderers.TreeNode]) -> List[interfaces.renderers.TreeNode]: + def children( + self, node: Optional[interfaces.renderers.TreeNode] + ) -> List[interfaces.renderers.TreeNode]: """Returns the subnodes of a particular node in order.""" return [node for node, _ in self._find_children(node)] @@ -269,12 +300,19 @@ class TreeGrid(interfaces.renderers.TreeGrid): raise TypeError("Node must be a valid node within the TreeGrid") return node.values - def _append(self, parent: Optional[interfaces.renderers.TreeNode], values: Any) -> TreeNode: + def _append( + self, parent: Optional[interfaces.renderers.TreeNode], values: Any + ) -> TreeNode: """Adds a new node at the top level if parent is None, or under the parent node otherwise, after all other children.""" return self._insert(parent, None, values) - def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: Optional[int], values: Any) -> TreeNode: + def _insert( + self, + parent: Optional[interfaces.renderers.TreeNode], + position: Optional[int], + values: Any, + ) -> TreeNode: """Inserts an element into the tree at a specific position.""" parent_path = "" children = self._find_children(parent) @@ -285,7 +323,9 @@ class TreeGrid(interfaces.renderers.TreeGrid): else: newpath = parent_path + str(position) for node, _ in children[position:]: - self.visit(node, lambda child, _: child.path_changed(newpath, True), None) + self.visit( + node, lambda child, _: child.path_changed(newpath, True), None + ) tree_item = TreeNode(newpath, self, parent, values) if position is None: @@ -304,11 +344,13 @@ class TreeGrid(interfaces.renderers.TreeGrid): _T = TypeVar("_T") - def visit(self, - node: Optional[interfaces.renderers.TreeNode], - function: Callable[[interfaces.renderers.TreeNode, _T], _T], - initial_accumulator: _T, - sort_key: Optional[interfaces.renderers.ColumnSortKey] = None): + def visit( + self, + node: Optional[interfaces.renderers.TreeNode], + function: Callable[[interfaces.renderers.TreeNode, _T], _T], + initial_accumulator: _T, + sort_key: Optional[interfaces.renderers.ColumnSortKey] = None, + ): """Visits all the nodes in a tree, calling function on each one. function should have the signature function(node, accumulator) and return new_accumulator @@ -334,24 +376,30 @@ class TreeGrid(interfaces.renderers.TreeGrid): if children is not None: if sort_key is not None: sort_key_not_none = sort_key # Only necessary because of mypy - children = sorted(children, key = lambda x: sort_key_not_none(x[0].values)) + children = sorted( + children, key=lambda x: sort_key_not_none(x[0].values) + ) if not sort_key.ascending: children = reversed(children) accumulator = self._visit(children, function, accumulator, sort_key) return accumulator - def _visit(self, - list_of_children: List[interfaces.renderers.TreeNode], - function: Callable, - accumulator: _T, - sort_key: Optional[interfaces.renderers.ColumnSortKey] = None) -> _T: + def _visit( + self, + list_of_children: List[interfaces.renderers.TreeNode], + function: Callable, + accumulator: _T, + sort_key: Optional[interfaces.renderers.ColumnSortKey] = None, + ) -> _T: """Visits all the nodes in a tree, calling function on each one.""" if list_of_children is not None: for n, children in list_of_children: accumulator = function(n, accumulator) if sort_key is not None: sort_key_not_none = sort_key # Only necessary because of mypy - children = sorted(children, key = lambda x: sort_key_not_none(x[0].values)) + children = sorted( + children, key=lambda x: sort_key_not_none(x[0].values) + ) if not sort_key.ascending: children = reversed(children) accumulator = self._visit(children, function, accumulator, sort_key) @@ -359,8 +407,9 @@ class TreeGrid(interfaces.renderers.TreeGrid): class ColumnSortKey(interfaces.renderers.ColumnSortKey): - - def __init__(self, treegrid: TreeGrid, column_name: str, ascending: bool = True) -> None: + def __init__( + self, treegrid: TreeGrid, column_name: str, ascending: bool = True + ) -> None: _index = None self._type = None self.ascending = ascending diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index 3ce49bbde..bf7da9ecb 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -11,7 +11,9 @@ from typing import Union from volatility3.framework import interfaces, renderers -def wintime_to_datetime(wintime: int) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: +def wintime_to_datetime( + wintime: int, +) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: unix_time = wintime // 10000000 if unix_time == 0: return renderers.NotApplicableValue() @@ -23,8 +25,12 @@ def wintime_to_datetime(wintime: int) -> Union[interfaces.renderers.BaseAbsentVa return renderers.UnparsableValue() -def unixtime_to_datetime(unixtime: int) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: - ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = renderers.UnparsableValue() +def unixtime_to_datetime( + unixtime: int, +) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: + ret: Union[ + interfaces.renderers.BaseAbsentValue, datetime.datetime + ] = renderers.UnparsableValue() if unixtime > 0: with contextlib.suppress(ValueError): @@ -49,8 +55,8 @@ def round(addr: int, align: int, up: bool = False) -> int: return addr else: if up: - return (addr + (align - (addr % align))) - return (addr - (addr % align)) + return addr + (align - (addr % align)) + return addr - (addr % align) # For vol3 devs: @@ -86,7 +92,7 @@ def convert_ipv6(packed_ip): def convert_port(port_as_integer): - return (port_as_integer >> 8) | ((port_as_integer & 0xff) << 8) + return (port_as_integer >> 8) | ((port_as_integer & 0xFF) << 8) def convert_network_four_tuple(family, four_tuple): @@ -98,11 +104,19 @@ def convert_network_four_tuple(family, four_tuple): """ if family == socket.AF_INET: - ret = (convert_ipv4(four_tuple[0]), convert_port(four_tuple[1]), convert_ipv4(four_tuple[2]), - convert_port(four_tuple[3])) + ret = ( + convert_ipv4(four_tuple[0]), + convert_port(four_tuple[1]), + convert_ipv4(four_tuple[2]), + convert_port(four_tuple[3]), + ) elif family == socket.AF_INET6: - ret = (convert_ipv6(four_tuple[0]), convert_port(four_tuple[1]), convert_ipv6(four_tuple[2]), - convert_port(four_tuple[3])) + ret = ( + convert_ipv6(four_tuple[0]), + convert_port(four_tuple[1]), + convert_ipv6(four_tuple[2]), + convert_port(four_tuple[3]), + ) else: ret = None diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index f386d8e9d..239acbde3 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -29,11 +29,13 @@ class HexBytes(bytes): class MultiTypeData(bytes): """The contents are supposed to be a string, but may contain binary data.""" - def __new__(cls: Type['MultiTypeData'], - original: Union[int, bytes], - encoding: str = 'utf-16-le', - split_nulls: bool = False, - show_hex: bool = False) -> 'MultiTypeData': + def __new__( + cls: Type["MultiTypeData"], + original: Union[int, bytes], + encoding: str = "utf-16-le", + split_nulls: bool = False, + show_hex: bool = False, + ) -> "MultiTypeData": if isinstance(original, int): data = str(original).encode(encoding) @@ -41,11 +43,13 @@ class MultiTypeData(bytes): data = original return super().__new__(cls, data) - def __init__(self, - original: bytes, - encoding: str = 'utf-16-le', - split_nulls: bool = False, - show_hex: bool = False) -> None: + def __init__( + self, + original: bytes, + encoding: str = "utf-16-le", + split_nulls: bool = False, + show_hex: bool = False, + ) -> None: self.converted_int: bool = False if isinstance(original, int): self.converted_int = True @@ -55,8 +59,10 @@ class MultiTypeData(bytes): bytes.__init__(original) def __eq__(self, other): - return super(self) == super(other) and \ - self.converted_int == other.converted_int and \ - self.encoding == other.encoding and \ - self.split_nulls == other.split_nulls and \ - self.show_hex == other.show_hex + return ( + super(self) == super(other) + and self.converted_int == other.converted_int + and self.encoding == other.encoding + and self.split_nulls == other.split_nulls + and self.show_hex == other.show_hex + ) diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index d6ecb5393..d1af56a26 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -12,8 +12,12 @@ from volatility3.framework import constants, exceptions, interfaces, objects vollog = logging.getLogger(__name__) -SymbolSpaceReturnType = TypeVar("SymbolSpaceReturnType", interfaces.objects.Template, - interfaces.symbols.SymbolInterface, Dict[str, Any]) +SymbolSpaceReturnType = TypeVar( + "SymbolSpaceReturnType", + interfaces.objects.Template, + interfaces.symbols.SymbolInterface, + Dict[str, Any], +) class SymbolType(enum.Enum): @@ -31,7 +35,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): def __init__(self) -> None: super().__init__() - self._dict: Dict[str, interfaces.symbols.BaseSymbolTableInterface] = collections.OrderedDict() + self._dict: Dict[ + str, interfaces.symbols.BaseSymbolTableInterface + ] = collections.OrderedDict() # Permanently cache all resolved symbols self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} @@ -63,16 +69,20 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): for symbol_name in self._dict[table].get_symbols_by_type(type_name): yield table + constants.BANG + symbol_name - def get_symbols_by_location(self, offset: int, size: int = 0, table_name: str = None) -> Iterable[str]: + def get_symbols_by_location( + self, offset: int, size: int = 0, table_name: str = None + ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" - table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = self._dict.values() + table_list: Iterable[ + interfaces.symbols.BaseSymbolTableInterface + ] = self._dict.values() if table_name is not None: if table_name in self._dict: table_list = [self._dict[table_name]] else: table_list = [] for table in table_list: - for symbol_name in table.get_symbols_by_location(offset = offset, size = size): + for symbol_name in table.get_symbols_by_location(offset=offset, size=size): yield table.name + constants.BANG + symbol_name ### Space functions @@ -118,16 +128,18 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): def __init__(self, type_name: str, **kwargs) -> None: vollog.debug(f"Unresolved reference: {type_name}") - super().__init__(type_name = type_name, **kwargs) + super().__init__(type_name=type_name, **kwargs) - def _weak_resolve(self, resolve_type: SymbolType, name: str) -> SymbolSpaceReturnType: + def _weak_resolve( + self, resolve_type: SymbolType, name: str + ) -> SymbolSpaceReturnType: """Takes a symbol name and resolves it with ReferentialTemplates.""" if resolve_type == SymbolType.TYPE: - get_function = 'get_type' + get_function = "get_type" elif resolve_type == SymbolType.SYMBOL: - get_function = 'get_symbol' + get_function = "get_symbol" elif resolve_type == SymbolType.ENUM: - get_function = 'get_enumeration' + get_function = "get_enumeration" else: raise TypeError("Weak_resolve called without a proper SymbolType") @@ -138,8 +150,11 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): try: return getattr(self._dict[table_name], get_function)(component_name) except KeyError as e: - raise exceptions.SymbolError(component_name, table_name, - f'Type {name} references missing Type/Symbol/Enum: {e}') + raise exceptions.SymbolError( + component_name, + table_name, + f"Type {name} references missing Type/Symbol/Enum: {e}", + ) raise exceptions.SymbolError(name, None, f"Malformed name: {name}") def _iterative_resolve(self, traverse_list): @@ -148,10 +163,15 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): replacements = set() # Whole Symbols that still need traversing while traverse_list: - template_traverse_list, traverse_list = [self._resolved[traverse_list[0]]], traverse_list[1:] + template_traverse_list, traverse_list = [ + self._resolved[traverse_list[0]] + ], traverse_list[1:] # Traverse a single symbol looking for any ReferenceTemplate objects while template_traverse_list: - traverser, template_traverse_list = template_traverse_list[0], template_traverse_list[1:] + traverser, template_traverse_list = ( + template_traverse_list[0], + template_traverse_list[1:], + ) for child in traverser.children: if isinstance(child, objects.templates.ReferenceTemplate): # If we haven't seen it before, subresolve it and also add it @@ -159,10 +179,15 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): if child.vol.type_name not in self._resolved: traverse_list.append(child.vol.type_name) try: - self._resolved[child.vol.type_name] = self._weak_resolve( - SymbolType.TYPE, child.vol.type_name) + self._resolved[ + child.vol.type_name + ] = self._weak_resolve( + SymbolType.TYPE, child.vol.type_name + ) except exceptions.SymbolError: - self._resolved[child.vol.type_name] = self.UnresolvedTemplate(child.vol.type_name) + self._resolved[ + child.vol.type_name + ] = self.UnresolvedTemplate(child.vol.type_name) # Stash the replacement replacements.add((traverser, child)) elif child.children: @@ -184,8 +209,10 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): table_name = None index = type_name.find(constants.BANG) if index > 0: - table_name, type_name = type_name[:index], type_name[index + 1:] - raise exceptions.SymbolError(type_name, table_name, f"Unresolvable symbol requested: {type_name}") + table_name, type_name = type_name[:index], type_name[index + 1 :] + raise exceptions.SymbolError( + type_name, table_name, f"Unresolvable symbol requested: {type_name}" + ) return self._resolved[type_name] def get_symbol(self, symbol_name: str) -> interfaces.symbols.SymbolInterface: @@ -197,18 +224,22 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): table_name = None index = symbol_name.find(constants.BANG) if index > 0: - table_name, symbol_name = symbol_name[:index], symbol_name[index + 1:] - raise exceptions.SymbolError(symbol_name, table_name, f"Unresolvable Symbol: {symbol_name}") + table_name, symbol_name = symbol_name[:index], symbol_name[index + 1 :] + raise exceptions.SymbolError( + symbol_name, table_name, f"Unresolvable Symbol: {symbol_name}" + ) return retval - def _subresolve(self, object_template: interfaces.objects.Template) -> interfaces.objects.Template: + def _subresolve( + self, object_template: interfaces.objects.Template + ) -> interfaces.objects.Template: """Ensure an ObjectTemplate doesn't contain any ReferenceTemplates""" for child in object_template.children: if isinstance(child, objects.templates.ReferenceTemplate): new_child = self.get_type(child.vol.type_name) else: new_child = self._subresolve(child) - object_template.replace_child(old_child = child, new_child = new_child) + object_template.replace_child(old_child=child, new_child=new_child) return object_template def get_enumeration(self, enum_name: str) -> interfaces.objects.Template: @@ -219,8 +250,10 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): table_name = None index = enum_name.find(constants.BANG) if index > 0: - table_name, enum_name = enum_name[:index], enum_name[index + 1:] - raise exceptions.SymbolError(enum_name, table_name, f"Unresolvable Enumeration: {enum_name}") + table_name, enum_name = enum_name[:index], enum_name[index + 1 :] + raise exceptions.SymbolError( + enum_name, table_name, f"Unresolvable Enumeration: {enum_name}" + ) return retval def _membership(self, member_type: SymbolType, name: str) -> bool: @@ -255,7 +288,14 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): return self._membership(SymbolType.ENUM, name) -def symbol_table_is_64bit(context: interfaces.context.ContextInterface, symbol_table_name: str) -> bool: +def symbol_table_is_64bit( + context: interfaces.context.ContextInterface, symbol_table_name: str +) -> bool: """Returns a boolean as to whether a particular symbol table within a context is 64-bit or not.""" - return context.symbol_space.get_type(symbol_table_name + constants.BANG + "pointer").size == 8 + return ( + context.symbol_space.get_type( + symbol_table_name + constants.BANG + "pointer" + ).size + == 8 + ) diff --git a/volatility3/framework/symbols/generic/__init__.py b/volatility3/framework/symbols/generic/__init__.py index 68b4e9252..9d6da5aa4 100644 --- a/volatility3/framework/symbols/generic/__init__.py +++ b/volatility3/framework/symbols/generic/__init__.py @@ -10,39 +10,48 @@ from volatility3.framework import objects, interfaces class GenericIntelProcess(objects.StructType): - - def _add_process_layer(self, - context: interfaces.context.ContextInterface, - dtb: Union[int, interfaces.objects.ObjectInterface], - config_prefix: str = None, - preferred_name: str = None) -> str: + def _add_process_layer( + self, + context: interfaces.context.ContextInterface, + dtb: Union[int, interfaces.objects.ObjectInterface], + config_prefix: str = None, + preferred_name: str = None, + ) -> str: """Constructs a new layer based on the process's DirectoryTableBase.""" if config_prefix is None: # TODO: Ensure collisions can't happen by verifying the config_prefix is empty - random_prefix = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) - for _ in range(8)) - config_prefix = interfaces.configuration.path_join("temporary", "_" + random_prefix) + random_prefix = "".join( + random.SystemRandom().choice(string.ascii_uppercase + string.digits) + for _ in range(8) + ) + config_prefix = interfaces.configuration.path_join( + "temporary", "_" + random_prefix + ) # Figure out a suitable name we can use for the new layer if preferred_name is None: - preferred_name = context.layers.free_layer_name(prefix = self.vol.layer_name + "_Process") + preferred_name = context.layers.free_layer_name( + prefix=self.vol.layer_name + "_Process" + ) else: if preferred_name in context.layers: - preferred_name = context.layers.free_layer_name(prefix = preferred_name) + preferred_name = context.layers.free_layer_name(prefix=preferred_name) # Copy the parent's config and then make suitable changes parent_layer = context.layers[self.vol.layer_name] parent_config = parent_layer.build_configuration() # It's an intel layer, because we hardwire the "memory_layer" config option # FIXME: this could be for other architectures if we don't hardwire this/these values - parent_config['memory_layer'] = parent_layer.config['memory_layer'] - parent_config['page_map_offset'] = dtb + parent_config["memory_layer"] = parent_layer.config["memory_layer"] + parent_config["page_map_offset"] = dtb # Set the new configuration and construct the layer config_path = interfaces.configuration.path_join(config_prefix, preferred_name) context.config.splice(config_path, parent_config) - new_layer = parent_layer.__class__(context, config_path = config_path, name = preferred_name) + new_layer = parent_layer.__class__( + context, config_path=config_path, name=preferred_name + ) # Add the constructed layer and return the name context.layers.add_layer(new_layer) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 3fde0978d..24e00bbd8 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -14,7 +14,13 @@ from abc import ABCMeta from typing import Any, Dict, Generator, Iterable, List, Mapping, Optional, Tuple, Type from volatility3 import schemas, symbols -from volatility3.framework import class_subclasses, constants, exceptions, interfaces, objects +from volatility3.framework import ( + class_subclasses, + constants, + exceptions, + interfaces, + objects, +) from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources from volatility3.framework.symbols import metadata, native @@ -74,16 +80,20 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): These are documented in JSONSchema JSON files located in volatility3/schemas. """ - def __init__(self, - context: interfaces.context.ContextInterface, - config_path: str, - name: str, - isf_url: str, - native_types: interfaces.symbols.NativeTableInterface = None, - table_mapping: Optional[Dict[str, str]] = None, - validate: bool = True, - class_types: Optional[Mapping[str, Type[interfaces.objects.ObjectInterface]]] = None, - symbol_mask: int = 0) -> None: + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + isf_url: str, + native_types: interfaces.symbols.NativeTableInterface = None, + table_mapping: Optional[Dict[str, str]] = None, + validate: bool = True, + class_types: Optional[ + Mapping[str, Type[interfaces.objects.ObjectInterface]] + ] = None, + symbol_mask: int = 0, + ) -> None: """Instantiates a SymbolTable based on an IntermediateSymbolFormat JSON file. This is validated against the appropriate schema. The validation can be disabled by passing validate = False, but this should almost never be done. @@ -108,38 +118,50 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): # Validation is expensive, but we cache to store the hashes of successfully validated json objects if validate and not schemas.validate(json_object): - raise exceptions.SymbolSpaceError(f"File does not pass version validation: {isf_url}") + raise exceptions.SymbolSpaceError( + f"File does not pass version validation: {isf_url}" + ) - metadata = json_object.get('metadata', None) + metadata = json_object.get("metadata", None) if not metadata: - raise exceptions.SymbolSpaceError(f"Invalid ISF file attempted to be parsed: {isf_url}") + raise exceptions.SymbolSpaceError( + f"Invalid ISF file attempted to be parsed: {isf_url}" + ) # Determine the delegate or throw an exception - self._delegate = self._closest_version(metadata.get('format', "0.0.0"), - self._versions)(context, config_path, name, json_object, native_types, - table_mapping) + self._delegate = self._closest_version( + metadata.get("format", "0.0.0"), self._versions + )(context, config_path, name, json_object, native_types, table_mapping) if self._delegate.version < constants.ISF_MINIMUM_SUPPORTED: - raise RuntimeError("ISF version {} is no longer supported: {}".format(metadata.get('format', "0.0.0"), - isf_url)) + raise RuntimeError( + "ISF version {} is no longer supported: {}".format( + metadata.get("format", "0.0.0"), isf_url + ) + ) elif self._delegate.version < constants.ISF_MINIMUM_DEPRECATED: - vollog.warning(f"ISF version {metadata.get('format', '0.0.0')} has been deprecated: {isf_url}") + vollog.warning( + f"ISF version {metadata.get('format', '0.0.0')} has been deprecated: {isf_url}" + ) # Inherit - super().__init__(context, - config_path, - name, - native_types or self._delegate.natives, - table_mapping = table_mapping, - class_types = class_types) + super().__init__( + context, + config_path, + name, + native_types or self._delegate.natives, + table_mapping=table_mapping, + class_types=class_types, + ) # Since we've been created with parameters, ensure our config is populated likewise - self.config['isf_url'] = isf_url - self.config['symbol_mask'] = symbol_mask + self.config["isf_url"] = isf_url + self.config["symbol_mask"] = symbol_mask @staticmethod - def _closest_version(version: str, versions: Dict[Tuple[int, int, int], Type['ISFormatTable']]) \ - -> Type['ISFormatTable']: + def _closest_version( + version: str, versions: Dict[Tuple[int, int, int], Type["ISFormatTable"]] + ) -> Type["ISFormatTable"]: """Determines the highest suitable handler for specified version format. @@ -153,23 +175,26 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): supported_versions = [x for x in versions if x[0] == major and x[1] >= minor] if not supported_versions: raise ValueError( - f"No Intermediate Format interface versions support file interface version: {version}") + f"No Intermediate Format interface versions support file interface version: {version}" + ) return versions[max(supported_versions)] - symbols = _construct_delegate_function('symbols', True) - types = _construct_delegate_function('types', True) - enumerations = _construct_delegate_function('enumerations', True) - metadata = _construct_delegate_function('metadata', True) - clear_symbol_cache = _construct_delegate_function('clear_symbol_cache') - get_type = _construct_delegate_function('get_type') - get_symbol = _construct_delegate_function('get_symbol') - get_enumeration = _construct_delegate_function('get_enumeration') - get_type_class = _construct_delegate_function('get_type_class') - set_type_class = _construct_delegate_function('set_type_class') - del_type_class = _construct_delegate_function('del_type_class') + symbols = _construct_delegate_function("symbols", True) + types = _construct_delegate_function("types", True) + enumerations = _construct_delegate_function("enumerations", True) + metadata = _construct_delegate_function("metadata", True) + clear_symbol_cache = _construct_delegate_function("clear_symbol_cache") + get_type = _construct_delegate_function("get_type") + get_symbol = _construct_delegate_function("get_symbol") + get_enumeration = _construct_delegate_function("get_enumeration") + get_type_class = _construct_delegate_function("get_type_class") + set_type_class = _construct_delegate_function("set_type_class") + del_type_class = _construct_delegate_function("del_type_class") @classmethod - def file_symbol_url(cls, sub_path: str, filename: Optional[str] = None) -> Generator[str, None, None]: + def file_symbol_url( + cls, sub_path: str, filename: Optional[str] = None + ) -> Generator[str, None, None]: """Returns an iterator of appropriate file-scheme symbol URLs that can be opened by a ResourceAccessor class. @@ -187,39 +212,57 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): zip_match = "/".join(os.path.split(filename)) # Check user symbol directory first, then fallback to the framework's library to allow for overloading - vollog.log(constants.LOGLEVEL_VVVV, f"Searching for symbols in {', '.join(symbols.__path__)}") + vollog.log( + constants.LOGLEVEL_VVVV, + f"Searching for symbols in {', '.join(symbols.__path__)}", + ) for path in symbols.__path__: if not os.path.isabs(path): path = os.path.abspath(os.path.join(__file__, path)) for extension in extensions: # Hopefully these will not be large lists, otherwise this might be slow try: - for found in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + extension): + for found in ( + pathlib.Path(path) + .joinpath(sub_path) + .resolve() + .rglob(filename + extension) + ): yield found.as_uri() except FileNotFoundError: # If there's no linux symbols, don't cry about it pass # Finally try looking in zip files - for zip_path in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + '.zip'): + for zip_path in ( + pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + ".zip") + ): # We have a zipfile, so run through it and look for sub files that match the filename with zipfile.ZipFile(zip_path) as zfile: for name in zfile.namelist(): for extension in extensions: # By ending with an extension (and therefore, not /), we should not return any directories - if name.endswith(zip_match + extension) or (zip_match == "*" and name.endswith(extension)): - yield "jar:file:" + str(pathlib.Path(zip_path)) + "!" + name + if name.endswith(zip_match + extension) or ( + zip_match == "*" and name.endswith(extension) + ): + yield "jar:file:" + str( + pathlib.Path(zip_path) + ) + "!" + name @classmethod - def create(cls, - context: interfaces.context.ContextInterface, - config_path: str, - sub_path: str, - filename: str, - native_types: Optional[interfaces.symbols.NativeTableInterface] = None, - table_mapping: Optional[Dict[str, str]] = None, - class_types: Optional[Mapping[str, Type[interfaces.objects.ObjectInterface]]] = None, - symbol_mask: int = 0) -> str: + def create( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + sub_path: str, + filename: str, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, + table_mapping: Optional[Dict[str, str]] = None, + class_types: Optional[ + Mapping[str, Type[interfaces.objects.ObjectInterface]] + ] = None, + symbol_mask: int = 0, + ) -> str: """Takes a context and loads an intermediate symbol table based on a filename. @@ -237,16 +280,20 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): """ urls = list(cls.file_symbol_url(sub_path, filename)) if not urls: - raise FileNotFoundError("No symbol files found at provided filename: {}", filename) + raise FileNotFoundError( + "No symbol files found at provided filename: {}", filename + ) table_name = context.symbol_space.free_table_name(filename) - table = cls(context = context, - config_path = config_path, - name = table_name, - isf_url = urls[0], - native_types = native_types, - table_mapping = table_mapping, - class_types = class_types, - symbol_mask = symbol_mask) + table = cls( + context=context, + config_path=config_path, + name=table_name, + isf_url=urls[0], + native_types=native_types, + table_mapping=table_mapping, + class_types=class_types, + symbol_mask=symbol_mask, + ) context.symbol_space.append(table) return table_name @@ -254,21 +301,26 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return super().get_requirements() + [ requirements.StringRequirement( - "isf_url", description = "JSON file containing the symbols encoded in the Intermediate Symbol Format"), + "isf_url", + description="JSON file containing the symbols encoded in the Intermediate Symbol Format", + ), ] -class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass = ABCMeta): +class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass=ABCMeta): """Provide a base class to identify all subclasses.""" + version = (0, 0, 0) - def __init__(self, - context: interfaces.context.ContextInterface, - config_path: str, - name: str, - json_object: Any, - native_types: interfaces.symbols.NativeTableInterface = None, - table_mapping: Optional[Dict[str, str]] = None) -> None: + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + json_object: Any, + native_types: interfaces.symbols.NativeTableInterface = None, + table_mapping: Optional[Dict[str, str]] = None, + ) -> None: self._json_object = json_object self._validate_json() self.name = name @@ -276,7 +328,7 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass = ABCMeta if nt is None: raise TypeError("Native table not provided") nt.name = name + "_natives" - super().__init__(context, config_path, name, nt, table_mapping = table_mapping) + super().__init__(context, config_path, name, nt, table_mapping=table_mapping) self._overrides: Dict[str, Type[interfaces.objects.ObjectInterface]] = {} self._symbol_cache: Dict[str, interfaces.symbols.SymbolInterface] = {} @@ -287,9 +339,12 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass = ABCMeta classes = {"x64": native.x64NativeTable, "x86": native.x86NativeTable} for nc in sorted(classes): native_class = classes[nc] - for base_type in self._json_object['base_types']: + for base_type in self._json_object["base_types"]: try: - if self._json_object['base_types'][base_type]['length'] != native_class.get_type(base_type).size: + if ( + self._json_object["base_types"][base_type]["length"] + != native_class.get_type(base_type).size + ): break except TypeError: # TODO: determine whether we should give voids a size - We don't give voids a length, whereas microsoft seemingly do @@ -302,9 +357,13 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass = ABCMeta # TODO: Check the format and make use of the other metadata def _validate_json(self) -> None: - if ('user_types' not in self._json_object or 'base_types' not in self._json_object - or 'metadata' not in self._json_object or 'symbols' not in self._json_object - or 'enums' not in self._json_object): + if ( + "user_types" not in self._json_object + or "base_types" not in self._json_object + or "metadata" not in self._json_object + or "symbols" not in self._json_object + or "enums" not in self._json_object + ): raise exceptions.SymbolSpaceError("Malformed JSON file provided") @property @@ -320,6 +379,7 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass = ABCMeta class Version1Format(ISFormatTable): """Class for storing intermediate debugging data as objects and classes.""" + version = (0, 0, 1) def get_symbol(self, name: str) -> interfaces.symbols.SymbolInterface: @@ -328,35 +388,39 @@ class Version1Format(ISFormatTable): # note that this should invalidate/update the cache if self._symbol_cache.get(name, None): return self._symbol_cache[name] - symbol = self._json_object['symbols'].get(name, None) + symbol = self._json_object["symbols"].get(name, None) if not symbol: raise exceptions.SymbolError(name, self.name, f"Unknown symbol: {name}") - address = symbol['address'] - if self.config.get('symbol_mask', 0): - address = address & self.config['symbol_mask'] + address = symbol["address"] + if self.config.get("symbol_mask", 0): + address = address & self.config["symbol_mask"] - self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address) + self._symbol_cache[name] = interfaces.symbols.SymbolInterface( + name=name, address=address + ) return self._symbol_cache[name] @property def symbols(self) -> Iterable[str]: """Returns an iterator of the symbol names.""" - return list(self._json_object.get('symbols', {})) + return list(self._json_object.get("symbols", {})) @property def enumerations(self) -> Iterable[str]: """Returns an iterator of the available enumerations.""" - return list(self._json_object.get('enums', {})) + return list(self._json_object.get("enums", {})) @property def types(self) -> Iterable[str]: """Returns an iterator of the symbol type names.""" - return list(self._json_object.get('user_types', {})) + list(self.natives.types) + return list(self._json_object.get("user_types", {})) + list(self.natives.types) def get_type_class(self, name: str) -> Type[interfaces.objects.ObjectInterface]: return self._overrides.get(name, objects.AggregateType) - def set_type_class(self, name: str, clazz: Type[interfaces.objects.ObjectInterface]) -> None: + def set_type_class( + self, name: str, clazz: Type[interfaces.objects.ObjectInterface] + ) -> None: if name not in self.types: raise ValueError(f"Symbol type not in {self.name} SymbolTable: {name}") self._overrides[name] = clazz @@ -365,112 +429,144 @@ class Version1Format(ISFormatTable): if name in self._overrides: del self._overrides[name] - def _interdict_to_template(self, dictionary: Dict[str, Any]) -> interfaces.objects.Template: + def _interdict_to_template( + self, dictionary: Dict[str, Any] + ) -> interfaces.objects.Template: """Converts an intermediate format dict into an object template.""" if not dictionary: - raise exceptions.SymbolSpaceError(f"Invalid intermediate dictionary: {dictionary}") + raise exceptions.SymbolSpaceError( + f"Invalid intermediate dictionary: {dictionary}" + ) - type_name = dictionary['kind'] - if type_name == 'base': - type_name = dictionary['name'] + type_name = dictionary["kind"] + if type_name == "base": + type_name = dictionary["name"] if type_name in self.natives.types: # The symbol is a native type - native_template = self.natives.get_type(self.name + constants.BANG + type_name) + native_template = self.natives.get_type( + self.name + constants.BANG + type_name + ) # Add specific additional parameters, etc update = {} - if type_name == 'array': - update['count'] = dictionary['count'] - update['subtype'] = self._interdict_to_template(dictionary['subtype']) - elif type_name == 'pointer': - if dictionary.get('base', None): - base_type = self.natives.get_type(self.name + constants.BANG + dictionary['base']) - update['data_format'] = base_type.vol['data_format'] - update['subtype'] = self._interdict_to_template(dictionary['subtype']) - elif type_name == 'enum': - update = self._lookup_enum(dictionary['name']) - elif type_name == 'bitfield': + if type_name == "array": + update["count"] = dictionary["count"] + update["subtype"] = self._interdict_to_template(dictionary["subtype"]) + elif type_name == "pointer": + if dictionary.get("base", None): + base_type = self.natives.get_type( + self.name + constants.BANG + dictionary["base"] + ) + update["data_format"] = base_type.vol["data_format"] + update["subtype"] = self._interdict_to_template(dictionary["subtype"]) + elif type_name == "enum": + update = self._lookup_enum(dictionary["name"]) + elif type_name == "bitfield": update = { - 'start_bit': dictionary['bit_position'], - 'end_bit': dictionary['bit_position'] + dictionary['bit_length'] + "start_bit": dictionary["bit_position"], + "end_bit": dictionary["bit_position"] + dictionary["bit_length"], } - update['base_type'] = self._interdict_to_template(dictionary['type']) + update["base_type"] = self._interdict_to_template(dictionary["type"]) # We do *not* call native_template.clone(), since it slows everything down a lot # We require that the native.get_type method always returns a newly constructed python object native_template.update_vol(**update) return native_template # Otherwise - if dictionary['kind'] not in objects.AggregateTypes.values(): - raise exceptions.SymbolSpaceError(f"Unknown Intermediate format: {dictionary}") + if dictionary["kind"] not in objects.AggregateTypes.values(): + raise exceptions.SymbolSpaceError( + f"Unknown Intermediate format: {dictionary}" + ) - reference_name = dictionary['name'] + reference_name = dictionary["name"] if constants.BANG not in reference_name: reference_name = self.name + constants.BANG + reference_name else: reference_parts = reference_name.split(constants.BANG) - reference_name = (self.table_mapping.get(reference_parts[0], reference_parts[0]) + constants.BANG + - constants.BANG.join(reference_parts[1:])) + reference_name = ( + self.table_mapping.get(reference_parts[0], reference_parts[0]) + + constants.BANG + + constants.BANG.join(reference_parts[1:]) + ) - return objects.templates.ReferenceTemplate(type_name = reference_name) + return objects.templates.ReferenceTemplate(type_name=reference_name) def _lookup_enum(self, name: str) -> Dict[str, Any]: """Looks up an enumeration and returns a dictionary of __init__ parameters for an Enum.""" - lookup = self._json_object['enums'].get(name, None) + lookup = self._json_object["enums"].get(name, None) if not lookup: raise exceptions.SymbolSpaceError(f"Unknown enumeration: {name}") - result = {"choices": copy.deepcopy(lookup['constants']), "base_type": self.natives.get_type(lookup['base'])} + result = { + "choices": copy.deepcopy(lookup["constants"]), + "base_type": self.natives.get_type(lookup["base"]), + } return result def get_enumeration(self, enum_name: str) -> interfaces.objects.Template: """Resolves an individual enumeration.""" if constants.BANG in enum_name: - raise exceptions.SymbolError(enum_name, self.name, - f"Enumeration for a different table requested: {enum_name}") - if enum_name not in self._json_object['enums']: + raise exceptions.SymbolError( + enum_name, + self.name, + f"Enumeration for a different table requested: {enum_name}", + ) + if enum_name not in self._json_object["enums"]: # Fall back to the natives table - raise exceptions.SymbolError(enum_name, self.name, - f"Enumeration not found in {self.name} table: {enum_name}") - curdict = self._json_object['enums'][enum_name] - base_type = self.natives.get_type(curdict['base']) + raise exceptions.SymbolError( + enum_name, + self.name, + f"Enumeration not found in {self.name} table: {enum_name}", + ) + curdict = self._json_object["enums"][enum_name] + base_type = self.natives.get_type(curdict["base"]) # The size isn't actually used, the base-type defines it. - return objects.templates.ObjectTemplate(type_name = self.name + constants.BANG + enum_name, - object_class = objects.Enumeration, - base_type = base_type, - choices = curdict['constants']) + return objects.templates.ObjectTemplate( + type_name=self.name + constants.BANG + enum_name, + object_class=objects.Enumeration, + base_type=base_type, + choices=curdict["constants"], + ) def get_type(self, type_name: str) -> interfaces.objects.Template: """Resolves an individual symbol.""" if constants.BANG in type_name: index = type_name.find(constants.BANG) - table_name, type_name = type_name[:index], type_name[index + 1:] + table_name, type_name = type_name[:index], type_name[index + 1 :] raise exceptions.SymbolError( - type_name, table_name, - f"Symbol for a different table requested: {table_name + constants.BANG + type_name}") - if type_name not in self._json_object['user_types']: + type_name, + table_name, + f"Symbol for a different table requested: {table_name + constants.BANG + type_name}", + ) + if type_name not in self._json_object["user_types"]: # Fall back to the natives table return self.natives.get_type(self.name + constants.BANG + type_name) - curdict = self._json_object['user_types'][type_name] + curdict = self._json_object["user_types"][type_name] members = {} - for member_name in curdict['fields']: - interdict = curdict['fields'][member_name] - member = (interdict['offset'], self._interdict_to_template(interdict['type'])) + for member_name in curdict["fields"]: + interdict = curdict["fields"][member_name] + member = ( + interdict["offset"], + self._interdict_to_template(interdict["type"]), + ) members[member_name] = member object_class = self.get_type_class(type_name) if object_class == objects.AggregateType: for clazz in objects.AggregateTypes: - if objects.AggregateTypes[clazz] == curdict['kind']: + if objects.AggregateTypes[clazz] == curdict["kind"]: object_class = clazz - return objects.templates.ObjectTemplate(type_name = self.name + constants.BANG + type_name, - object_class = object_class, - size = curdict['length'], - members = members) + return objects.templates.ObjectTemplate( + type_name=self.name + constants.BANG + type_name, + object_class=object_class, + size=curdict["length"], + members=members, + ) class Version2Format(Version1Format): """Class for storing intermediate debugging data as objects and classes.""" + version = (2, 0, 0) def _get_natives(self) -> Optional[interfaces.symbols.NativeTableInterface]: @@ -479,9 +575,12 @@ class Version2Format(Version1Format): classes = {"x64": native.x64NativeTable, "x86": native.x86NativeTable} for nc in sorted(classes): native_class = classes[nc] - for base_type in self._json_object['base_types']: + for base_type in self._json_object["base_types"]: try: - if self._json_object['base_types'][base_type]['size'] != native_class.get_type(base_type).size: + if ( + self._json_object["base_types"][base_type]["size"] + != native_class.get_type(base_type).size + ): break except TypeError: # TODO: determine whether we should give voids a size - We don't give voids a length, whereas microsoft seemingly do @@ -495,162 +594,184 @@ class Version2Format(Version1Format): """Resolves an individual symbol.""" if constants.BANG in type_name: index = type_name.find(constants.BANG) - table_name, type_name = type_name[:index], type_name[index + 1:] + table_name, type_name = type_name[:index], type_name[index + 1 :] raise exceptions.SymbolError( - type_name, table_name, - f"Symbol for a different table requested: {table_name + constants.BANG + type_name}") - if type_name not in self._json_object['user_types']: + type_name, + table_name, + f"Symbol for a different table requested: {table_name + constants.BANG + type_name}", + ) + if type_name not in self._json_object["user_types"]: # Fall back to the natives table if type_name in self.natives.types: return self.natives.get_type(self.name + constants.BANG + type_name) else: - raise exceptions.SymbolError(type_name, self.name, f"Unknown symbol: {type_name}") - curdict = self._json_object['user_types'][type_name] + raise exceptions.SymbolError( + type_name, self.name, f"Unknown symbol: {type_name}" + ) + curdict = self._json_object["user_types"][type_name] members = {} - for member_name in curdict['fields']: - interdict = curdict['fields'][member_name] - member = (interdict['offset'], self._interdict_to_template(interdict['type'])) + for member_name in curdict["fields"]: + interdict = curdict["fields"][member_name] + member = ( + interdict["offset"], + self._interdict_to_template(interdict["type"]), + ) members[member_name] = member object_class = self.get_type_class(type_name) if object_class == objects.AggregateType: for clazz in objects.AggregateTypes: - if objects.AggregateTypes[clazz] == curdict['kind']: + if objects.AggregateTypes[clazz] == curdict["kind"]: object_class = clazz - return objects.templates.ObjectTemplate(type_name = self.name + constants.BANG + type_name, - object_class = object_class, - size = curdict['size'], - members = members) + return objects.templates.ObjectTemplate( + type_name=self.name + constants.BANG + type_name, + object_class=object_class, + size=curdict["size"], + members=members, + ) class Version3Format(Version2Format): """Class for storing intermediate debugging data as objects and classes.""" + version = (2, 1, 0) def get_symbol(self, name: str) -> interfaces.symbols.SymbolInterface: """Returns the symbol given by the symbol name.""" if self._symbol_cache.get(name, None): return self._symbol_cache[name] - symbol = self._json_object['symbols'].get(name, None) + symbol = self._json_object["symbols"].get(name, None) if not symbol: raise exceptions.SymbolError(name, self.name, f"Unknown symbol: {name}") - address = symbol['address'] - if self.config.get('symbol_mask', 0): - address = address & self.config['symbol_mask'] + address = symbol["address"] + if self.config.get("symbol_mask", 0): + address = address & self.config["symbol_mask"] symbol_type = None - if 'type' in symbol: - symbol_type = self._interdict_to_template(symbol['type']) + if "type" in symbol: + symbol_type = self._interdict_to_template(symbol["type"]) - self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address, - type = symbol_type) + self._symbol_cache[name] = interfaces.symbols.SymbolInterface( + name=name, address=address, type=symbol_type + ) return self._symbol_cache[name] class Version4Format(Version3Format): """Class for storing intermediate debugging data as objects and classes.""" + version = (4, 0, 0) format_mapping = { - 'int': objects.Integer, - 'float': objects.Float, - 'void': objects.Integer, - 'bool': objects.Boolean, - 'char': objects.Char + "int": objects.Integer, + "float": objects.Float, + "void": objects.Integer, + "bool": objects.Boolean, + "char": objects.Char, } def _get_natives(self) -> Optional[interfaces.symbols.NativeTableInterface]: """Determines the appropriate native_types to use from the JSON data.""" native_dict = {} - base_types = self._json_object['base_types'] + base_types = self._json_object["base_types"] for base_type in base_types: # Void are ignored because voids are not a volatility primitive, they are a specific Volatility object - if base_type != 'void': + if base_type != "void": current = base_types[base_type] # TODO: Fix up the typing of this, it bugs out because of the tuple assignment - if current['kind'] not in self.format_mapping: + if current["kind"] not in self.format_mapping: raise ValueError("Unsupported base kind") - format_val = (current['size'], current['endian'], current['signed']) - object_type = self.format_mapping[current['kind']] - if base_type == 'pointer': + format_val = (current["size"], current["endian"], current["signed"]) + object_type = self.format_mapping[current["kind"]] + if base_type == "pointer": object_type = objects.Pointer native_dict[base_type] = (object_type, format_val) - return native.NativeTable(name = "native", native_dictionary = native_dict) + return native.NativeTable(name="native", native_dictionary=native_dict) class Version5Format(Version4Format): """Class for storing intermediate debugging data as objects and classes.""" + version = (4, 1, 0) def get_symbol(self, name: str) -> interfaces.symbols.SymbolInterface: """Returns the symbol given by the symbol name.""" if self._symbol_cache.get(name, None): return self._symbol_cache[name] - symbol = self._json_object['symbols'].get(name, None) + symbol = self._json_object["symbols"].get(name, None) if not symbol: raise exceptions.SymbolError(name, self.name, f"Unknown symbol: {name}") - address = symbol['address'] - if self.config.get('symbol_mask', 0): - address = address & self.config['symbol_mask'] + address = symbol["address"] + if self.config.get("symbol_mask", 0): + address = address & self.config["symbol_mask"] symbol_type = None - if 'type' in symbol: - symbol_type = self._interdict_to_template(symbol['type']) + if "type" in symbol: + symbol_type = self._interdict_to_template(symbol["type"]) symbol_constant_data = None - if 'constant_data' in symbol: - symbol_constant_data = base64.b64decode(symbol.get('constant_data')) + if "constant_data" in symbol: + symbol_constant_data = base64.b64decode(symbol.get("constant_data")) - self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, - address = address, - type = symbol_type, - constant_data = symbol_constant_data) + self._symbol_cache[name] = interfaces.symbols.SymbolInterface( + name=name, + address=address, + type=symbol_type, + constant_data=symbol_constant_data, + ) return self._symbol_cache[name] class Version6Format(Version5Format): """Class for storing intermediate debugging data as objects and classes.""" + version = (6, 0, 0) @property def metadata(self) -> Optional[interfaces.symbols.MetadataInterface]: """Returns a MetadataInterface object.""" - if self._json_object.get('metadata', {}).get('windows'): - return metadata.WindowsMetadata(self._json_object['metadata']['windows']) - if self._json_object.get('metadata', {}).get('linux'): - return metadata.LinuxMetadata(self._json_object['metadata']['linux']) + if self._json_object.get("metadata", {}).get("windows"): + return metadata.WindowsMetadata(self._json_object["metadata"]["windows"]) + if self._json_object.get("metadata", {}).get("linux"): + return metadata.LinuxMetadata(self._json_object["metadata"]["linux"]) return None class Version7Format(Version6Format): """Class for storing intermediate debugging data as objects and classes.""" + version = (6, 1, 0) class Version8Format(Version7Format): """Class for storing intermediate debugging data as objects and classes.""" + version = (6, 2, 0) - def _process_fields(self, fields: Dict[str, Dict[str, Any]]) -> Dict[Any, Tuple[int, interfaces.objects.Template]]: + def _process_fields( + self, fields: Dict[str, Dict[str, Any]] + ) -> Dict[Any, Tuple[int, interfaces.objects.Template]]: """For each type field, it walks its tree of subtypes, reducing the hierarchy to just one level. It creates a tuple of offset and object templates for each field. """ members = {} for new_offset, member_name, member_value in self._reduce_fields(fields): - member = (new_offset, self._interdict_to_template(member_value['type'])) + member = (new_offset, self._interdict_to_template(member_value["type"])) members[member_name] = member return members - def _reduce_fields(self, - fields: Dict[str, Dict[str, Any]], - parent_offset: int = 0) -> Generator[Tuple[int, str, Dict], None, None]: + def _reduce_fields( + self, fields: Dict[str, Dict[str, Any]], parent_offset: int = 0 + ) -> Generator[Tuple[int, str, Dict], None, None]: """Reduce the fields bringing them one level up. It supports anonymous types such as structs or unions in any level of depth.""" for member_name, member_value in fields.items(): - new_offset = parent_offset + member_value.get('offset', 0) - if member_value.get('anonymous', False) and isinstance(member_value, dict): + new_offset = parent_offset + member_value.get("offset", 0) + if member_value.get("anonymous", False) and isinstance(member_value, dict): # Gets the subtype from the json ISF and recursively reduce its fields - subtype = self._json_object['user_types'].get(member_value['type']['name'], {}) - yield from self._reduce_fields(subtype['fields'], new_offset) + subtype = self._json_object["user_types"].get( + member_value["type"]["name"], {} + ) + yield from self._reduce_fields(subtype["fields"], new_offset) else: yield new_offset, member_name, member_value @@ -658,24 +779,28 @@ class Version8Format(Version7Format): """Resolves an individual symbol.""" index = type_name.find(constants.BANG) if index != -1: - table_name, type_name = type_name[:index], type_name[index + 1:] + table_name, type_name = type_name[:index], type_name[index + 1 :] raise exceptions.SymbolError( - type_name, table_name, - f"Symbol for a different table requested: {table_name + constants.BANG + type_name}") + type_name, + table_name, + f"Symbol for a different table requested: {table_name + constants.BANG + type_name}", + ) - type_definition = self._json_object['user_types'].get(type_name) + type_definition = self._json_object["user_types"].get(type_name) if type_definition is None: # Fall back to the natives table return self.natives.get_type(self.name + constants.BANG + type_name) - members = self._process_fields(type_definition['fields']) + members = self._process_fields(type_definition["fields"]) object_class = self.get_type_class(type_name) if object_class == objects.AggregateType: for clazz in objects.AggregateTypes: - if objects.AggregateTypes[clazz] == type_definition['kind']: + if objects.AggregateTypes[clazz] == type_definition["kind"]: object_class = clazz - return objects.templates.ObjectTemplate(type_name = self.name + constants.BANG + type_name, - object_class = object_class, - size = type_definition['size'], - members = members) + return objects.templates.ObjectTemplate( + type_name=self.name + constants.BANG + type_name, + object_class=object_class, + size=type_definition["size"], + members=members, + ) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index d59a95db5..739de120c 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -17,27 +17,27 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): super().__init__(*args, **kwargs) # Set-up Linux specific types - self.set_type_class('file', extensions.struct_file) - self.set_type_class('list_head', extensions.list_head) - self.set_type_class('mm_struct', extensions.mm_struct) - self.set_type_class('super_block', extensions.super_block) - self.set_type_class('task_struct', extensions.task_struct) - self.set_type_class('vm_area_struct', extensions.vm_area_struct) - self.set_type_class('qstr', extensions.qstr) - self.set_type_class('dentry', extensions.dentry) - self.set_type_class('fs_struct', extensions.fs_struct) - self.set_type_class('files_struct', extensions.files_struct) - self.set_type_class('vfsmount', extensions.vfsmount) - self.set_type_class('kobject', extensions.kobject) + self.set_type_class("file", extensions.struct_file) + self.set_type_class("list_head", extensions.list_head) + self.set_type_class("mm_struct", extensions.mm_struct) + self.set_type_class("super_block", extensions.super_block) + self.set_type_class("task_struct", extensions.task_struct) + self.set_type_class("vm_area_struct", extensions.vm_area_struct) + self.set_type_class("qstr", extensions.qstr) + self.set_type_class("dentry", extensions.dentry) + self.set_type_class("fs_struct", extensions.fs_struct) + self.set_type_class("files_struct", extensions.files_struct) + self.set_type_class("vfsmount", extensions.vfsmount) + self.set_type_class("kobject", extensions.kobject) - if 'mnt_namespace' in self.types: - self.set_type_class('mnt_namespace', extensions.mnt_namespace) + if "mnt_namespace" in self.types: + self.set_type_class("mnt_namespace", extensions.mnt_namespace) - if 'module' in self.types: - self.set_type_class('module', extensions.module) + if "module" in self.types: + self.set_type_class("module", extensions.module) - if 'mount' in self.types: - self.set_type_class('mount', extensions.mount) + if "mount" in self.types: + self.set_type_class("mount", extensions.mount) class LinuxUtilities(interfaces.configuration.VersionableInterface): @@ -59,7 +59,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): if dname == "": break - ret_path.insert(0, dname.strip('/')) + ret_path.insert(0, dname.strip("/")) if dentry == vfsmnt.get_mnt_root() or dentry == dentry.d_parent: if vfsmnt.get_mnt_parent() == vfsmnt: break @@ -79,7 +79,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): if not ret_path: return "" - ret_val = '/'.join([str(p) for p in ret_path if p != ""]) + ret_val = "/".join([str(p) for p in ret_path if p != ""]) if ret_val.startswith(("socket:", "pipe:")): if ret_val.find("]") == -1: @@ -94,7 +94,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ret_val = ret_val.replace("/", "") elif ret_val != "inotify": - ret_val = '/' + ret_val + ret_val = "/" + ret_val return ret_val @@ -169,7 +169,11 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): # TODO COMPARE THIS IN LSOF OUTPUT TO VOL2 try: - if dentry.d_op and dentry.d_op.has_member("d_dname") and dentry.d_op.d_dname: + if ( + dentry.d_op + and dentry.d_op.has_member("d_dname") + and dentry.d_op.d_dname + ): dname_is_valid = True except exceptions.InvalidAddressException: @@ -183,8 +187,12 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return ret @classmethod - def files_descriptors_for_process(cls, context: interfaces.context.ContextInterface, symbol_table: str, - task: interfaces.objects.ObjectInterface): + def files_descriptors_for_process( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + task: interfaces.objects.ObjectInterface, + ): fd_table = task.files.get_fds() if fd_table == 0: @@ -196,9 +204,11 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): if max_fds > 500000: return - file_type = symbol_table + constants.BANG + 'file' + file_type = symbol_table + constants.BANG + "file" - fds = objects.utility.array_of_pointers(fd_table, count = max_fds, subtype = file_type, context = context) + fds = objects.utility.array_of_pointers( + fd_table, count=max_fds, subtype=file_type, context=context + ) for (fd_num, filp) in enumerate(fds): if filp != 0: @@ -207,20 +217,33 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): yield fd_num, filp, full_path @classmethod - def mask_mods_list(cls, context: interfaces.context.ContextInterface, layer_name: str, - mods: Iterator[interfaces.objects.ObjectInterface]) -> List[Tuple[str, int, int]]: + def mask_mods_list( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + mods: Iterator[interfaces.objects.ObjectInterface], + ) -> List[Tuple[str, int, int]]: """ A helper function to mask the starting and end address of kernel modules """ mask = context.layers[layer_name].address_mask - return [(utility.array_to_string(mod.name), mod.get_module_base() & mask, - (mod.get_module_base() & mask) + mod.get_core_size()) for mod in mods] + return [ + ( + utility.array_to_string(mod.name), + mod.get_module_base() & mask, + (mod.get_module_base() & mask) + mod.get_core_size(), + ) + for mod in mods + ] @classmethod def generate_kernel_handler_info( - cls, context: interfaces.context.ContextInterface, kernel_module_name: str, - mods_list: Iterator[interfaces.objects.ObjectInterface]) -> List[Tuple[str, int, int]]: + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + mods_list: Iterator[interfaces.objects.ObjectInterface], + ) -> List[Tuple[str, int, int]]: """ A helper function that gets the beginning and end address of the kernel module """ @@ -235,13 +258,17 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): end_addr = kernel.object_from_symbol("_etext") end_addr = end_addr.vol.offset & mask - return [(constants.linux.KERNEL_NAME, start_addr, end_addr)] + \ - LinuxUtilities.mask_mods_list(context, kernel.layer_name, mods_list) + return [ + (constants.linux.KERNEL_NAME, start_addr, end_addr) + ] + LinuxUtilities.mask_mods_list(context, kernel.layer_name, mods_list) @classmethod - def lookup_module_address(cls, kernel_module: interfaces.context.ModuleInterface, - handlers: List[Tuple[str, int, int]], - target_address: int): + def lookup_module_address( + cls, + kernel_module: interfaces.context.ModuleInterface, + handlers: List[Tuple[str, int, int]], + target_address: int, + ): """ Searches between the start and end address of the kernel module using target_address. Returns the module and symbol name of the address provided. @@ -254,11 +281,16 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): if start <= target_address <= end: mod_name = name if name == constants.linux.KERNEL_NAME: - symbols = list(kernel_module.get_symbols_by_absolute_location(target_address)) + symbols = list( + kernel_module.get_symbols_by_absolute_location(target_address) + ) if len(symbols): - symbol_name = symbols[0].split(constants.BANG)[1] if constants.BANG in symbols[0] else \ - symbols[0] + symbol_name = ( + symbols[0].split(constants.BANG)[1] + if constants.BANG in symbols[0] + else symbols[0] + ) break @@ -267,6 +299,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): @classmethod def walk_internal_list(cls, vmlinux, struct_name, list_member, list_start): while list_start: - list_struct = vmlinux.object(object_type = struct_name, offset = list_start.vol.offset) + list_struct = vmlinux.object( + object_type=struct_name, offset=list_start.vol.offset + ) yield list_struct list_start = getattr(list_struct, list_member) diff --git a/volatility3/framework/symbols/linux/bash.py b/volatility3/framework/symbols/linux/bash.py index f1f40ace1..9df2bb255 100644 --- a/volatility3/framework/symbols/linux/bash.py +++ b/volatility3/framework/symbols/linux/bash.py @@ -7,8 +7,7 @@ from volatility3.framework.symbols.linux.extensions import bash class BashIntermedSymbols(intermed.IntermediateSymbolTable): - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.set_type_class('hist_entry', bash.hist_entry) + self.set_type_class("hist_entry", bash.hist_entry) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index ce002b905..d230607f5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -20,7 +20,6 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): - def get_module_base(self): if self.has_member("core_layout"): return self.core_layout.base @@ -34,7 +33,9 @@ class module(generic.GenericIntelProcess): elif self.has_member("init_size"): return self.init_size - raise AttributeError("module -> get_init_size: Unable to determine .init section size of module") + raise AttributeError( + "module -> get_init_size: Unable to determine .init section size of module" + ) def get_core_size(self): if self.has_member("core_layout"): @@ -43,7 +44,9 @@ class module(generic.GenericIntelProcess): elif self.has_member("core_size"): return self.core_size - raise AttributeError("module -> get_core_size: Unable to determine core size of module") + raise AttributeError( + "module -> get_core_size: Unable to determine core size of module" + ) def get_module_core(self): if self.has_member("core_layout"): @@ -62,17 +65,20 @@ class module(generic.GenericIntelProcess): raise AttributeError("module -> get_module_core: Unable to get module init") def get_name(self): - """ Get the name of the module as a string """ + """Get the name of the module as a string""" return utility.array_to_string(self.name) def _get_sect_count(self, grp): - """ Try to determine the number of valid sections """ + """Try to determine the number of valid sections""" arr = self._context.object( self.get_symbol_table().name + constants.BANG + "array", - layer_name = self.vol.layer_name, - offset = grp.attrs, - subtype = self._context.symbol_space.get_type(self.get_symbol_table().name + constants.BANG + "pointer"), - count = 25) + layer_name=self.vol.layer_name, + offset=grp.attrs, + subtype=self._context.symbol_space.get_type( + self.get_symbol_table().name + constants.BANG + "pointer" + ), + count=25, + ) idx = 0 while arr[idx]: @@ -81,18 +87,21 @@ class module(generic.GenericIntelProcess): return idx def get_sections(self): - """ Get sections of the module """ + """Get sections of the module""" if self.sect_attrs.has_member("nsections"): num_sects = self.sect_attrs.nsections else: num_sects = self._get_sect_count(self.sect_attrs.grp) - arr = self._context.object(self.get_symbol_table().name + constants.BANG + "array", - layer_name = self.vol.layer_name, - offset = self.sect_attrs.attrs.vol.offset, - subtype = self._context.symbol_space.get_type(self.get_symbol_table().name + - constants.BANG + 'module_sect_attr'), - count = num_sects) + arr = self._context.object( + self.get_symbol_table().name + constants.BANG + "array", + layer_name=self.vol.layer_name, + offset=self.sect_attrs.attrs.vol.offset, + subtype=self._context.symbol_space.get_type( + self.get_symbol_table().name + constants.BANG + "module_sect_attr" + ), + count=num_sects, + ) for attr in arr: yield attr @@ -103,26 +112,31 @@ class module(generic.GenericIntelProcess): else: prefix = "Elf32_" - elf_table_name = intermed.IntermediateSymbolTable.create(self.context, - self.config_path, - "linux", - "elf", - native_types = None, - class_types = elf.class_types) + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "linux", + "elf", + native_types=None, + class_types=elf.class_types, + ) syms = self._context.object( self.get_symbol_table().name + constants.BANG + "array", - layer_name = self.vol.layer_name, - offset = self.section_symtab, - subtype = self._context.symbol_space.get_type(elf_table_name + constants.BANG + prefix + "Sym"), - count = self.num_symtab + 1) + layer_name=self.vol.layer_name, + offset=self.section_symtab, + subtype=self._context.symbol_space.get_type( + elf_table_name + constants.BANG + prefix + "Sym" + ), + count=self.num_symtab + 1, + ) if self.section_strtab: for sym in syms: sym.set_cached_strtab(self.section_strtab) yield sym def get_symbol(self, wanted_sym_name): - """ Get value for a given symbol name """ + """Get value for a given symbol name""" for sym in self.get_symbols(): sym_name = sym.get_name() sym_addr = sym.st_value @@ -146,7 +160,9 @@ class module(generic.GenericIntelProcess): elif self.has_member("num_symtab"): return int(self.num_symtab) - raise AttributeError("module -> num_symtab: Unable to determine number of symbols") + raise AttributeError( + "module -> num_symtab: Unable to determine number of symbols" + ) @property def section_strtab(self): @@ -161,8 +177,9 @@ class module(generic.GenericIntelProcess): class task_struct(generic.GenericIntelProcess): - - def add_process_layer(self, config_prefix: str = None, preferred_name: str = None) -> Optional[str]: + def add_process_layer( + self, config_prefix: str = None, preferred_name: str = None + ) -> Optional[str]: """Constructs a new layer based on the process's DTB. Returns the name of the Layer or None. @@ -175,7 +192,9 @@ class task_struct(generic.GenericIntelProcess): return None if not isinstance(parent_layer, linear.LinearlyMappedLayer): - raise TypeError("Parent layer is not a translation layer, unable to construct process layer") + raise TypeError( + "Parent layer is not a translation layer, unable to construct process layer" + ) dtb, layer_name = parent_layer.translate(pgd) if not dtb: @@ -185,9 +204,13 @@ class task_struct(generic.GenericIntelProcess): preferred_name = self.vol.layer_name + f"_Process{self.pid}" # Add the constructed layer and return the name - return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) + return self._add_process_layer( + self._context, dtb, config_prefix, preferred_name + ) - def get_process_memory_sections(self, heap_only: bool = False) -> Generator[Tuple[int, int], None, None]: + def get_process_memory_sections( + self, heap_only: bool = False + ) -> Generator[Tuple[int, int], None, None]: """Returns a list of sections based on the memory manager's view of this task's virtual memory.""" for vma in self.mm.get_mmap_iter(): @@ -198,7 +221,9 @@ class task_struct(generic.GenericIntelProcess): continue else: # FIXME: Check if this actually needs to be printed out or not - vollog.info(f"adding vma: {start:x} {self.mm.brk:x} | {end:x} {self.mm.start_brk:x}") + vollog.info( + f"adding vma: {start:x} {self.mm.brk:x} | {end:x} {self.mm.start_brk:x}" + ) yield (start, end - start) @@ -240,13 +265,12 @@ class task_struct(generic.GenericIntelProcess): # threads and using the thread_group offset to get the # corresponding task_struct for task in self.thread_group.to_list( - f"{task_symbol_table_name}{constants.BANG}task_struct", - "thread_group" + f"{task_symbol_table_name}{constants.BANG}task_struct", "thread_group" ): yield task -class fs_struct(objects.StructType): +class fs_struct(objects.StructType): def get_root_dentry(self): # < 2.6.26 if self.has_member("rootmnt"): @@ -267,7 +291,6 @@ class fs_struct(objects.StructType): class mm_struct(objects.StructType): - def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct.""" @@ -290,26 +313,26 @@ class super_block(objects.StructType): MINORBITS = 20 # Superblock flags - SB_RDONLY = 1 # Mount read-only - SB_NOSUID = 2 # Ignore suid and sgid bits - SB_NODEV = 4 # Disallow access to device special files - SB_NOEXEC = 8 # Disallow program execution - SB_SYNCHRONOUS = 16 # Writes are synced at once - SB_MANDLOCK = 64 # Allow mandatory locks on an FS - SB_DIRSYNC = 128 # Directory modifications are synchronous - SB_NOATIME = 1024 # Do not update access times - SB_NODIRATIME = 2048 # Do not update directory access times + SB_RDONLY = 1 # Mount read-only + SB_NOSUID = 2 # Ignore suid and sgid bits + SB_NODEV = 4 # Disallow access to device special files + SB_NOEXEC = 8 # Disallow program execution + SB_SYNCHRONOUS = 16 # Writes are synced at once + SB_MANDLOCK = 64 # Allow mandatory locks on an FS + SB_DIRSYNC = 128 # Directory modifications are synchronous + SB_NOATIME = 1024 # Do not update access times + SB_NODIRATIME = 2048 # Do not update directory access times SB_SILENT = 32768 - SB_POSIXACL = (1 << 16) # VFS does not apply the umask - SB_KERNMOUNT = (1 << 22) # this is a kern_mount call - SB_I_VERSION = (1 << 23) # Update inode I_version field - SB_LAZYTIME = (1 << 25) # Update the on-disk [acm]times lazily + SB_POSIXACL = 1 << 16 # VFS does not apply the umask + SB_KERNMOUNT = 1 << 22 # this is a kern_mount call + SB_I_VERSION = 1 << 23 # Update inode I_version field + SB_LAZYTIME = 1 << 25 # Update the on-disk [acm]times lazily SB_OPTS = { SB_SYNCHRONOUS: "sync", SB_DIRSYNC: "dirsync", SB_MANDLOCK: "mand", - SB_LAZYTIME: "lazytime" + SB_LAZYTIME: "lazytime", } @property @@ -321,10 +344,12 @@ class super_block(objects.StructType): return self.s_dev & ((1 << self.MINORBITS) - 1) def get_flags_access(self) -> str: - return 'ro' if self.s_flags & self.SB_RDONLY else 'rw' + return "ro" if self.s_flags & self.SB_RDONLY else "rw" def get_flags_opts(self) -> Iterable[str]: - sb_opts = [self.SB_OPTS[sb_opt] for sb_opt in self.SB_OPTS if sb_opt & self.s_flags] + sb_opts = [ + self.SB_OPTS[sb_opt] for sb_opt in self.SB_OPTS if sb_opt & self.s_flags + ] return sb_opts def get_type(self): @@ -387,7 +412,7 @@ class vm_area_struct(objects.StructType): if (vm_flags & mask) == mask: retval = retval + char else: - retval = retval + '-' + retval = retval + "-" return retval @@ -412,7 +437,10 @@ class vm_area_struct(objects.StructType): fname = "[heap]" elif self.vm_start <= task.mm.start_stack <= self.vm_end: fname = "[stack]" - elif self.vm_mm.context.has_member("vdso") and self.vm_start == self.vm_mm.context.vdso: + elif ( + self.vm_mm.context.has_member("vdso") + and self.vm_start == self.vm_mm.context.vdso + ): fname = "[vdso]" else: fname = "Anonymous Mapping" @@ -435,7 +463,6 @@ class vm_area_struct(objects.StructType): class qstr(objects.StructType): - def name_as_str(self) -> str: if self.has_member("len"): str_length = self.len + 1 # Maximum length should include null terminator @@ -451,14 +478,15 @@ class qstr(objects.StructType): class dentry(objects.StructType): - def path(self) -> str: """Based on __dentry_path Linux kernel function""" reversed_path = [] dentry_seen = set() current_dentry = self - while (not current_dentry.is_root() and - current_dentry.vol.offset not in dentry_seen): + while ( + not current_dentry.is_root() + and current_dentry.vol.offset not in dentry_seen + ): parent = current_dentry.d_parent reversed_path.append(current_dentry.d_name.name_as_str()) dentry_seen.add(current_dentry.vol.offset) @@ -488,8 +516,10 @@ class dentry(objects.StructType): dentry_seen = set() current_dentry = self - while (not current_dentry.is_root() and - current_dentry.vol.offset not in dentry_seen): + while ( + not current_dentry.is_root() + and current_dentry.vol.offset not in dentry_seen + ): if current_dentry.d_parent == ancestor_dentry.vol.offset: return current_dentry @@ -500,7 +530,6 @@ class dentry(objects.StructType): class struct_file(objects.StructType): - def get_dentry(self) -> interfaces.objects.ObjectInterface: if self.has_member("f_dentry"): return self.f_dentry @@ -519,13 +548,14 @@ class struct_file(objects.StructType): class list_head(objects.StructType, collections.abc.Iterable): - - def to_list(self, - symbol_type: str, - member: str, - forward: bool = True, - sentinel: bool = True, - layer: Optional[str] = None) -> Iterator[interfaces.objects.ObjectInterface]: + def to_list( + self, + symbol_type: str, + member: str, + forward: bool = True, + sentinel: bool = True, + layer: Optional[str] = None, + ) -> Iterator[interfaces.objects.ObjectInterface]: """Returns an iterator of the entries in the list. Args: @@ -541,23 +571,29 @@ class list_head(objects.StructType, collections.abc.Iterable): """ layer = layer or self.vol.layer_name - relative_offset = self._context.symbol_space.get_type(symbol_type).relative_child_offset(member) + relative_offset = self._context.symbol_space.get_type( + symbol_type + ).relative_child_offset(member) - direction = 'prev' + direction = "prev" if forward: - direction = 'next' + direction = "next" try: link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: return if not sentinel: - yield self._context.object(symbol_type, layer, offset = self.vol.offset - relative_offset) + yield self._context.object( + symbol_type, layer, offset=self.vol.offset - relative_offset + ) seen = {self.vol.offset} while link.vol.offset not in seen: - obj = self._context.object(symbol_type, layer, offset = link.vol.offset - relative_offset) + obj = self._context.object( + symbol_type, layer, offset=link.vol.offset - relative_offset + ) yield obj seen.add(link.vol.offset) @@ -571,7 +607,6 @@ class list_head(objects.StructType, collections.abc.Iterable): class files_struct(objects.StructType): - def get_fds(self) -> interfaces.objects.ObjectInterface: if self.has_member("fdt"): return self.fdt.fd.dereference() @@ -646,7 +681,11 @@ class mount(objects.StructType): return "ro" if self.get_mnt_flags() & self.MNT_READONLY else "rw" def get_flags_opts(self) -> Iterable[str]: - flags = [self.MNT_FLAGS[mntflag] for mntflag in self.MNT_FLAGS if mntflag & self.get_mnt_flags()] + flags = [ + self.MNT_FLAGS[mntflag] + for mntflag in self.MNT_FLAGS + if mntflag & self.get_mnt_flags() + ] return flags def is_shared(self) -> bool: @@ -668,9 +707,11 @@ class mount(objects.StructType): """Get ID of closest dominating peer group having a representative under the given root.""" mnt_seen = set() current_mnt = self.mnt_master - while (current_mnt and - current_mnt.vol.offset != 0 and - current_mnt.vol.offset not in mnt_seen): + while ( + current_mnt + and current_mnt.vol.offset != 0 + and current_mnt.vol.offset not in mnt_seen + ): peer = current_mnt.get_peer_under_root(self.mnt_ns, root) if peer and peer.vol.offset != 0: return peer.mnt_group_id @@ -686,7 +727,9 @@ class mount(objects.StructType): mnt_seen = set() current_mnt = self while current_mnt.vol.offset not in mnt_seen: - if current_mnt.mnt_ns == ns and current_mnt.is_path_reachable(current_mnt.mnt.mnt_root, root): + if current_mnt.mnt_ns == ns and current_mnt.is_path_reachable( + current_mnt.mnt.mnt_root, root + ): return current_mnt mnt_seen.add(current_mnt.vol.offset) @@ -702,36 +745,52 @@ class mount(objects.StructType): """ mnt_seen = set() current_mnt = self - while (current_mnt.mnt.vol.offset != root.mnt and - current_mnt.has_parent() and - current_mnt.vol.offset not in mnt_seen): + while ( + current_mnt.mnt.vol.offset != root.mnt + and current_mnt.has_parent() + and current_mnt.vol.offset not in mnt_seen + ): current_dentry = current_mnt.mnt_mountpoint mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.mnt_parent - return current_mnt.mnt.vol.offset == root.mnt and current_dentry.is_subdir(root.dentry) + return current_mnt.mnt.vol.offset == root.mnt and current_dentry.is_subdir( + root.dentry + ) def next_peer(self): table_name = self.vol.type_name.split(constants.BANG)[0] mount_struct = "{0}{1}mount".format(table_name, constants.BANG) - offset = self._context.symbol_space.get_type(mount_struct).relative_child_offset("mnt_share") + offset = self._context.symbol_space.get_type( + mount_struct + ).relative_child_offset("mnt_share") + + return self._context.object( + mount_struct, + self.vol.layer_name, + offset=self.mnt_share.next.vol.offset - offset, + ) - return self._context.object(mount_struct, self.vol.layer_name, offset=self.mnt_share.next.vol.offset - offset) class vfsmount(objects.StructType): - def is_valid(self): - return self.get_mnt_sb() != 0 and \ - self.get_mnt_root() != 0 and \ - self.get_mnt_parent() != 0 + return ( + self.get_mnt_sb() != 0 + and self.get_mnt_root() != 0 + and self.get_mnt_parent() != 0 + ) def _get_real_mnt(self): table_name = self.vol.type_name.split(constants.BANG)[0] mount_struct = f"{table_name}{constants.BANG}mount" - offset = self._context.symbol_space.get_type(mount_struct).relative_child_offset("mnt") + offset = self._context.symbol_space.get_type( + mount_struct + ).relative_child_offset("mnt") - return self._context.object(mount_struct, self.vol.layer_name, offset = self.vol.offset - offset) + return self._context.object( + mount_struct, self.vol.layer_name, offset=self.vol.offset - offset + ) def get_mnt_parent(self): if self.has_member("mnt_parent"): @@ -750,7 +809,6 @@ class vfsmount(objects.StructType): class kobject(objects.StructType): - def reference_count(self): refcnt = self.kref.refcount if self.has_member("counter"): @@ -760,6 +818,7 @@ class kobject(objects.StructType): return ret + class mnt_namespace(objects.StructType): def get_inode(self): if self.has_member("proc_inum"): diff --git a/volatility3/framework/symbols/linux/extensions/bash.py b/volatility3/framework/symbols/linux/extensions/bash.py index 40fa2521d..29ecb4f76 100644 --- a/volatility3/framework/symbols/linux/extensions/bash.py +++ b/volatility3/framework/symbols/linux/extensions/bash.py @@ -9,7 +9,6 @@ from volatility3.framework.renderers import conversion class hist_entry(objects.StructType): - def is_valid(self): try: cmd = self.get_command() diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 1277afe93..df6b23df8 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -9,36 +9,47 @@ from volatility3.framework import objects, interfaces class elf(objects.StructType): - ''' + """ Class used to create elf objects. It overrides the typename to `Elf32_` or `Elf64_`, depending on the corresponding value on e_ident - ''' + """ - def __init__(self, context: interfaces.context.ContextInterface, type_name: str, - object_info: interfaces.objects.ObjectInformation, size: int, - members: Dict[str, Tuple[int, interfaces.objects.Template]]) -> None: + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + size: int, + members: Dict[str, Tuple[int, interfaces.objects.Template]], + ) -> None: - super().__init__(context = context, - type_name = type_name, - object_info = object_info, - size = size, - members = members) + super().__init__( + context=context, + type_name=type_name, + object_info=object_info, + size=size, + members=members, + ) layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() # We read the MAGIC: (0x0 to 0x4) 0x7f 0x45 0x4c 0x46 - magic = self._context.object(symbol_table_name + constants.BANG + "unsigned long", - layer_name = layer_name, - offset = object_info.offset) + magic = self._context.object( + symbol_table_name + constants.BANG + "unsigned long", + layer_name=layer_name, + offset=object_info.offset, + ) # Check validity - if magic != 0x464c457f: + if magic != 0x464C457F: return None # We need to read the EI_CLASS (0x4 offset) - ei_class = self._context.object(symbol_table_name + constants.BANG + "unsigned char", - layer_name = layer_name, - offset = object_info.offset + 0x4) + ei_class = self._context.object( + symbol_table_name + constants.BANG + "unsigned char", + layer_name=layer_name, + offset=object_info.offset + 0x4, + ) if ei_class == 1: self._type_prefix = "Elf32_" @@ -48,18 +59,20 @@ class elf(objects.StructType): raise ValueError(f"Unsupported ei_class value {ei_class}") # Construct the full header - self._hdr = self._context.object(symbol_table_name + constants.BANG + self._type_prefix + "Ehdr", - layer_name = layer_name, - offset = object_info.offset) + self._hdr = self._context.object( + symbol_table_name + constants.BANG + self._type_prefix + "Ehdr", + layer_name=layer_name, + offset=object_info.offset, + ) self._offset = object_info.offset self._cached_symtab = None self._cached_strtab = None def is_valid(self): - ''' + """ Determine whether it is a valid object - ''' + """ return self._type_prefix is not None and self._hdr is not None def __getattr__(self, name): @@ -71,17 +84,26 @@ class elf(objects.StructType): def __dir__(self): return self._hdr.__dir__() + [ - "get_program_headers", "is_valid", "get_section_headers", "get_symbols", "__dir__" + "get_program_headers", + "is_valid", + "get_section_headers", + "get_symbols", + "__dir__", ] def get_program_headers(self): program_headers = self._context.object( self.get_symbol_table_name() + constants.BANG + "array", - layer_name = self.vol.layer_name, - offset = self._offset + self.e_phoff, - subtype = self._context.symbol_space.get_type(self.get_symbol_table_name() + constants.BANG + - self._type_prefix + "Phdr"), - count = self.e_phnum) + layer_name=self.vol.layer_name, + offset=self._offset + self.e_phoff, + subtype=self._context.symbol_space.get_type( + self.get_symbol_table_name() + + constants.BANG + + self._type_prefix + + "Phdr" + ), + count=self.e_phnum, + ) for prog_header in program_headers: prog_header.parent_e_type = self.e_type @@ -92,11 +114,16 @@ class elf(objects.StructType): def get_section_headers(self): section_headers = self._context.object( self.get_symbol_table_name() + constants.BANG + "array", - layer_name = self.vol.layer_name, - offset = self._offset + self.e_shoff, - subtype = self._context.symbol_space.get_type(self.get_symbol_table_name() + constants.BANG + - self._type_prefix + "Shdr"), - count = self.e_shnum) + layer_name=self.vol.layer_name, + offset=self._offset + self.e_shoff, + subtype=self._context.symbol_space.get_type( + self.get_symbol_table_name() + + constants.BANG + + self._type_prefix + + "Shdr" + ), + count=self.e_shnum, + ) return section_headers def _find_symbols(self): @@ -107,7 +134,7 @@ class elf(objects.StructType): for phdr in self.get_program_headers(): try: # Find PT_DYNAMIC segment - if str(phdr.p_type.description) != 'PT_DYNAMIC': + if str(phdr.p_type.description) != "PT_DYNAMIC": continue except ValueError: # If the p_type value is outside the ones declared in the enumeration, an @@ -149,11 +176,16 @@ class elf(objects.StructType): symtab_arr = self._context.object( self.get_symbol_table_name() + constants.BANG + "array", - layer_name = self.vol.layer_name, - offset = self._cached_symtab, - subtype = self._context.symbol_space.get_type(self.get_symbol_table_name() + constants.BANG + - self._type_prefix + "Sym"), - count = self._cached_numsyms) + layer_name=self.vol.layer_name, + offset=self._cached_symtab, + subtype=self._context.symbol_space.get_type( + self.get_symbol_table_name() + + constants.BANG + + self._type_prefix + + "Sym" + ), + count=self._cached_numsyms, + ) for sym in symtab_arr: sym.cached_strtab = self._cached_strtab @@ -161,7 +193,7 @@ class elf(objects.StructType): class elf_sym(objects.StructType): - """ An elf symbol entry""" + """An elf symbol entry""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -179,13 +211,13 @@ class elf_sym(objects.StructType): addr = self._cached_strtab + self.st_name # Just get the first 255 characters, it should be enough for a symbol name - name_bytes = self._context.layers[self.vol.layer_name].read(addr, 255, pad = True) + name_bytes = self._context.layers[self.vol.layer_name].read(addr, 255, pad=True) if name_bytes: idx = name_bytes.find(b"\x00") if idx != -1: name_bytes = name_bytes[:idx] - return name_bytes.decode('utf-8', errors = 'ignore') + return name_bytes.decode("utf-8", errors="ignore") else: # If we cannot read the name from the address space, # we return None. @@ -193,7 +225,7 @@ class elf_sym(objects.StructType): class elf_phdr(objects.StructType): - """ An elf program header """ + """An elf program header""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -236,7 +268,7 @@ class elf_phdr(objects.StructType): def dynamic_sections(self): # sanity check try: - if str(self.p_type.description) != 'PT_DYNAMIC': + if str(self.p_type.description) != "PT_DYNAMIC": return None except ValueError: # If the value is outside the ones declared in the enumeration, an @@ -248,17 +280,19 @@ class elf_phdr(objects.StructType): symbol_table_name = self.get_symbol_table_name() - rtsize = self._context.symbol_space.get_type(symbol_table_name + \ - constants.BANG + \ - self._type_prefix + "Dyn").size + rtsize = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + self._type_prefix + "Dyn" + ).size for i in range(256): # use the real size idx = i * rtsize - dyn = self._context.object(symbol_table_name + constants.BANG + self._type_prefix + "Dyn", - layer_name = self.vol.layer_name, - offset = arr_start + idx) + dyn = self._context.object( + symbol_table_name + constants.BANG + self._type_prefix + "Dyn", + layer_name=self.vol.layer_name, + offset=arr_start + idx, + ) yield dyn @@ -266,4 +300,10 @@ class elf_phdr(objects.StructType): break -class_types = {'Elf': elf, 'Elf64_Phdr': elf_phdr, 'Elf32_Phdr': elf_phdr, 'Elf32_Sym': elf_sym, 'Elf64_Sym': elf_sym} +class_types = { + "Elf": elf, + "Elf64_Phdr": elf_phdr, + "Elf32_Phdr": elf_phdr, + "Elf32_Sym": elf_sym, + "Elf64_Sym": elf_sym, +} diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index 7d094ea4f..3909817ea 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -14,23 +14,24 @@ class MacKernelIntermedSymbols(intermed.IntermediateSymbolTable): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.set_type_class('proc', extensions.proc) - self.set_type_class('fileglob', extensions.fileglob) - self.set_type_class('vnode', extensions.vnode) - self.set_type_class('vm_map_entry', extensions.vm_map_entry) - self.set_type_class('vm_map_object', extensions.vm_map_object) - self.set_type_class('socket', extensions.socket) - self.set_type_class('inpcb', extensions.inpcb) - self.set_type_class('queue_entry', extensions.queue_entry) - self.set_type_class('ifnet', extensions.ifnet) - self.set_type_class('sockaddr_dl', extensions.sockaddr_dl) - self.set_type_class('sockaddr', extensions.sockaddr) - self.set_type_class('sysctl_oid', extensions.sysctl_oid) - self.set_type_class('kauth_scope', extensions.kauth_scope) + self.set_type_class("proc", extensions.proc) + self.set_type_class("fileglob", extensions.fileglob) + self.set_type_class("vnode", extensions.vnode) + self.set_type_class("vm_map_entry", extensions.vm_map_entry) + self.set_type_class("vm_map_object", extensions.vm_map_object) + self.set_type_class("socket", extensions.socket) + self.set_type_class("inpcb", extensions.inpcb) + self.set_type_class("queue_entry", extensions.queue_entry) + self.set_type_class("ifnet", extensions.ifnet) + self.set_type_class("sockaddr_dl", extensions.sockaddr_dl) + self.set_type_class("sockaddr", extensions.sockaddr) + self.set_type_class("sysctl_oid", extensions.sysctl_oid) + self.set_type_class("kauth_scope", extensions.kauth_scope) class MacUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful mac functions.""" + """ Version History: 1.1.0 -> added walk_list_head API @@ -41,23 +42,34 @@ class MacUtilities(interfaces.configuration.VersionableInterface): _required_framework_version = (2, 0, 0) @classmethod - def mask_mods_list(cls, context: interfaces.context.ContextInterface, layer_name: str, - mods: Iterator[Any]) -> List[Tuple[interfaces.objects.ObjectInterface, Any, Any]]: + def mask_mods_list( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + mods: Iterator[Any], + ) -> List[Tuple[interfaces.objects.ObjectInterface, Any, Any]]: """ A helper function to mask the starting and end address of kernel modules """ mask = context.layers[layer_name].address_mask - return [(objects.utility.array_to_string(mod.name), mod.address & mask, (mod.address & mask) + mod.size) - for mod in mods] + return [ + ( + objects.utility.array_to_string(mod.name), + mod.address & mask, + (mod.address & mask) + mod.size, + ) + for mod in mods + ] @classmethod def generate_kernel_handler_info( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - kernel, # ikelos - how to type this?? - mods_list: Iterator[Any]): + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + kernel, # ikelos - how to type this?? + mods_list: Iterator[Any], + ): try: start_addr = kernel.object_from_symbol("vm_kernel_stext") @@ -74,12 +86,18 @@ class MacUtilities(interfaces.configuration.VersionableInterface): start_addr = start_addr & mask end_addr = end_addr & mask - return [("__kernel__", start_addr, end_addr)] + \ - MacUtilities.mask_mods_list(context, layer_name, mods_list) + return [("__kernel__", start_addr, end_addr)] + MacUtilities.mask_mods_list( + context, layer_name, mods_list + ) @classmethod - def lookup_module_address(cls, context: interfaces.context.ContextInterface, handlers: Iterator[Any], - target_address, kernel_module_name: str = None): + def lookup_module_address( + cls, + context: interfaces.context.ContextInterface, + handlers: Iterator[Any], + target_address, + kernel_module_name: str = None, + ): mod_name = "UNKNOWN" symbol_name = "N/A" @@ -92,19 +110,30 @@ class MacUtilities(interfaces.configuration.VersionableInterface): if start <= target_address <= end: mod_name = name if name == "__kernel__": - symbols = list(context.symbol_space.get_symbols_by_location(target_address - module_shift)) + symbols = list( + context.symbol_space.get_symbols_by_location( + target_address - module_shift + ) + ) if len(symbols) > 0: - symbol_name = str(symbols[0].split(constants.BANG)[1]) if constants.BANG in symbols[0] else \ - str(symbols[0]) + symbol_name = ( + str(symbols[0].split(constants.BANG)[1]) + if constants.BANG in symbols[0] + else str(symbols[0]) + ) break return mod_name, symbol_name @classmethod - def files_descriptors_for_process(cls, context: interfaces.context.ContextInterface, symbol_table_name: str, - task: interfaces.objects.ObjectInterface): + def files_descriptors_for_process( + cls, + context: interfaces.context.ContextInterface, + symbol_table_name: str, + task: interfaces.objects.ObjectInterface, + ): """Creates a generator for the file descriptors of a process Args: @@ -136,14 +165,16 @@ class MacUtilities(interfaces.configuration.VersionableInterface): if num_fds > 4096: num_fds = 1024 - file_type = symbol_table_name + constants.BANG + 'fileproc' + file_type = symbol_table_name + constants.BANG + "fileproc" try: table_addr = task.p_fd.fd_ofiles.dereference() except exceptions.InvalidAddressException: return - fds = objects.utility.array_of_pointers(table_addr, count = num_fds, subtype = file_type, context = context) + fds = objects.utility.array_of_pointers( + table_addr, count=num_fds, subtype=file_type, context=context + ) for fd_num, f in enumerate(fds): if f != 0: @@ -152,7 +183,7 @@ class MacUtilities(interfaces.configuration.VersionableInterface): except exceptions.InvalidAddressException: continue - if ftype == 'VNODE': + if ftype == "VNODE": vnode = f.f_fglob.fg_data.dereference().cast("vnode") path = vnode.full_path() elif ftype: @@ -161,16 +192,18 @@ class MacUtilities(interfaces.configuration.VersionableInterface): yield f, path, fd_num @classmethod - def _walk_iterable(cls, - queue: interfaces.objects.ObjectInterface, - list_head_member: str, - list_next_member: str, - next_member: str, - max_elements: int = 4096) -> Iterable[interfaces.objects.ObjectInterface]: + def _walk_iterable( + cls, + queue: interfaces.objects.ObjectInterface, + list_head_member: str, + list_next_member: str, + next_member: str, + max_elements: int = 4096, + ) -> Iterable[interfaces.objects.ObjectInterface]: seen: Set[int] = set() try: - current = queue.member(attr = list_head_member) + current = queue.member(attr=list_head_member) except exceptions.InvalidAddressException: return @@ -187,33 +220,45 @@ class MacUtilities(interfaces.configuration.VersionableInterface): yield current try: - current = current.member(attr = next_member).member(attr = list_next_member) + current = current.member(attr=next_member).member(attr=list_next_member) except exceptions.InvalidAddressException: break @classmethod - def walk_tailq(cls, - queue: interfaces.objects.ObjectInterface, - next_member: str, - max_elements: int = 4096) -> Iterable[interfaces.objects.ObjectInterface]: + def walk_tailq( + cls, + queue: interfaces.objects.ObjectInterface, + next_member: str, + max_elements: int = 4096, + ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable(queue, "tqh_first", "tqe_next", next_member, max_elements): + for element in cls._walk_iterable( + queue, "tqh_first", "tqe_next", next_member, max_elements + ): yield element @classmethod - def walk_list_head(cls, - queue: interfaces.objects.ObjectInterface, - next_member: str, - max_elements: int = 4096) -> Iterable[interfaces.objects.ObjectInterface]: + def walk_list_head( + cls, + queue: interfaces.objects.ObjectInterface, + next_member: str, + max_elements: int = 4096, + ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable(queue, "lh_first", "le_next", next_member, max_elements): + for element in cls._walk_iterable( + queue, "lh_first", "le_next", next_member, max_elements + ): yield element @classmethod - def walk_slist(cls, - queue: interfaces.objects.ObjectInterface, - next_member: str, - max_elements: int = 4096) -> Iterable[interfaces.objects.ObjectInterface]: + def walk_slist( + cls, + queue: interfaces.objects.ObjectInterface, + next_member: str, + max_elements: int = 4096, + ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable(queue, "slh_first", "sle_next", next_member, max_elements): + for element in cls._walk_iterable( + queue, "slh_first", "sle_next", next_member, max_elements + ): yield element diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 45dc1db70..b678304b8 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -14,11 +14,12 @@ vollog = logging.getLogger(__name__) class proc(generic.GenericIntelProcess): - def get_task(self): return self.task.dereference().cast("task") - def add_process_layer(self, config_prefix: str = None, preferred_name: str = None) -> Optional[str]: + def add_process_layer( + self, config_prefix: str = None, preferred_name: str = None + ) -> Optional[str]: """Constructs a new layer based on the process's DTB. Returns the name of the Layer or None. @@ -26,7 +27,9 @@ class proc(generic.GenericIntelProcess): parent_layer = self._context.layers[self.vol.layer_name] if not isinstance(parent_layer, interfaces.layers.TranslationLayerInterface): - raise TypeError("Parent layer is not a translation layer, unable to construct process layer") + raise TypeError( + "Parent layer is not a translation layer, unable to construct process layer" + ) try: dtb = self.get_task().map.pmap.pm_cr3 @@ -38,7 +41,9 @@ class proc(generic.GenericIntelProcess): preferred_name = self.vol.layer_name + f"_Process{self.p_pid}" # Add the constructed layer and return the name - return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) + return self._add_process_layer( + self._context, dtb, config_prefix, preferred_name + ) def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: try: @@ -50,15 +55,25 @@ class proc(generic.GenericIntelProcess): seen: Set[int] = set() for i in range(task.map.hdr.nentries): - if (not current_map or - current_map.vol.offset in seen or - not self._context.layers[task.vol.native_layer_name].is_valid(current_map.dereference().vol.offset, - current_map.dereference().vol.size)): - vollog.log(constants.LOGLEVEL_VVV, "Breaking process maps iteration due to invalid state.") + if ( + not current_map + or current_map.vol.offset in seen + or not self._context.layers[task.vol.native_layer_name].is_valid( + current_map.dereference().vol.offset, + current_map.dereference().vol.size, + ) + ): + vollog.log( + constants.LOGLEVEL_VVV, + "Breaking process maps iteration due to invalid state.", + ) break # ZP_POISON value used to catch programming errors - if current_map.links.start == 0xdeadbeefdeadbeef or current_map.links.end == 0xdeadbeefdeadbeef: + if ( + current_map.links.start == 0xDEADBEEFDEADBEEF + or current_map.links.end == 0xDEADBEEFDEADBEEF + ): break yield current_map @@ -71,11 +86,12 @@ class proc(generic.GenericIntelProcess): # the fix for linux was to call int() so that we were not returning vol objects. # I call int() on these and the code works nearly 1-1 with the linux one so I am very confused ###### - def get_process_memory_sections(self, - context: interfaces.context.ContextInterface, - config_prefix: str, - rw_no_file: bool = False) -> \ - Generator[Tuple[int, int], None, None]: + def get_process_memory_sections( + self, + context: interfaces.context.ContextInterface, + config_prefix: str, + rw_no_file: bool = False, + ) -> Generator[Tuple[int, int], None, None]: """Returns a list of sections based on the memory manager's view of this task's virtual memory.""" for vma in self.get_map_iter(): @@ -83,7 +99,10 @@ class proc(generic.GenericIntelProcess): end = int(vma.links.end) if rw_no_file: - if vma.get_perms() != "rw" or vma.get_path(context, config_prefix) != "": + if ( + vma.get_perms() != "rw" + or vma.get_path(context, config_prefix) != "" + ): if vma.get_special_path() != "[heap]": continue @@ -91,7 +110,6 @@ class proc(generic.GenericIntelProcess): class fileglob(objects.StructType): - def get_fg_type(self): ret = None @@ -108,7 +126,6 @@ class fileglob(objects.StructType): class vm_map_object(objects.StructType): - def get_map_object(self): if self.has_member("vm_object"): return self.vm_object @@ -119,7 +136,6 @@ class vm_map_object(objects.StructType): class vnode(objects.StructType): - def _do_calc_path(self, ret, vnodeobj, vname): if vnodeobj is None: return @@ -132,7 +148,11 @@ class vnode(objects.StructType): if int(vnodeobj.v_flag) & 0x000001 != 0 and int(vnodeobj.v_mount) != 0: if int(vnodeobj.v_mount.mnt_vnodecovered) != 0: - self._do_calc_path(ret, vnodeobj.v_mount.mnt_vnodecovered, vnodeobj.v_mount.mnt_vnodecovered.v_name) + self._do_calc_path( + ret, + vnodeobj.v_mount.mnt_vnodecovered, + vnodeobj.v_mount.mnt_vnodecovered.v_name, + ) else: try: parent = vnodeobj.v_parent @@ -143,7 +163,11 @@ class vnode(objects.StructType): self._do_calc_path(ret, parent, parent_name) def full_path(self): - if self.v_flag & 0x000001 != 0 and self.v_mount != 0 and self.v_mount.mnt_flag & 0x00004000 != 0: + if ( + self.v_flag & 0x000001 != 0 + and self.v_mount != 0 + and self.v_mount.mnt_flag & 0x00004000 != 0 + ): ret = b"/" else: elements = [] @@ -163,7 +187,6 @@ class vnode(objects.StructType): class vm_map_entry(objects.StructType): - def is_suspicious(self, context, config_prefix): """Flags memory regions that are mapped rwx or that map an executable not back from a file on disk.""" @@ -195,7 +218,7 @@ class vm_map_entry(objects.StructType): if self.has_member("alias"): ret = int(self.alias) else: - ret = int(self.vme_offset) & 0xfff + ret = int(self.vme_offset) & 0xFFF return ret @@ -296,9 +319,11 @@ class vm_map_entry(objects.StructType): break if found: - vpager = context.object(config_prefix + constants.BANG + "vnode_pager", - layer_name = vnode_object.vol.native_layer_name, - offset = vnode_object.pager) + vpager = context.object( + config_prefix + constants.BANG + "vnode_pager", + layer_name=vnode_object.vol.native_layer_name, + offset=vnode_object.pager, + ) ret = vpager.vnode_handle else: ret = None @@ -307,7 +332,6 @@ class vm_map_entry(objects.StructType): class socket(objects.StructType): - def get_inpcb(self): try: ret = self.so_pcb.dereference().cast("inpcb") @@ -365,10 +389,20 @@ class socket(objects.StructType): class inpcb(objects.StructType): - def get_tcp_state(self): - tcp_states = ("CLOSED", "LISTEN", "SYN_SENT", "SYN_RECV", "ESTABLISHED", "CLOSE_WAIT", "FIN_WAIT1", "CLOSING", - "LAST_ACK", "FIN_WAIT2", "TIME_WAIT") + tcp_states = ( + "CLOSED", + "LISTEN", + "SYN_SENT", + "SYN_RECV", + "ESTABLISHED", + "CLOSE_WAIT", + "FIN_WAIT1", + "CLOSING", + "LAST_ACK", + "FIN_WAIT2", + "TIME_WAIT", + ) try: tcpcb = self.inp_ppcb.dereference().cast("tcpcb") @@ -402,14 +436,18 @@ class inpcb(objects.StructType): def get_ipv6_info(self): try: - lip = self.inp_dependladdr.inp6_local.member(attr = '__u6_addr').member(attr = '__u6_addr32') + lip = self.inp_dependladdr.inp6_local.member(attr="__u6_addr").member( + attr="__u6_addr32" + ) except exceptions.InvalidAddressException: return None lport = self.inp_lport try: - rip = self.inp_dependfaddr.inp6_foreign.member(attr = '__u6_addr').member(attr = '__u6_addr32') + rip = self.inp_dependfaddr.inp6_foreign.member(attr="__u6_addr").member( + attr="__u6_addr32" + ) except exceptions.InvalidAddressException: return None @@ -419,12 +457,13 @@ class inpcb(objects.StructType): class queue_entry(objects.StructType): - - def walk_list(self, - list_head: interfaces.objects.ObjectInterface, - member_name: str, - type_name: str, - max_size: int = 4096) -> Iterable[interfaces.objects.ObjectInterface]: + def walk_list( + self, + list_head: interfaces.objects.ObjectInterface, + member_name: str, + type_name: str, + max_size: int = 4096, + ) -> Iterable[interfaces.objects.ObjectInterface]: """ Walks a queue in a smear-aware and smear-resistant manner @@ -449,7 +488,7 @@ class queue_entry(objects.StructType): seen = set() - for attr in ['next', 'prev']: + for attr in ["next", "prev"]: with contextlib.suppress(exceptions.InvalidAddressException): n = getattr(self, attr).dereference().cast(type_name) @@ -465,11 +504,14 @@ class queue_entry(objects.StructType): if yielded == max_size: return - n = getattr(n.member(attr = member_name), attr).dereference().cast(type_name) + n = ( + getattr(n.member(attr=member_name), attr) + .dereference() + .cast(type_name) + ) class ifnet(objects.StructType): - def sockaddr_dl(self): if self.has_member("if_lladdr"): try: @@ -478,7 +520,9 @@ class ifnet(objects.StructType): val = None else: try: - val = self.if_addrhead.tqh_first.ifa_addr.dereference().cast("sockaddr_dl") + val = self.if_addrhead.tqh_first.ifa_addr.dereference().cast( + "sockaddr_dl" + ) except exceptions.InvalidAddressException: val = None @@ -487,7 +531,6 @@ class ifnet(objects.StructType): # this is used for MAC addresses class sockaddr_dl(objects.StructType): - def __str__(self): ret = "" @@ -511,7 +554,6 @@ class sockaddr_dl(objects.StructType): class sockaddr(objects.StructType): - def get_address(self): ip = "" @@ -522,7 +564,9 @@ class sockaddr(objects.StructType): elif family == 30: # AF_INET6 addr_in6 = self.cast("sockaddr_in6") - ip = conversion.convert_ipv6(addr_in6.sin6_addr.member(attr = "__u6_addr").member(attr = "__u6_addr32")) + ip = conversion.convert_ipv6( + addr_in6.sin6_addr.member(attr="__u6_addr").member(attr="__u6_addr32") + ) elif family == 18: # AF_LINK addr_dl = self.cast("sockaddr_dl") @@ -532,7 +576,6 @@ class sockaddr(objects.StructType): class sysctl_oid(objects.StructType): - def get_perms(self) -> str: """ Returns the actions allowed on the node @@ -575,9 +618,15 @@ class sysctl_oid(objects.StructType): Based on sysctl_sysctl_debug_dump_node """ - types = {1: 'CTLTYPE_NODE', 2: 'CTLTYPE_INT', 3: 'CTLTYPE_STRING', 4: 'CTLTYPE_QUAD', 5: 'CTLTYPE_OPAQUE'} + types = { + 1: "CTLTYPE_NODE", + 2: "CTLTYPE_INT", + 3: "CTLTYPE_STRING", + 4: "CTLTYPE_QUAD", + 5: "CTLTYPE_OPAQUE", + } - ctltype = self.oid_kind & 0xf + ctltype = self.oid_kind & 0xF if 0 < ctltype < 6: ret = types[ctltype] @@ -588,7 +637,6 @@ class sysctl_oid(objects.StructType): class kauth_scope(objects.StructType): - def get_listeners(self): for listener in self.ks_listeners: if listener != 0 and listener.kll_callback != 0: diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index f42ac78fe..61947be69 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -11,11 +11,13 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): """Class to handle the metadata from a Windows symbol table.""" @property - def pe_version(self) -> Optional[Union[Tuple[int, int, int], Tuple[int, int, int, int]]]: - build = self._json_data.get('pe', {}).get('build', None) - revision = self._json_data.get('pe', {}).get('revision', None) - minor = self._json_data.get('pe', {}).get('minor', None) - major = self._json_data.get('pe', {}).get('major', None) + def pe_version( + self, + ) -> Optional[Union[Tuple[int, int, int], Tuple[int, int, int, int]]]: + build = self._json_data.get("pe", {}).get("build", None) + revision = self._json_data.get("pe", {}).get("revision", None) + minor = self._json_data.get("pe", {}).get("minor", None) + major = self._json_data.get("pe", {}).get("major", None) if revision is None or minor is None or major is None: return None if build is None: @@ -30,11 +32,11 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): @property def pdb_guid(self) -> Optional[str]: - return self._json_data.get('pdb', {}).get('GUID', None) + return self._json_data.get("pdb", {}).get("GUID", None) @property def pdb_age(self) -> Optional[int]: - return self._json_data.get('pdb', {}).get('age', None) + return self._json_data.get("pdb", {}).get("age", None) class LinuxMetadata(interfaces.symbols.MetadataInterface): diff --git a/volatility3/framework/symbols/native.py b/volatility3/framework/symbols/native.py index d9833e26d..7c3e1b312 100644 --- a/volatility3/framework/symbols/native.py +++ b/volatility3/framework/symbols/native.py @@ -21,7 +21,8 @@ class NativeTable(interfaces.symbols.NativeTableInterface): self._overrides[native_type] = native_class # Create this once early, because it may get used a lot self._types = set(self._native_dictionary).union( - {'enum', 'array', 'bitfield', 'void', 'string', 'bytes', 'function'}) + {"enum", "array", "bitfield", "void", "string", "bytes", "function"} + ) def get_type_class(self, name: str) -> Type[interfaces.objects.ObjectInterface]: ntype, _ = self._native_dictionary.get(name, (objects.Integer, None)) @@ -45,62 +46,71 @@ class NativeTable(interfaces.symbols.NativeTableInterface): if constants.BANG in type_name: name_split = type_name.split(constants.BANG) if len(name_split) > 2: - raise ValueError(f"SymbolName cannot contain multiple {constants.BANG} separators") + raise ValueError( + f"SymbolName cannot contain multiple {constants.BANG} separators" + ) table_name, type_name = name_split prefix = table_name + constants.BANG additional: Dict[str, Any] = {} obj: Optional[Type[interfaces.objects.ObjectInterface]] = None - if type_name == 'void' or type_name == 'function': + if type_name == "void" or type_name == "function": obj = objects.Void - elif type_name == 'array': + elif type_name == "array": obj = objects.Array - additional = {"count": 0, "subtype": self.get_type('void')} - elif type_name == 'enum': + additional = {"count": 0, "subtype": self.get_type("void")} + elif type_name == "enum": obj = objects.Enumeration - additional = {"base_type": self.get_type('void'), "choices": {}} - elif type_name == 'bitfield': + additional = {"base_type": self.get_type("void"), "choices": {}} + elif type_name == "bitfield": obj = objects.BitField - additional = {"start_bit": 0, "end_bit": 0, "base_type": self.get_type('void')} - elif type_name == 'string': + additional = { + "start_bit": 0, + "end_bit": 0, + "base_type": self.get_type("void"), + } + elif type_name == "string": obj = objects.String additional = {"max_length": 0} - elif type_name == 'bytes': + elif type_name == "bytes": obj = objects.Bytes additional = {"length": 0} if obj is not None: - return objects.templates.ObjectTemplate(obj, type_name = prefix + type_name, **additional) + return objects.templates.ObjectTemplate( + obj, type_name=prefix + type_name, **additional + ) _native_type, native_format = self._native_dictionary[type_name] - if type_name == 'pointer': - additional = {'subtype': self.get_type('void')} + if type_name == "pointer": + additional = {"subtype": self.get_type("void")} return objects.templates.ObjectTemplate( self.get_type_class(type_name), # pylint: disable=W0142 - type_name = prefix + type_name, - data_format = objects.DataFormatInfo(*native_format), - **additional) + type_name=prefix + type_name, + data_format=objects.DataFormatInfo(*native_format), + **additional, + ) std_ctypes = { - 'int': (objects.Integer, (4, "little", True)), - 'long': (objects.Integer, (4, "little", True)), - 'unsigned long': (objects.Integer, (4, "little", False)), - 'unsigned int': (objects.Integer, (4, "little", False)), - 'char': (objects.Integer, (1, "little", True)), - 'byte': (objects.Bytes, (1, "little", True)), - 'unsigned char': (objects.Integer, (1, "little", False)), - 'unsigned short int': (objects.Integer, (2, "little", False)), - 'unsigned short': (objects.Integer, (2, "little", False)), - 'unsigned be short': (objects.Integer, (2, "big", False)), - 'short': (objects.Integer, (2, "little", True)), - 'long long': (objects.Integer, (8, "little", True)), - 'unsigned long long': (objects.Integer, (8, "little", True)), - 'float': (objects.Float, (4, "little", True)), - 'double': (objects.Float, (8, "little", True)), - 'wchar': (objects.Integer, (2, "little", False)) + "int": (objects.Integer, (4, "little", True)), + "long": (objects.Integer, (4, "little", True)), + "unsigned long": (objects.Integer, (4, "little", False)), + "unsigned int": (objects.Integer, (4, "little", False)), + "char": (objects.Integer, (1, "little", True)), + "byte": (objects.Bytes, (1, "little", True)), + "unsigned char": (objects.Integer, (1, "little", False)), + "unsigned short int": (objects.Integer, (2, "little", False)), + "unsigned short": (objects.Integer, (2, "little", False)), + "unsigned be short": (objects.Integer, (2, "big", False)), + "short": (objects.Integer, (2, "little", True)), + "long long": (objects.Integer, (8, "little", True)), + "unsigned long long": (objects.Integer, (8, "little", True)), + "float": (objects.Float, (4, "little", True)), + "double": (objects.Float, (8, "little", True)), + "wchar": (objects.Integer, (2, "little", False)), } native_types = std_ctypes.copy() -native_types['pointer'] = (objects.Pointer, (4, "little", False)) +native_types["pointer"] = (objects.Pointer, (4, "little", False)) x86NativeTable = NativeTable("native", native_types) -native_types['pointer'] = (objects.Pointer, (8, "little", False)) +native_types["pointer"] = (objects.Pointer, (8, "little", False)) x64NativeTable = NativeTable("native", native_types) diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index cfac87e2c..abf9f6da3 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -9,54 +9,53 @@ from volatility3.framework.symbols.windows.extensions import pe, pool, registry class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): - def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) # Set-up windows specific types - self.set_type_class('_ETHREAD', extensions.ETHREAD) - self.set_type_class('_KTHREAD', extensions.KTHREAD) - self.set_type_class('_LIST_ENTRY', extensions.LIST_ENTRY) - self.set_type_class('_EPROCESS', extensions.EPROCESS) - self.set_type_class('_UNICODE_STRING', extensions.UNICODE_STRING) - self.set_type_class('_EX_FAST_REF', extensions.EX_FAST_REF) - self.set_type_class('_TOKEN', extensions.TOKEN) - self.set_type_class('_OBJECT_HEADER', pool.OBJECT_HEADER) - self.set_type_class('_FILE_OBJECT', extensions.FILE_OBJECT) - self.set_type_class('_DEVICE_OBJECT', extensions.DEVICE_OBJECT) - self.set_type_class('_CM_KEY_BODY', registry.CM_KEY_BODY) - self.set_type_class('_CMHIVE', registry.CMHIVE) - self.set_type_class('_CM_KEY_NODE', registry.CM_KEY_NODE) - self.set_type_class('_CM_KEY_VALUE', registry.CM_KEY_VALUE) - self.set_type_class('_HMAP_ENTRY', registry.HMAP_ENTRY) - self.set_type_class('_MMVAD_SHORT', extensions.MMVAD_SHORT) - self.set_type_class('_MMVAD', extensions.MMVAD) - self.set_type_class('_KSYSTEM_TIME', extensions.KSYSTEM_TIME) - self.set_type_class('_KMUTANT', extensions.KMUTANT) - self.set_type_class('_DRIVER_OBJECT', extensions.DRIVER_OBJECT) - self.set_type_class('_OBJECT_SYMBOLIC_LINK', extensions.OBJECT_SYMBOLIC_LINK) - self.set_type_class('_CONTROL_AREA', extensions.CONTROL_AREA) - self.set_type_class('_SHARED_CACHE_MAP', extensions.SHARED_CACHE_MAP) - self.set_type_class('_VACB', extensions.VACB) - self.set_type_class('_POOL_TRACKER_BIG_PAGES', pool.POOL_TRACKER_BIG_PAGES) - self.set_type_class('_IMAGE_DOS_HEADER', pe.IMAGE_DOS_HEADER) + self.set_type_class("_ETHREAD", extensions.ETHREAD) + self.set_type_class("_KTHREAD", extensions.KTHREAD) + self.set_type_class("_LIST_ENTRY", extensions.LIST_ENTRY) + self.set_type_class("_EPROCESS", extensions.EPROCESS) + self.set_type_class("_UNICODE_STRING", extensions.UNICODE_STRING) + self.set_type_class("_EX_FAST_REF", extensions.EX_FAST_REF) + self.set_type_class("_TOKEN", extensions.TOKEN) + self.set_type_class("_OBJECT_HEADER", pool.OBJECT_HEADER) + self.set_type_class("_FILE_OBJECT", extensions.FILE_OBJECT) + self.set_type_class("_DEVICE_OBJECT", extensions.DEVICE_OBJECT) + self.set_type_class("_CM_KEY_BODY", registry.CM_KEY_BODY) + self.set_type_class("_CMHIVE", registry.CMHIVE) + self.set_type_class("_CM_KEY_NODE", registry.CM_KEY_NODE) + self.set_type_class("_CM_KEY_VALUE", registry.CM_KEY_VALUE) + self.set_type_class("_HMAP_ENTRY", registry.HMAP_ENTRY) + self.set_type_class("_MMVAD_SHORT", extensions.MMVAD_SHORT) + self.set_type_class("_MMVAD", extensions.MMVAD) + self.set_type_class("_KSYSTEM_TIME", extensions.KSYSTEM_TIME) + self.set_type_class("_KMUTANT", extensions.KMUTANT) + self.set_type_class("_DRIVER_OBJECT", extensions.DRIVER_OBJECT) + self.set_type_class("_OBJECT_SYMBOLIC_LINK", extensions.OBJECT_SYMBOLIC_LINK) + self.set_type_class("_CONTROL_AREA", extensions.CONTROL_AREA) + self.set_type_class("_SHARED_CACHE_MAP", extensions.SHARED_CACHE_MAP) + self.set_type_class("_VACB", extensions.VACB) + self.set_type_class("_POOL_TRACKER_BIG_PAGES", pool.POOL_TRACKER_BIG_PAGES) + self.set_type_class("_IMAGE_DOS_HEADER", pe.IMAGE_DOS_HEADER) # Might not necessarily defined in every version of windows - self.optional_set_type_class('_IMAGE_NT_HEADERS', pe.IMAGE_NT_HEADERS) - self.optional_set_type_class('_IMAGE_NT_HEADERS64', pe.IMAGE_NT_HEADERS) + self.optional_set_type_class("_IMAGE_NT_HEADERS", pe.IMAGE_NT_HEADERS) + self.optional_set_type_class("_IMAGE_NT_HEADERS64", pe.IMAGE_NT_HEADERS) # This doesn't exist in very specific versions of windows with contextlib.suppress(ValueError): if self.get_type("_POOL_TRACKER_BIG_PAGES").has_member("PoolType"): - self.set_type_class('_POOL_HEADER', pool.POOL_HEADER_VISTA) + self.set_type_class("_POOL_HEADER", pool.POOL_HEADER_VISTA) else: - self.set_type_class('_POOL_HEADER', pool.POOL_HEADER) + self.set_type_class("_POOL_HEADER", pool.POOL_HEADER) # these don't exist in windows XP - self.optional_set_type_class('_MMADDRESS_NODE', extensions.MMVAD_SHORT) + self.optional_set_type_class("_MMADDRESS_NODE", extensions.MMVAD_SHORT) # these were introduced starting in windows 8 - self.optional_set_type_class('_MM_AVL_NODE', extensions.MMVAD_SHORT) + self.optional_set_type_class("_MM_AVL_NODE", extensions.MMVAD_SHORT) # these were introduced starting in windows 7 - self.optional_set_type_class('_RTL_BALANCED_NODE', extensions.MMVAD_SHORT) + self.optional_set_type_class("_RTL_BALANCED_NODE", extensions.MMVAD_SHORT) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 87e8e0f45..d34d6a22f 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -10,7 +10,14 @@ import logging import math from typing import Generator, Iterable, Iterator, List, Optional, Tuple, Union -from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols +from volatility3.framework import ( + constants, + exceptions, + interfaces, + objects, + renderers, + symbols, +) from volatility3.framework.interfaces.objects import ObjectInterface from volatility3.framework.layers import intel from volatility3.framework.renderers import conversion @@ -38,7 +45,7 @@ class MMVAD_SHORT(objects.StructType): by VadRoot. """ - @functools.lru_cache(maxsize = None) + @functools.lru_cache(maxsize=None) def get_tag(self): vad_address = self.vol.offset @@ -51,11 +58,13 @@ class MMVAD_SHORT(objects.StructType): try: # TODO: instantiate a _POOL_HEADER and return PoolTag - bytesobj = self._context.object(symbol_table_name + constants.BANG + "bytes", - layer_name = self.vol.layer_name, - offset = vad_address, - native_layer_name = self.vol.native_layer_name, - length = 4) + bytesobj = self._context.object( + symbol_table_name + constants.BANG + "bytes", + layer_name=self.vol.layer_name, + offset=vad_address, + native_layer_name=self.vol.native_layer_name, + length=4, + ) return bytesobj.decode() except exceptions.InvalidAddressException: @@ -63,14 +72,16 @@ class MMVAD_SHORT(objects.StructType): except UnicodeDecodeError: return None - def traverse(self, visited = None, depth = 0): + def traverse(self, visited=None, depth=0): """Traverse the VAD tree, determining each underlying VAD node type by looking up the pool tag for the structure and then casting into a new object.""" # TODO: this is an arbitrary limit chosen based on past observations if depth > 100: - vollog.log(constants.LOGLEVEL_VVV, "Vad tree is too deep, something went wrong!") + vollog.log( + constants.LOGLEVEL_VVV, "Vad tree is too deep, something went wrong!" + ) raise RuntimeError("Vad tree is too deep") if visited is None: @@ -96,8 +107,10 @@ class MMVAD_SHORT(objects.StructType): else: # any node other than the root that doesn't have a recognized tag # is just garbage and we skip the node entirely - vollog.log(constants.LOGLEVEL_VVV, - f"Skipping VAD at {self.vol.offset} depth {depth} with tag {tag}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Skipping VAD at {self.vol.offset} depth {depth} with tag {tag}", + ) return if target: @@ -105,16 +118,26 @@ class MMVAD_SHORT(objects.StructType): yield vad_object try: - for vad_node in self.get_left_child().dereference().traverse(visited, depth + 1): + for vad_node in ( + self.get_left_child().dereference().traverse(visited, depth + 1) + ): yield vad_node except exceptions.InvalidAddressException as excp: - vollog.log(constants.LOGLEVEL_VVV, f"Invalid address on LeftChild: {excp.invalid_address:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Invalid address on LeftChild: {excp.invalid_address:#x}", + ) try: - for vad_node in self.get_right_child().dereference().traverse(visited, depth + 1): + for vad_node in ( + self.get_right_child().dereference().traverse(visited, depth + 1) + ): yield vad_node except exceptions.InvalidAddressException as excp: - vollog.log(constants.LOGLEVEL_VVV, f"Invalid address on RightChild: {excp.invalid_address:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Invalid address on RightChild: {excp.invalid_address:#x}", + ) def get_right_child(self): """Get the right child member.""" @@ -228,7 +251,9 @@ class MMVAD_SHORT(objects.StructType): elif self.has_member("Core"): if self.Core.has_member("EndingVpnHigh"): - return (((self.Core.EndingVpn + 1) << 12) | (self.Core.EndingVpnHigh << 44)) - 1 + return ( + ((self.Core.EndingVpn + 1) << 12) | (self.Core.EndingVpnHigh << 44) + ) - 1 else: return ((self.Core.EndingVpn + 1) << 12) - 1 @@ -255,19 +280,33 @@ class MMVAD_SHORT(objects.StructType): def get_private_memory(self): """Get the VAD's private memory setting.""" - if self.has_member("u1") and self.u1.has_member("VadFlags1") and self.u1.VadFlags1.has_member("PrivateMemory"): + if ( + self.has_member("u1") + and self.u1.has_member("VadFlags1") + and self.u1.VadFlags1.has_member("PrivateMemory") + ): return self.u1.VadFlags1.PrivateMemory - elif self.has_member("u") and self.u.has_member("VadFlags") and self.u.VadFlags.has_member("PrivateMemory"): + elif ( + self.has_member("u") + and self.u.has_member("VadFlags") + and self.u.VadFlags.has_member("PrivateMemory") + ): return self.u.VadFlags.PrivateMemory elif self.has_member("Core"): - if (self.Core.has_member("u1") and self.Core.u1.has_member("VadFlags1") - and self.Core.u1.VadFlags1.has_member("PrivateMemory")): + if ( + self.Core.has_member("u1") + and self.Core.u1.has_member("VadFlags1") + and self.Core.u1.VadFlags1.has_member("PrivateMemory") + ): return self.Core.u1.VadFlags1.PrivateMemory - elif (self.Core.has_member("u") and self.Core.u.has_member("VadFlags") - and self.Core.u.VadFlags.has_member("PrivateMemory")): + elif ( + self.Core.has_member("u") + and self.Core.u.has_member("VadFlags") + and self.Core.u.VadFlags.has_member("PrivateMemory") + ): return self.Core.u.VadFlags.PrivateMemory raise AttributeError("Unable to find the private memory member") @@ -317,8 +356,11 @@ class MMVAD(MMVAD_SHORT): # this is for vista through windows 7 else: - filename_obj = self.Subsection.ControlArea.FilePointer.dereference().cast( - "_FILE_OBJECT").FileName + filename_obj = ( + self.Subsection.ControlArea.FilePointer.dereference() + .cast("_FILE_OBJECT") + .FileName + ) if filename_obj.Length > 0: file_name = filename_obj.get_string() @@ -336,7 +378,9 @@ class EX_FAST_REF(objects.StructType): def dereference(self) -> interfaces.objects.ObjectInterface: if constants.BANG not in self.vol.type_name: - raise ValueError(f"Invalid symbol table name syntax (no {constants.BANG} found)") + raise ValueError( + f"Invalid symbol table name syntax (no {constants.BANG} found)" + ) # the mask value is different on 32 and 64 bits symbol_table_name = self.vol.type_name.split(constants.BANG)[0] @@ -345,10 +389,12 @@ class EX_FAST_REF(objects.StructType): else: max_fast_ref = 15 - return self._context.object(symbol_table_name + constants.BANG + "pointer", - layer_name = self.vol.layer_name, - offset = self.Object & ~max_fast_ref, - native_layer_name = self.vol.native_layer_name) + return self._context.object( + symbol_table_name + constants.BANG + "pointer", + layer_name=self.vol.layer_name, + offset=self.Object & ~max_fast_ref, + native_layer_name=self.vol.native_layer_name, + ) class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): @@ -407,11 +453,14 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): def is_valid(self) -> bool: """Determine if the object is valid.""" - return self.FileName.Length > 0 and self._context.layers[self.FileName.Buffer.vol.native_layer_name].is_valid( - self.FileName.Buffer) + return self.FileName.Length > 0 and self._context.layers[ + self.FileName.Buffer.vol.native_layer_name + ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[str, interfaces.renderers.BaseAbsentValue] = renderers.UnreadableValue() + name: Union[ + str, interfaces.renderers.BaseAbsentValue + ] = renderers.UnreadableValue() # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. @@ -426,9 +475,14 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): def access_string(self): ## Make a nicely formatted ACL string - return (('R' if self.ReadAccess else '-') + ('W' if self.WriteAccess else '-') + - ('D' if self.DeleteAccess else '-') + ('r' if self.SharedRead else '-') + - ('w' if self.SharedWrite else '-') + ('d' if self.SharedDelete else '-')) + return ( + ("R" if self.ReadAccess else "-") + + ("W" if self.WriteAccess else "-") + + ("D" if self.DeleteAccess else "-") + + ("r" if self.SharedRead else "-") + + ("w" if self.SharedWrite else "-") + + ("d" if self.SharedDelete else "-") + ) class KMUTANT(objects.StructType, pool.ExecutiveObject): @@ -449,36 +503,40 @@ class ETHREAD(objects.StructType): def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" - + # For Windows XPs - if(self.has_member("ThreadsProcess")): + if self.has_member("ThreadsProcess"): return self.ThreadsProcess.dereference().cast("_EPROCESS") # For Windows Vista and later versions - elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): + elif self.has_member("Tcb") and self.Tcb.has_member("Process"): return self.Tcb.Process.dereference().cast("_EPROCESS") else: raise AttributeError("Unable to find the owning process of ethread") def get_cross_thread_flags(self) -> str: dictCrossThreadFlags = { - 'PS_CROSS_THREAD_FLAGS_TERMINATED': 0, - 'PS_CROSS_THREAD_FLAGS_DEADTHREAD': 1, - 'PS_CROSS_THREAD_FLAGS_HIDEFROMDBG': 2, - 'PS_CROSS_THREAD_FLAGS_IMPERSONATING': 3, - 'PS_CROSS_THREAD_FLAGS_SYSTEM': 4, - 'PS_CROSS_THREAD_FLAGS_HARD_ERRORS_DISABLED': 5, - 'PS_CROSS_THREAD_FLAGS_BREAK_ON_TERMINATION': 6, - 'PS_CROSS_THREAD_FLAGS_SKIP_CREATION_MSG': 7, - 'PS_CROSS_THREAD_FLAGS_SKIP_TERMINATION_MSG': 8 + "PS_CROSS_THREAD_FLAGS_TERMINATED": 0, + "PS_CROSS_THREAD_FLAGS_DEADTHREAD": 1, + "PS_CROSS_THREAD_FLAGS_HIDEFROMDBG": 2, + "PS_CROSS_THREAD_FLAGS_IMPERSONATING": 3, + "PS_CROSS_THREAD_FLAGS_SYSTEM": 4, + "PS_CROSS_THREAD_FLAGS_HARD_ERRORS_DISABLED": 5, + "PS_CROSS_THREAD_FLAGS_BREAK_ON_TERMINATION": 6, + "PS_CROSS_THREAD_FLAGS_SKIP_CREATION_MSG": 7, + "PS_CROSS_THREAD_FLAGS_SKIP_TERMINATION_MSG": 8, } flags = self.CrossThreadFlags - stringCrossThreadFlags = '' + stringCrossThreadFlags = "" for flag in dictCrossThreadFlags: if flags & 2 ** dictCrossThreadFlags[flag]: - stringCrossThreadFlags += f'{flag} ' + stringCrossThreadFlags += f"{flag} " - return stringCrossThreadFlags[:-1] if stringCrossThreadFlags else stringCrossThreadFlags + return ( + stringCrossThreadFlags[:-1] + if stringCrossThreadFlags + else stringCrossThreadFlags + ) class UNICODE_STRING(objects.StructType): @@ -491,10 +549,14 @@ class UNICODE_STRING(objects.StructType): # We manually construct an object rather than casting a dereferenced pointer in case # the buffer length is 0 and the pointer is a NULL pointer - return self._context.object(self.vol.type_name.split(constants.BANG)[0] + constants.BANG + 'string', - layer_name = self.Buffer.vol.native_layer_name, - offset = self.Buffer, - max_length = self.Length, errors = 'replace', encoding = 'utf16') + return self._context.object( + self.vol.type_name.split(constants.BANG)[0] + constants.BANG + "string", + layer_name=self.Buffer.vol.native_layer_name, + offset=self.Buffer, + max_length=self.Length, + errors="replace", + encoding="utf16", + ) String = property(get_string) @@ -536,7 +598,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return False # check for all 0s besides the PCID entries - if dtb & ~0xfff == 0: + if dtb & ~0xFFF == 0: return False ## TODO: we can also add the thread Flink and Blink tests if necessary @@ -553,7 +615,9 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): if not isinstance(parent_layer, intel.Intel): # We can't get bits_per_register unless we're an intel space (since that's not defined at the higher layer) - raise TypeError("Parent layer is not a translation layer, unable to construct process layer") + raise TypeError( + "Parent layer is not a translation layer, unable to construct process layer" + ) # Presumably for 64-bit systems, the DTB is defined as an array, rather than an unsigned long long dtb: int = 0 @@ -567,12 +631,16 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): preferred_name = self.vol.layer_name + f"_Process{self.UniqueProcessId}" # Add the constructed layer and return the name - return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) + return self._add_process_layer( + self._context, dtb, config_prefix, preferred_name + ) def get_peb(self) -> interfaces.objects.ObjectInterface: """Constructs a PEB object""" if constants.BANG not in self.vol.type_name: - raise ValueError(f"Invalid symbol table name syntax (no {constants.BANG} found)") + raise ValueError( + f"Invalid symbol table name syntax (no {constants.BANG} found)" + ) # add_process_layer can raise InvalidAddressException. # if that happens, we let the exception propagate upwards @@ -580,13 +648,16 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): proc_layer = self._context.layers[proc_layer_name] if not proc_layer.is_valid(self.Peb): - raise exceptions.InvalidAddressException(proc_layer_name, self.Peb, - f"Invalid Peb address at {self.Peb:0x}") + raise exceptions.InvalidAddressException( + proc_layer_name, self.Peb, f"Invalid Peb address at {self.Peb:0x}" + ) sym_table = self.get_symbol_table_name() - peb = self._context.object(f"{sym_table}{constants.BANG}_PEB", - layer_name = proc_layer_name, - offset = self.Peb) + peb = self._context.object( + f"{sym_table}{constants.BANG}_PEB", + layer_name=proc_layer_name, + offset=self.Peb, + ) return peb def load_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: @@ -595,8 +666,9 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() for entry in peb.Ldr.InLoadOrderModuleList.to_list( - f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", - "InLoadOrderLinks"): + f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", + "InLoadOrderLinks", + ): yield entry except exceptions.InvalidAddressException: return @@ -607,8 +679,9 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() for entry in peb.Ldr.InInitializationOrderModuleList.to_list( - f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", - "InInitializationOrderLinks"): + f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", + "InInitializationOrderLinks", + ): yield entry except exceptions.InvalidAddressException: return @@ -619,8 +692,9 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() for entry in peb.Ldr.InMemoryOrderModuleList.to_list( - f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", - "InMemoryOrderLinks"): + f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", + "InMemoryOrderLinks", + ): yield entry except exceptions.InvalidAddressException: return @@ -632,8 +706,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return self.ObjectTable.HandleCount except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot access _EPROCESS.ObjectTable.HandleCount at {self.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Cannot access _EPROCESS.ObjectTable.HandleCount at {self.vol.offset:#x}", + ) return renderers.UnreadableValue() @@ -644,19 +720,27 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return renderers.NotApplicableValue() symbol_table_name = self.get_symbol_table_name() - kvo = self._context.layers[self.vol.native_layer_name].config['kernel_virtual_offset'] - ntkrnlmp = self._context.module(symbol_table_name, - layer_name = self.vol.native_layer_name, - offset = kvo, - native_layer_name = self.vol.native_layer_name) - session = ntkrnlmp.object(object_type = "_MM_SESSION_SPACE", offset = self.Session, absolute = True) + kvo = self._context.layers[self.vol.native_layer_name].config[ + "kernel_virtual_offset" + ] + ntkrnlmp = self._context.module( + symbol_table_name, + layer_name=self.vol.native_layer_name, + offset=kvo, + native_layer_name=self.vol.native_layer_name, + ) + session = ntkrnlmp.object( + object_type="_MM_SESSION_SPACE", offset=self.Session, absolute=True + ) if session.has_member("SessionId"): return session.SessionId except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot access _EPROCESS.Session.SessionId at {self.vol.offset:#x}") + vollog.log( + constants.LOGLEVEL_VVV, + f"Cannot access _EPROCESS.Session.SessionId at {self.vol.offset:#x}", + ) return renderers.UnreadableValue() @@ -716,40 +800,48 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): block_size = self.get_peb().ProcessParameters.EnvironmentSize except AttributeError: # Windows XP block_size = self.get_peb().ProcessParameters.Length - envars = context.layers[process_space].read(block, block_size).decode("utf-16-le", - errors = 'replace').split('\x00')[:-1] + envars = ( + context.layers[process_space] + .read(block, block_size) + .decode("utf-16-le", errors="replace") + .split("\x00")[:-1] + ) except exceptions.InvalidAddressException: - return # Generation finished + return # Generation finished for envar in envars: - split_index = envar.find('=') + split_index = envar.find("=") env = envar[:split_index] - var = envar[split_index + 1:] + var = envar[split_index + 1 :] # Exclude parse problem with some types of env if env and var: yield env, var - return # Generation finished + return # Generation finished class LIST_ENTRY(objects.StructType, collections.abc.Iterable): """A class for double-linked lists on Windows.""" - def to_list(self, - symbol_type: str, - member: str, - forward: bool = True, - sentinel: bool = True, - layer: Optional[str] = None) -> Iterator[interfaces.objects.ObjectInterface]: + def to_list( + self, + symbol_type: str, + member: str, + forward: bool = True, + sentinel: bool = True, + layer: Optional[str] = None, + ) -> Iterator[interfaces.objects.ObjectInterface]: """Returns an iterator of the entries in the list.""" layer = layer or self.vol.layer_name - relative_offset = self._context.symbol_space.get_type(symbol_type).relative_child_offset(member) + relative_offset = self._context.symbol_space.get_type( + symbol_type + ).relative_child_offset(member) - direction = 'Blink' + direction = "Blink" if forward: - direction = 'Flink' + direction = "Flink" trans_layer = self._context.layers[layer] @@ -763,10 +855,12 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): return if not sentinel: - yield self._context.object(symbol_type, - layer, - offset = self.vol.offset - relative_offset, - native_layer_name = layer or self.vol.native_layer_name) + yield self._context.object( + symbol_type, + layer, + offset=self.vol.offset - relative_offset, + native_layer_name=layer or self.vol.native_layer_name, + ) seen = {self.vol.offset} while link.vol.offset not in seen: @@ -775,10 +869,12 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): if not trans_layer.is_valid(obj_offset): return - obj = self._context.object(symbol_type, - layer, - offset = obj_offset, - native_layer_name = layer or self.vol.native_layer_name) + obj = self._context.object( + symbol_type, + layer, + offset=obj_offset, + native_layer_name=layer or self.vol.native_layer_name, + ) yield obj seen.add(link.vol.offset) @@ -802,11 +898,15 @@ class TOKEN(objects.StructType): layer_name = self.vol.layer_name kvo = self._context.layers[layer_name].config["kernel_virtual_offset"] symbol_table = self.get_symbol_table_name() - ntkrnlmp = self._context.module(symbol_table, layer_name = layer_name, offset = kvo) - UserAndGroups = ntkrnlmp.object(object_type = "array", - offset = self.UserAndGroups.dereference().vol.get("offset") - kvo, - subtype = ntkrnlmp.get_type("_SID_AND_ATTRIBUTES"), - count = self.UserAndGroupCount) + ntkrnlmp = self._context.module( + symbol_table, layer_name=layer_name, offset=kvo + ) + UserAndGroups = ntkrnlmp.object( + object_type="array", + offset=self.UserAndGroups.dereference().vol.get("offset") - kvo, + subtype=ntkrnlmp.get_type("_SID_AND_ATTRIBUTES"), + count=self.UserAndGroupCount, + ) for sid_and_attr in UserAndGroups: try: sid = sid_and_attr.Sid.dereference().cast("_SID") @@ -819,30 +919,42 @@ class TOKEN(objects.StructType): id_auth = "" for i in sid.IdentifierAuthority.Value: id_auth = i - SubAuthority = ntkrnlmp.object(object_type = "array", - offset = sid.SubAuthority.vol.offset - kvo, - subtype = ntkrnlmp.get_type("unsigned long"), - count = int(sid.SubAuthorityCount)) - yield "S-" + "-".join(str(i) for i in (sid.Revision, id_auth) + tuple(SubAuthority)) + SubAuthority = ntkrnlmp.object( + object_type="array", + offset=sid.SubAuthority.vol.offset - kvo, + subtype=ntkrnlmp.get_type("unsigned long"), + count=int(sid.SubAuthorityCount), + ) + yield "S-" + "-".join( + str(i) for i in (sid.Revision, id_auth) + tuple(SubAuthority) + ) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVVV, "InvalidAddressException while parsing for token sid") + vollog.log( + constants.LOGLEVEL_VVVV, + "InvalidAddressException while parsing for token sid", + ) def privileges(self): """Return a list of privileges for the current token object.""" try: for priv_index in range(64): - yield (priv_index, bool(self.Privileges.Present & (2 ** priv_index)), - bool(self.Privileges.Enabled & (2 ** priv_index)), - bool(self.Privileges.EnabledByDefault & (2 ** priv_index))) + yield ( + priv_index, + bool(self.Privileges.Present & (2**priv_index)), + bool(self.Privileges.Enabled & (2**priv_index)), + bool(self.Privileges.EnabledByDefault & (2**priv_index)), + ) except AttributeError: # Windows XP if self.PrivilegeCount < 1024: # This is a pointer to an array of _LUID_AND_ATTRIBUTES for luid in self.Privileges.dereference().cast( - "array", - count = self.PrivilegeCount, - subtype = self._context.symbol_space[self.get_symbol_table_name()].get_type( - "_LUID_AND_ATTRIBUTES")): + "array", + count=self.PrivilegeCount, + subtype=self._context.symbol_space[ + self.get_symbol_table_name() + ].get_type("_LUID_AND_ATTRIBUTES"), + ): # The Attributes member is a flag enabled = luid.Attributes & 2 != 0 default = luid.Attributes & 1 != 0 @@ -856,58 +968,58 @@ class KTHREAD(objects.StructType): def get_state(self) -> str: dictState = { - 0: 'Initialized', - 1: 'Ready', - 2: 'Running', - 3: 'Standby', - 4: 'Terminated', - 5: 'Waiting', - 6: 'Transition', - 7: 'DeferredReady', - 8: 'GateWait' + 0: "Initialized", + 1: "Ready", + 2: "Running", + 3: "Standby", + 4: "Terminated", + 5: "Waiting", + 6: "Transition", + 7: "DeferredReady", + 8: "GateWait", } return dictState.get(self.State, renderers.NotApplicableValue()) def get_wait_reason(self) -> str: dictWaitReason = { - 0: 'Executive', - 1: 'FreePage', - 2: 'PageIn', - 3: 'PoolAllocation', - 4: 'DelayExecution', - 5: 'Suspended', - 6: 'UserRequest', - 7: 'WrExecutive', - 8: 'WrFreePage', - 9: 'WrPageIn', - 10: 'WrPoolAllocation', - 11: 'WrDelayExecution', - 12: 'WrSuspended', - 13: 'WrUserRequest', - 14: 'WrEventPair', - 15: 'WrQueue', - 16: 'WrLpcReceive', - 17: 'WrLpcReply', - 18: 'WrVirtualMemory', - 19: 'WrPageOut', - 20: 'WrRendezvous', - 21: 'Spare2', - 22: 'Spare3', - 23: 'Spare4', - 24: 'Spare5', - 25: 'Spare6', - 26: 'WrKernel', - 27: 'WrResource', - 28: 'WrPushLock', - 29: 'WrMutex', - 30: 'WrQuantumEnd', - 31: 'WrDispatchInt', - 32: 'WrPreempted', - 33: 'WrYieldExecution', - 34: 'WrFastMutex', - 35: 'WrGuardedMutex', - 36: 'WrRundown', - 37: 'MaximumWaitReason' + 0: "Executive", + 1: "FreePage", + 2: "PageIn", + 3: "PoolAllocation", + 4: "DelayExecution", + 5: "Suspended", + 6: "UserRequest", + 7: "WrExecutive", + 8: "WrFreePage", + 9: "WrPageIn", + 10: "WrPoolAllocation", + 11: "WrDelayExecution", + 12: "WrSuspended", + 13: "WrUserRequest", + 14: "WrEventPair", + 15: "WrQueue", + 16: "WrLpcReceive", + 17: "WrLpcReply", + 18: "WrVirtualMemory", + 19: "WrPageOut", + 20: "WrRendezvous", + 21: "Spare2", + 22: "Spare3", + 23: "Spare4", + 24: "Spare5", + 25: "Spare6", + 26: "WrKernel", + 27: "WrResource", + 28: "WrPushLock", + 29: "WrMutex", + 30: "WrQuantumEnd", + 31: "WrDispatchInt", + 32: "WrPreempted", + 33: "WrYieldExecution", + 34: "WrFastMutex", + 35: "WrGuardedMutex", + 36: "WrRundown", + 37: "MaximumWaitReason", } return dictWaitReason.get(self.WaitReason, renderers.NotApplicableValue()) @@ -926,7 +1038,9 @@ class CONTROL_AREA(objects.StructType): return False # The SizeOfSegment should match the total PTEs multiplied by a default page size - if self.Segment.SizeOfSegment != (self.Segment.TotalNumberOfPtes * self.PAGE_SIZE): + if self.Segment.SizeOfSegment != ( + self.Segment.TotalNumberOfPtes * self.PAGE_SIZE + ): return False # The first SubsectionBase should not be page aligned @@ -942,18 +1056,22 @@ class CONTROL_AREA(objects.StructType): def get_subsection(self) -> interfaces.objects.ObjectInterface: """Get the Subsection object, which is found immediately after the _CONTROL_AREA.""" - return self._context.object(self.get_symbol_table_name() + constants.BANG + "_SUBSECTION", - layer_name = self.vol.layer_name, - offset = self.vol.offset + self.vol.size, - native_layer_name = self.vol.native_layer_name) + return self._context.object( + self.get_symbol_table_name() + constants.BANG + "_SUBSECTION", + layer_name=self.vol.layer_name, + offset=self.vol.offset + self.vol.size, + native_layer_name=self.vol.native_layer_name, + ) def get_pte(self, offset: int) -> interfaces.objects.ObjectInterface: """Get a PTE object at the requested offset""" - return self._context.object(self.get_symbol_table_name() + constants.BANG + "_MMPTE", - layer_name = self.vol.layer_name, - offset = offset, - native_layer_name = self.vol.native_layer_name) + return self._context.object( + self.get_symbol_table_name() + constants.BANG + "_MMPTE", + layer_name=self.vol.layer_name, + offset=offset, + native_layer_name=self.vol.native_layer_name, + ) def get_available_pages(self) -> Iterable[Tuple[int, int, int]]: """Get the available pages that correspond to a cached file. @@ -961,7 +1079,9 @@ class CONTROL_AREA(objects.StructType): The tuples generated are (physical_offset, file_offset, page_size). """ symbol_table_name = self.get_symbol_table_name() - mmpte_type = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + "_MMPTE") + mmpte_type = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "_MMPTE" + ) mmpte_size = mmpte_type.size subsection = self.get_subsection() is_64bit = symbols.symbol_table_is_64bit(self._context, symbol_table_name) @@ -1002,8 +1122,9 @@ class CONTROL_AREA(objects.StructType): elif mmpte.u.Soft.Prototype == 1: if not is_64bit and not is_pae: - subsection_offset = ((mmpte.u.Subsect.SubsectionAddressHigh << 7) | - (mmpte.u.Subsect.SubsectionAddressLow << 3)) + subsection_offset = ( + mmpte.u.Subsect.SubsectionAddressHigh << 7 + ) | (mmpte.u.Subsect.SubsectionAddressLow << 3) # If the entry is not a valid physical address then see if it is in transition. elif mmpte.u.Trans.Transition == 1: @@ -1049,17 +1170,21 @@ class SHARED_CACHE_MAP(objects.StructType): if self.FileSize.QuadPart <= 0 or self.ValidDataLength.QuadPart <= 0: return False - if self.SectionSize.QuadPart < 0 or ((self.FileSize.QuadPart < self.ValidDataLength.QuadPart) and - (self.ValidDataLength.QuadPart != 0x7fffffffffffffff)): + if self.SectionSize.QuadPart < 0 or ( + (self.FileSize.QuadPart < self.ValidDataLength.QuadPart) + and (self.ValidDataLength.QuadPart != 0x7FFFFFFFFFFFFFFF) + ): return False return True - def process_index_array(self, - array_pointer: interfaces.objects.ObjectInterface, - level: int, - limit: int, - vacb_list: Optional[List] = None) -> List: + def process_index_array( + self, + array_pointer: interfaces.objects.ObjectInterface, + level: int, + limit: int, + vacb_list: Optional[List] = None, + ) -> List: """Recursively process the sparse multilevel VACB index array. :param array_pointer: The address of a possible index array @@ -1075,14 +1200,18 @@ class SHARED_CACHE_MAP(objects.StructType): return [] symbol_table_name = self.get_symbol_table_name() - pointer_type = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + "pointer") + pointer_type = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "pointer" + ) # Create an array of 128 entries for the VACB index array - vacb_array = self._context.object(object_type = symbol_table_name + constants.BANG + "array", - layer_name = self.vol.layer_name, - offset = array_pointer, - count = self.VACB_ARRAY, - subtype = pointer_type) + vacb_array = self._context.object( + object_type=symbol_table_name + constants.BANG + "array", + layer_name=self.vol.layer_name, + offset=array_pointer, + count=self.VACB_ARRAY, + subtype=pointer_type, + ) # Iterate through the entries for counter in range(0, self.VACB_ARRAY): @@ -1090,16 +1219,26 @@ class SHARED_CACHE_MAP(objects.StructType): if not vacb_array[counter]: continue - vacb_obj = vacb_array[counter].dereference().cast(symbol_table_name + constants.BANG + "_VACB") + vacb_obj = ( + vacb_array[counter] + .dereference() + .cast(symbol_table_name + constants.BANG + "_VACB") + ) if vacb_obj.SharedCacheMap == self.vol.offset: self.save_vacb(vacb_obj, vacb_list) else: # Process the next level of the multi-level array - vacb_list = self.process_index_array(vacb_array[counter], level + 1, limit, vacb_list) + vacb_list = self.process_index_array( + vacb_array[counter], level + 1, limit, vacb_list + ) return vacb_list def save_vacb(self, vacb_obj: interfaces.objects.ObjectInterface, vacb_list: List): - data = (int(vacb_obj.BaseAddress), int(vacb_obj.get_file_offset()), self.VACB_BLOCK) + data = ( + int(vacb_obj.BaseAddress), + int(vacb_obj.get_file_offset()), + self.VACB_BLOCK, + ) vacb_list.append(data) def get_available_pages(self) -> List: @@ -1152,15 +1291,19 @@ class SHARED_CACHE_MAP(objects.StructType): # If the file is less than 32 MB than it can be found in a single level VACB index array. symbol_table_name = self.get_symbol_table_name() - pointer_type = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + "pointer") + pointer_type = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "pointer" + ) size_of_pointer = pointer_type.size if not section_size > self.VACB_SIZE_OF_FIRST_LEVEL: array_head = vacb_obj for counter in range(0, full_blocks): - vacb_entry = self._context.object(symbol_table_name + constants.BANG + "pointer", - layer_name = self.vol.layer_name, - offset = array_head + (counter * size_of_pointer)) + vacb_entry = self._context.object( + symbol_table_name + constants.BANG + "pointer", + layer_name=self.vol.layer_name, + offset=array_head + (counter * size_of_pointer), + ) # If we find a zero entry, then we proceed to the next one. If the entry is zero, # then the view is not mapped and we skip. We do not pad because we use the @@ -1168,19 +1311,25 @@ class SHARED_CACHE_MAP(objects.StructType): if not vacb_entry: continue - vacb = vacb_entry.dereference().cast(symbol_table_name + constants.BANG + "_VACB") + vacb = vacb_entry.dereference().cast( + symbol_table_name + constants.BANG + "_VACB" + ) if vacb.SharedCacheMap == self.vol.offset: self.save_vacb(vacb, vacb_list) if left_over > 0: - vacb_entry = self._context.object(symbol_table_name + constants.BANG + "pointer", - layer_name = self.vol.layer_name, - offset = array_head + ((counter + 1) * size_of_pointer)) + vacb_entry = self._context.object( + symbol_table_name + constants.BANG + "pointer", + layer_name=self.vol.layer_name, + offset=array_head + ((counter + 1) * size_of_pointer), + ) if not vacb_entry: return vacb_list - vacb = vacb_entry.dereference().cast(symbol_table_name + constants.BANG + "_VACB") + vacb = vacb_entry.dereference().cast( + symbol_table_name + constants.BANG + "_VACB" + ) if vacb.SharedCacheMap == self.vol.offset: self.save_vacb(vacb, vacb_list) @@ -1199,11 +1348,13 @@ class SHARED_CACHE_MAP(objects.StructType): if section_size > self.VACB_SIZE_OF_FIRST_LEVEL: # Create an array of 128 entries for the VACB index array. - vacb_array = self._context.object(object_type = symbol_table_name + constants.BANG + "array", - layer_name = self.vol.layer_name, - offset = vacb_obj, - count = self.VACB_ARRAY, - subtype = pointer_type) + vacb_array = self._context.object( + object_type=symbol_table_name + constants.BANG + "array", + layer_name=self.vol.layer_name, + offset=vacb_obj, + count=self.VACB_ARRAY, + subtype=pointer_type, + ) # Walk the array and if any entry points to the shared cache map object then we extract it. # Otherwise, if it is non-zero, then traverse to the next level. @@ -1211,13 +1362,19 @@ class SHARED_CACHE_MAP(objects.StructType): if not vacb_array[counter]: continue - vacb = vacb_array[counter].dereference().cast(symbol_table_name + constants.BANG + "_VACB") + vacb = ( + vacb_array[counter] + .dereference() + .cast(symbol_table_name + constants.BANG + "_VACB") + ) if vacb.SharedCacheMap == self.vol.offset: self.save_vacb(vacb, vacb_list) else: # Process the next level of the multi-level array. We set the limit_depth to be # the depth of the tree as determined from the size and we initialize the # current level to 2. - vacb_list = self.process_index_array(vacb_array[counter], 2, limit_depth, vacb_list) + vacb_list = self.process_index_array( + vacb_array[counter], 2, limit_depth, vacb_list + ) return vacb_list diff --git a/volatility3/framework/symbols/windows/extensions/crash.py b/volatility3/framework/symbols/windows/extensions/crash.py index 8d8200aeb..599367a0c 100644 --- a/volatility3/framework/symbols/windows/extensions/crash.py +++ b/volatility3/framework/symbols/windows/extensions/crash.py @@ -7,21 +7,30 @@ from volatility3.framework import objects class SUMMARY_DUMP(objects.StructType): - - def get_buffer(self, sub_type: str, count: int) -> interfaces.objects.ObjectInterface: + def get_buffer( + self, sub_type: str, count: int + ) -> interfaces.objects.ObjectInterface: symbol_table_name = self.get_symbol_table_name() - subtype = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + sub_type) - return self._context.object(object_type = symbol_table_name + constants.BANG + "array", - layer_name = self.vol.layer_name, - offset = self.BufferChar.vol.offset, - count = count, - subtype = subtype) + subtype = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + sub_type + ) + return self._context.object( + object_type=symbol_table_name + constants.BANG + "array", + layer_name=self.vol.layer_name, + offset=self.BufferChar.vol.offset, + count=count, + subtype=subtype, + ) def get_buffer_char(self) -> interfaces.objects.ObjectInterface: - return self.get_buffer(sub_type = "unsigned char", count = (self.BitmapSize + 7) // 8) + return self.get_buffer( + sub_type="unsigned char", count=(self.BitmapSize + 7) // 8 + ) def get_buffer_long(self) -> interfaces.objects.ObjectInterface: - return self.get_buffer(sub_type = "unsigned long", count = (self.BitmapSize + 31) // 32) + return self.get_buffer( + sub_type="unsigned long", count=(self.BitmapSize + 31) // 32 + ) -class_types = {'_SUMMARY_DUMP': SUMMARY_DUMP} +class_types = {"_SUMMARY_DUMP": SUMMARY_DUMP} diff --git a/volatility3/framework/symbols/windows/extensions/kdbg.py b/volatility3/framework/symbols/windows/extensions/kdbg.py index 437edf823..b576f4ef2 100644 --- a/volatility3/framework/symbols/windows/extensions/kdbg.py +++ b/volatility3/framework/symbols/windows/extensions/kdbg.py @@ -7,18 +7,19 @@ from volatility3.framework import objects class KDDEBUGGER_DATA64(objects.StructType): - def get_build_lab(self): """Returns the NT build lab string from the KDBG.""" layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() - return self._context.object(symbol_table_name + constants.BANG + "string", - layer_name = layer_name, - offset = self.NtBuildLab, - max_length = 32, - errors = "replace") + return self._context.object( + symbol_table_name + constants.BANG + "string", + layer_name=layer_name, + offset=self.NtBuildLab, + max_length=32, + errors="replace", + ) def get_csdversion(self): """Returns the CSDVersion as an integer (i.e. Service Pack number)""" @@ -26,11 +27,13 @@ class KDDEBUGGER_DATA64(objects.StructType): layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() - csdresult = self._context.object(symbol_table_name + constants.BANG + "unsigned long", - layer_name = layer_name, - offset = self.CmNtCSDVersion) + csdresult = self._context.object( + symbol_table_name + constants.BANG + "unsigned long", + layer_name=layer_name, + offset=self.CmNtCSDVersion, + ) - return (csdresult >> 8) & 0xffffffff + return (csdresult >> 8) & 0xFFFFFFFF -class_types = {'_KDDEBUGGER_DATA64': KDDEBUGGER_DATA64} +class_types = {"_KDDEBUGGER_DATA64": KDDEBUGGER_DATA64} diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index fc7996c52..afdc73a17 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -4,30 +4,34 @@ from volatility3.framework import objects -class PARTITION_TABLE(objects.StructType): +class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: """Get Disk Signature (GUID).""" return "{0:02x}-{1:02x}-{2:02x}-{3:02x}".format( - self.DiskSignature[0], - self.DiskSignature[1], - self.DiskSignature[2], - self.DiskSignature[3] + self.DiskSignature[0], + self.DiskSignature[1], + self.DiskSignature[2], + self.DiskSignature[3], ) + class PARTITION_ENTRY(objects.StructType): - def get_bootable_flag(self) -> int: """Get Bootable Flag.""" return self.BootableFlag - + def is_bootable(self) -> bool: """Check Bootable Partition.""" return False if not (self.get_bootable_flag() == 0x80) else True def get_partition_type(self) -> str: """Get Partition Type.""" - return self.PartitionType.lookup() if self.PartitionType.is_valid_choice else "Not Defined PartitionType" + return ( + self.PartitionType.lookup() + if self.PartitionType.is_valid_choice + else "Not Defined PartitionType" + ) def get_starting_chs(self) -> int: """Get Starting CHS (Cylinder Header Sector) Address.""" @@ -47,16 +51,18 @@ class PARTITION_ENTRY(objects.StructType): def get_starting_cylinder(self) -> int: """Get Starting Cylinder.""" - return (self.StartingCHS[1] - self.get_starting_sector()) * 4 + self.StartingCHS[2] + return ( + self.StartingCHS[1] - self.get_starting_sector() + ) * 4 + self.StartingCHS[2] def get_ending_cylinder(self) -> int: """Get Ending Cylinder.""" return (self.EndingCHS[1] - self.get_ending_sector()) * 4 + self.EndingCHS[2] - + def get_starting_lba(self) -> int: """Get Starting LBA (Logical Block Addressing).""" return self.StartingLBA - + def get_size_in_sectors(self) -> int: """Get Size in Sectors.""" return self.SizeInSectors diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index ba79b7c8b..17b6c8325 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -9,7 +9,7 @@ class MFTEntry(objects.StructType): """This represents the base MFT Record""" def get_signature(self) -> str: - signature = self.Signature.cast('string', max_length = 4, encoding = 'latin-1') + signature = self.Signature.cast("string", max_length=4, encoding="latin-1") return signature @@ -17,5 +17,7 @@ class MFTFileName(objects.StructType): """This represents an MFT $FILE_NAME Attribute""" def get_full_name(self) -> str: - output = self.Name.cast("string", encoding = "utf16", max_length = self.NameLength * 2, errors = "replace") + output = self.Name.cast( + "string", encoding="utf16", max_length=self.NameLength * 2, errors="replace" + ) return output diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 6e13ad45b..c0f2bd61a 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -19,7 +19,9 @@ def inet_ntop(address_family: int, packed_ip: Union[List[int], Array]) -> str: try: return socket.inet_ntop(address_family, bytes(packed_ip)) except AttributeError: - raise RuntimeError("This version of python does not have socket.inet_ntop, please upgrade") + raise RuntimeError( + "This version of python does not have socket.inet_ntop, please upgrade" + ) raise socket.error("[Errno 97] Address family not supported by protocol") @@ -54,15 +56,22 @@ class _TCP_LISTENER(objects.StructType): MIN_CREATETIME_YEAR = 1950 MAX_CREATETIME_YEAR = 2200 - def __init__(self, context: interfaces.context.ContextInterface, type_name: str, - object_info: interfaces.objects.ObjectInformation, size: int, - members: Dict[str, Tuple[int, interfaces.objects.Template]]) -> None: + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + size: int, + members: Dict[str, Tuple[int, interfaces.objects.Template]], + ) -> None: - super().__init__(context = context, - type_name = type_name, - object_info = object_info, - size = size, - members = members) + super().__init__( + context=context, + type_name=type_name, + object_info=object_info, + size=size, + members=members, + ) def get_address_family(self): try: @@ -73,7 +82,7 @@ class _TCP_LISTENER(objects.StructType): def get_owner(self): try: - return self.member('Owner').dereference() + return self.member("Owner").dereference() except exceptions.InvalidAddressException: return None @@ -88,9 +97,11 @@ class _TCP_LISTENER(objects.StructType): def get_owner_procname(self): if self.get_owner().is_valid(): if self.get_owner().has_valid_member("ImageFileName"): - return self.get_owner().ImageFileName.cast("string", - max_length = self.get_owner().ImageFileName.vol.count, - errors = "replace") + return self.get_owner().ImageFileName.cast( + "string", + max_length=self.get_owner().ImageFileName.vol.count, + errors="replace", + ) return None @@ -159,12 +170,17 @@ class _TCP_LISTENER(objects.StructType): try: if not self.get_address_family() in (AF_INET, AF_INET6): - vollog.debug("netw obj 0x{:x} invalid due to invalid address_family {}".format( - self.vol.offset, self.get_address_family())) + vollog.debug( + "netw obj 0x{:x} invalid due to invalid address_family {}".format( + self.vol.offset, self.get_address_family() + ) + ) return False except exceptions.InvalidAddressException: - vollog.debug(f"netw obj 0x{self.vol.offset:x} invalid due to invalid address access") + vollog.debug( + f"netw obj 0x{self.vol.offset:x} invalid due to invalid address access" + ) return False return True @@ -200,21 +216,32 @@ class _TCP_ENDPOINT(_TCP_LISTENER): def is_valid(self): if self.State not in self.State.choices.values(): - vollog.debug(f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid tcp state {self.State}") + vollog.debug( + f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid tcp state {self.State}" + ) return False try: if self.get_address_family() not in (AF_INET, AF_INET6): - vollog.debug(f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid address_family {self.get_address_family()}") + vollog.debug( + f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid address_family {self.get_address_family()}" + ) return False - if not self.get_local_address() and (not self.get_owner() or self.get_owner().UniqueProcessId == 0 - or self.get_owner().UniqueProcessId > 65535): - vollog.debug(f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid owner data") + if not self.get_local_address() and ( + not self.get_owner() + or self.get_owner().UniqueProcessId == 0 + or self.get_owner().UniqueProcessId > 65535 + ): + vollog.debug( + f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid owner data" + ) return False except exceptions.InvalidAddressException: - vollog.debug(f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid address access") + vollog.debug( + f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid address access" + ) return False return True @@ -225,30 +252,28 @@ class _UDP_ENDPOINT(_TCP_LISTENER): class _LOCAL_ADDRESS(objects.StructType): - @property def inaddr(self): return self.pData.dereference().dereference() class _LOCAL_ADDRESS_WIN10_UDP(objects.StructType): - @property def inaddr(self): return self.pData.dereference() win10_x64_class_types = { - '_TCP_ENDPOINT': _TCP_ENDPOINT, - '_TCP_LISTENER': _TCP_LISTENER, - '_UDP_ENDPOINT': _UDP_ENDPOINT, - '_LOCAL_ADDRESS': _LOCAL_ADDRESS, - '_LOCAL_ADDRESS_WIN10_UDP': _LOCAL_ADDRESS_WIN10_UDP + "_TCP_ENDPOINT": _TCP_ENDPOINT, + "_TCP_LISTENER": _TCP_LISTENER, + "_UDP_ENDPOINT": _UDP_ENDPOINT, + "_LOCAL_ADDRESS": _LOCAL_ADDRESS, + "_LOCAL_ADDRESS_WIN10_UDP": _LOCAL_ADDRESS_WIN10_UDP, } class_types = { - '_TCP_ENDPOINT': _TCP_ENDPOINT, - '_TCP_LISTENER': _TCP_LISTENER, - '_UDP_ENDPOINT': _UDP_ENDPOINT, - '_LOCAL_ADDRESS': _LOCAL_ADDRESS + "_TCP_ENDPOINT": _TCP_ENDPOINT, + "_TCP_LISTENER": _TCP_LISTENER, + "_UDP_ENDPOINT": _UDP_ENDPOINT, + "_LOCAL_ADDRESS": _LOCAL_ADDRESS, } diff --git a/volatility3/framework/symbols/windows/extensions/pe.py b/volatility3/framework/symbols/windows/extensions/pe.py index 2f271da5d..adee956f7 100644 --- a/volatility3/framework/symbols/windows/extensions/pe.py +++ b/volatility3/framework/symbols/windows/extensions/pe.py @@ -12,7 +12,6 @@ vollog = logging.getLogger(__name__) class IMAGE_DOS_HEADER(objects.StructType): - def get_nt_header(self) -> interfaces.objects.ObjectInterface: """Carve out the NT header from this DOS header. This reflects on the PE file's Machine type to create a 32- or 64-bit NT header structure. @@ -21,18 +20,24 @@ class IMAGE_DOS_HEADER(objects.StructType): <_IMAGE_NT_HEADERS> or <_IMAGE_NT_HEADERS64> instance """ - if self.e_magic != 0x5a4d: - raise ValueError(f"e_magic {self.e_magic:04X} is not a valid DOS signature.") + if self.e_magic != 0x5A4D: + raise ValueError( + f"e_magic {self.e_magic:04X} is not a valid DOS signature." + ) layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() - nt_header = self._context.object(symbol_table_name + constants.BANG + "_IMAGE_NT_HEADERS", - layer_name = layer_name, - offset = self.vol.offset + self.e_lfanew) + nt_header = self._context.object( + symbol_table_name + constants.BANG + "_IMAGE_NT_HEADERS", + layer_name=layer_name, + offset=self.vol.offset + self.e_lfanew, + ) if nt_header.Signature != 0x4550: - raise ValueError(f"NT header signature {nt_header.Signature:04X} is not a valid") + raise ValueError( + f"NT header signature {nt_header.Signature:04X} is not a valid" + ) # this checks if we need a PE32+ header if nt_header.FileHeader.Machine == 34404: @@ -40,8 +45,13 @@ class IMAGE_DOS_HEADER(objects.StructType): return nt_header - def replace_header_field(self, sect: interfaces.objects.ObjectInterface, header: bytes, - item: interfaces.objects.ObjectInterface, value: int) -> bytes: + def replace_header_field( + self, + sect: interfaces.objects.ObjectInterface, + header: bytes, + item: interfaces.objects.ObjectInterface, + value: int, + ) -> bytes: """Replaces a member in an _IMAGE_SECTION_HEADER structure. Args: @@ -57,10 +67,12 @@ class IMAGE_DOS_HEADER(objects.StructType): member_size = self._context.symbol_space.get_type(item.vol.type_name).size start = item.vol.offset - sect.vol.offset newval = objects.convert_value_to_data(value, int, item.vol.data_format) - result = header[:start] + newval + header[start + member_size:] + result = header[:start] + newval + header[start + member_size :] return result - def fix_image_base(self, raw_data: bytes, nt_header: interfaces.objects.ObjectInterface) -> bytes: + def fix_image_base( + self, raw_data: bytes, nt_header: interfaces.objects.ObjectInterface + ) -> bytes: """Fix the _OPTIONAL_HEADER.ImageBase value (which is either an unsigned long for 32-bit PE's or unsigned long long for 64-bit PE's) to match the address where the PE file was carved out of memory. @@ -73,17 +85,26 @@ class IMAGE_DOS_HEADER(objects.StructType): patched with the correct address """ - image_base_offset = nt_header.OptionalHeader.ImageBase.vol.offset - self.vol.offset + image_base_offset = ( + nt_header.OptionalHeader.ImageBase.vol.offset - self.vol.offset + ) image_base_type = nt_header.OptionalHeader.ImageBase.vol.type_name member_size = self._context.symbol_space.get_type(image_base_type).size try: - newval = objects.convert_value_to_data(self.vol.offset, int, - nt_header.OptionalHeader.ImageBase.vol.data_format) - new_pe = raw_data[:image_base_offset] + newval + raw_data[image_base_offset + member_size:] + newval = objects.convert_value_to_data( + self.vol.offset, int, nt_header.OptionalHeader.ImageBase.vol.data_format + ) + new_pe = ( + raw_data[:image_base_offset] + + newval + + raw_data[image_base_offset + member_size :] + ) except OverflowError: - vollog.warning("Volatility was unable to fix the image base for the PE file at base address {:#x}. " \ - "This will cause issues with many static analysis tools if you do not inform the " \ - "tool of the in-memory load address.".format(self.vol.offset)) + vollog.warning( + "Volatility was unable to fix the image base for the PE file at base address {:#x}. " + "This will cause issues with many static analysis tools if you do not inform the " + "tool of the in-memory load address.".format(self.vol.offset) + ) new_pe = raw_data return new_pe @@ -104,8 +125,9 @@ class IMAGE_DOS_HEADER(objects.StructType): section_alignment = nt_header.OptionalHeader.SectionAlignment - sect_header_size = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + - "_IMAGE_SECTION_HEADER").size + sect_header_size = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "_IMAGE_SECTION_HEADER" + ).size size_of_image = nt_header.OptionalHeader.SizeOfImage @@ -115,34 +137,51 @@ class IMAGE_DOS_HEADER(objects.StructType): read_layer = self._context.layers[layer_name] - raw_data = read_layer.read(self.vol.offset, nt_header.OptionalHeader.SizeOfImage, pad = True) + raw_data = read_layer.read( + self.vol.offset, nt_header.OptionalHeader.SizeOfImage, pad=True + ) # fix the PE image base before yielding the initial view of the data fixed_data = self.fix_image_base(raw_data, nt_header) yield 0, fixed_data - start_addr = nt_header.FileHeader.SizeOfOptionalHeader + \ - (nt_header.OptionalHeader.vol.offset - self.vol.offset) + start_addr = nt_header.FileHeader.SizeOfOptionalHeader + ( + nt_header.OptionalHeader.vol.offset - self.vol.offset + ) counter = 0 for sect in nt_header.get_sections(): if sect.VirtualAddress > size_of_image: - raise ValueError(f"Section VirtualAddress is too large: {sect.VirtualAddress}") + raise ValueError( + f"Section VirtualAddress is too large: {sect.VirtualAddress}" + ) if sect.Misc.VirtualSize > size_of_image: - raise ValueError(f"Section VirtualSize is too large: {sect.Misc.VirtualSize}") + raise ValueError( + f"Section VirtualSize is too large: {sect.Misc.VirtualSize}" + ) if sect.SizeOfRawData > size_of_image: - raise ValueError(f"Section SizeOfRawData is too large: {sect.SizeOfRawData}") + raise ValueError( + f"Section SizeOfRawData is too large: {sect.SizeOfRawData}" + ) if sect is not None: # It doesn't matter if this is too big, because it'll get overwritten by the later layers - sect_size = conversion.round(sect.Misc.VirtualSize, section_alignment, up = True) + sect_size = conversion.round( + sect.Misc.VirtualSize, section_alignment, up=True + ) sectheader = read_layer.read(sect.vol.offset, sect_header_size) - sectheader = self.replace_header_field(sect, sectheader, sect.PointerToRawData, sect.VirtualAddress) - sectheader = self.replace_header_field(sect, sectheader, sect.SizeOfRawData, sect_size) - sectheader = self.replace_header_field(sect, sectheader, sect.Misc.VirtualSize, sect_size) + sectheader = self.replace_header_field( + sect, sectheader, sect.PointerToRawData, sect.VirtualAddress + ) + sectheader = self.replace_header_field( + sect, sectheader, sect.SizeOfRawData, sect_size + ) + sectheader = self.replace_header_field( + sect, sectheader, sect.Misc.VirtualSize, sect_size + ) offset = start_addr + (counter * sect_header_size) yield offset, sectheader @@ -150,7 +189,6 @@ class IMAGE_DOS_HEADER(objects.StructType): class IMAGE_NT_HEADERS(objects.StructType): - def get_sections(self) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Iterate through the section headers for this PE file. @@ -160,20 +198,25 @@ class IMAGE_NT_HEADERS(objects.StructType): layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() - sect_header_size = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + - "_IMAGE_SECTION_HEADER").size - start_addr = self.FileHeader.SizeOfOptionalHeader + self.OptionalHeader.vol.offset + sect_header_size = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "_IMAGE_SECTION_HEADER" + ).size + start_addr = ( + self.FileHeader.SizeOfOptionalHeader + self.OptionalHeader.vol.offset + ) for i in range(self.FileHeader.NumberOfSections): sect_addr = start_addr + (i * sect_header_size) - yield self._context.object(symbol_table_name + constants.BANG + "_IMAGE_SECTION_HEADER", - offset = sect_addr, - layer_name = layer_name) + yield self._context.object( + symbol_table_name + constants.BANG + "_IMAGE_SECTION_HEADER", + offset=sect_addr, + layer_name=layer_name, + ) class_types = { - '_IMAGE_DOS_HEADER': IMAGE_DOS_HEADER, + "_IMAGE_DOS_HEADER": IMAGE_DOS_HEADER, # the 32- and 64-bit extensions behave the same way, but the underlying structure is different - '_IMAGE_NT_HEADERS': IMAGE_NT_HEADERS, - '_IMAGE_NT_HEADERS64': IMAGE_NT_HEADERS + "_IMAGE_NT_HEADERS": IMAGE_NT_HEADERS, + "_IMAGE_NT_HEADERS64": IMAGE_NT_HEADERS, } diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index 79ea60027..ac7f36a99 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -6,7 +6,14 @@ from typing import Dict, List, Optional, Tuple, Union from volatility3.plugins.windows.poolscanner import PoolConstraint -from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols +from volatility3.framework import ( + constants, + exceptions, + interfaces, + objects, + renderers, + symbols, +) from volatility3.framework.renderers import conversion vollog = logging.getLogger(__name__) @@ -19,15 +26,17 @@ class POOL_HEADER(objects.StructType): scan for. """ - def get_object(self, - constraint: PoolConstraint, - use_top_down: bool, - kernel_symbol_table: Optional[str] = None, - native_layer_name: Optional[str] = None) -> Optional[interfaces.objects.ObjectInterface]: + def get_object( + self, + constraint: PoolConstraint, + use_top_down: bool, + kernel_symbol_table: Optional[str] = None, + native_layer_name: Optional[str] = None, + ) -> Optional[interfaces.objects.ObjectInterface]: """Carve an object or data structure from a kernel pool allocation Args: - constraint: a PoolConstraint object used to get the pool allocation header object + constraint: a PoolConstraint object used to get the pool allocation header object use_top_down: for delineating how a windows version finds the size of the object body kernel_symbol_table: in case objects of a different symbol table are scanned for native_layer_name: the name of the layer where the data originally lived @@ -46,21 +55,25 @@ class POOL_HEADER(objects.StructType): # when checking for symbols from a table other than nt_symbols grab _OBJECT_HEADER from the kernel # because symbol_table_name will be different from kernel_symbol_table. if kernel_symbol_table: - object_header_type = self._context.symbol_space.get_type(kernel_symbol_table + constants.BANG + - "_OBJECT_HEADER") + object_header_type = self._context.symbol_space.get_type( + kernel_symbol_table + constants.BANG + "_OBJECT_HEADER" + ) else: # otherwise symbol_table_name *is* the kernel symbol table, so just use that. - object_header_type = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + - "_OBJECT_HEADER") + object_header_type = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "_OBJECT_HEADER" + ) pool_header_size = self.vol.size # if there is no object type, then just instantiate a structure if not executive: - mem_object = self._context.object(symbol_table_name + constants.BANG + type_name, - layer_name = self.vol.layer_name, - offset = self.vol.offset + pool_header_size, - native_layer_name = native_layer_name) + mem_object = self._context.object( + symbol_table_name + constants.BANG + type_name, + layer_name=self.vol.layer_name, + offset=self.vol.offset + pool_header_size, + native_layer_name=native_layer_name, + ) yield mem_object # otherwise we have an executive object in the pool @@ -72,36 +85,53 @@ class POOL_HEADER(objects.StructType): # use the top down approach for windows 8 and later if use_top_down: - body_offset = object_header_type.relative_child_offset('Body') - infomask_offset = object_header_type.relative_child_offset('InfoMask') - pointercount_offset = object_header_type.relative_child_offset('PointerCount') - pointercount_size = object_header_type.members['PointerCount'][1].size - optional_headers, lengths_of_optional_headers = self._calculate_optional_header_lengths( - self._context, symbol_table_name) - padding_available = None if 'PADDING_INFO' not in optional_headers else optional_headers.index( - 'PADDING_INFO') + body_offset = object_header_type.relative_child_offset("Body") + infomask_offset = object_header_type.relative_child_offset("InfoMask") + pointercount_offset = object_header_type.relative_child_offset( + "PointerCount" + ) + pointercount_size = object_header_type.members["PointerCount"][1].size + ( + optional_headers, + lengths_of_optional_headers, + ) = self._calculate_optional_header_lengths( + self._context, symbol_table_name + ) + padding_available = ( + None + if "PADDING_INFO" not in optional_headers + else optional_headers.index("PADDING_INFO") + ) max_optional_headers_length = sum(lengths_of_optional_headers) # define the starting and ending bounds for the scan start_offset = self.vol.offset + pool_header_size - addr_limit = min(max_optional_headers_length, self.BlockSize * alignment) + addr_limit = min( + max_optional_headers_length, self.BlockSize * alignment + ) # A single read is better than lots of little one-byte reads. # We're ok padding this, because the byte we'd check would be 0 which would only be valid if there # were no optional headers in the first place (ie, if we read too much for headers that don't exist, # but the bit we could read were valid) - infomask_data = self._context.layers[self.vol.layer_name].read(start_offset, - addr_limit + infomask_offset, - pad = True) + infomask_data = self._context.layers[self.vol.layer_name].read( + start_offset, addr_limit + infomask_offset, pad=True + ) # Addr stores the offset to the potential start of the OBJECT_HEADER from just after the POOL_HEADER # It will always be aligned to a particular alignment for addr in range(0, addr_limit, alignment): infomask_value = infomask_data[addr + infomask_offset] pointercount_value = int.from_bytes( - infomask_data[addr + pointercount_offset:addr + pointercount_offset + pointercount_size], - byteorder = 'little', - signed = True) + infomask_data[ + addr + + pointercount_offset : addr + + pointercount_offset + + pointercount_size + ], + byteorder="little", + signed=True, + ) if not 0x1000000 > pointercount_value >= 0: continue @@ -130,9 +160,18 @@ class POOL_HEADER(objects.StructType): # --------------- if addr - optional_headers_length < 0: continue - padding_length, = struct.unpack( - "= padding_length > addr: continue - with contextlib.suppress(TypeError, exceptions.InvalidAddressException): - mem_object = self._context.object(symbol_table_name + constants.BANG + type_name, - layer_name = self.vol.layer_name, - offset = addr + body_offset + start_offset, - native_layer_name = native_layer_name) + with contextlib.suppress( + TypeError, exceptions.InvalidAddressException + ): + mem_object = self._context.object( + symbol_table_name + constants.BANG + type_name, + layer_name=self.vol.layer_name, + offset=addr + body_offset + start_offset, + native_layer_name=native_layer_name, + ) if mem_object.is_valid(): yield mem_object # use the bottom up approach for windows 7 and earlier else: - type_size = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + type_name).size + type_size = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + type_name + ).size if constraint.additional_structures: for additional_structure in constraint.additional_structures: type_size += self._context.symbol_space.get_type( - symbol_table_name + constants.BANG + additional_structure).size + symbol_table_name + constants.BANG + additional_structure + ).size - rounded_size = conversion.round(type_size, alignment, up = True) + rounded_size = conversion.round(type_size, alignment, up=True) - mem_object = self._context.object(symbol_table_name + constants.BANG + type_name, - layer_name = self.vol.layer_name, - offset = self.vol.offset + self.BlockSize * alignment - rounded_size, - native_layer_name = native_layer_name) + mem_object = self._context.object( + symbol_table_name + constants.BANG + type_name, + layer_name=self.vol.layer_name, + offset=self.vol.offset + self.BlockSize * alignment - rounded_size, + native_layer_name=native_layer_name, + ) with contextlib.suppress(TypeError, exceptions.InvalidAddressException): if mem_object.is_valid(): @@ -170,16 +218,26 @@ class POOL_HEADER(objects.StructType): @classmethod @functools.lru_cache() - def _calculate_optional_header_lengths(cls, context: interfaces.context.ContextInterface, - symbol_table_name: str) -> Tuple[List[str], List[int]]: + def _calculate_optional_header_lengths( + cls, context: interfaces.context.ContextInterface, symbol_table_name: str + ) -> Tuple[List[str], List[int]]: headers = [] sizes = [] for header in [ - 'CREATOR_INFO', 'NAME_INFO', 'HANDLE_INFO', 'QUOTA_INFO', 'PROCESS_INFO', 'AUDIT_INFO', 'EXTENDED_INFO', - 'HANDLE_REVOCATION_INFO', 'PADDING_INFO' + "CREATOR_INFO", + "NAME_INFO", + "HANDLE_INFO", + "QUOTA_INFO", + "PROCESS_INFO", + "AUDIT_INFO", + "EXTENDED_INFO", + "HANDLE_REVOCATION_INFO", + "PADDING_INFO", ]: with contextlib.suppress(AttributeError, exceptions.SymbolError): - type_name = f"{symbol_table_name}{constants.BANG}_OBJECT_HEADER_{header}" + type_name = ( + f"{symbol_table_name}{constants.BANG}_OBJECT_HEADER_{header}" + ) header_type = context.symbol_space.get_type(type_name) headers.append(header) sizes.append(header_type.size) @@ -222,7 +280,9 @@ class POOL_TRACKER_BIG_PAGES(objects.StructType): # Enumeration._generate_inverse_choices() raises ValueError because multiple enum names map to the same # value in the kernel _POOL_TYPE so create a custom mapping here and take the first match symbol_table_name = self.vol.type_name.split(constants.BANG)[0] - pool_type_enum = self._context.symbol_space.get_enumeration(symbol_table_name + constants.BANG + "_POOL_TYPE") + pool_type_enum = self._context.symbol_space.get_enumeration( + symbol_table_name + constants.BANG + "_POOL_TYPE" + ) for k, v in pool_type_enum.choices.items(): if v not in self.pool_type_lookup: self.pool_type_lookup[v] = k @@ -236,16 +296,20 @@ class POOL_TRACKER_BIG_PAGES(objects.StructType): def get_key(self) -> str: """Returns the Key value as a 4 character string""" - tag_bytes = objects.convert_value_to_data(self.Key, int, objects.DataFormatInfo(4, "little", False)) - return "".join([chr(x) if 32 < x < 127 else '' for x in tag_bytes]) + tag_bytes = objects.convert_value_to_data( + self.Key, int, objects.DataFormatInfo(4, "little", False) + ) + return "".join([chr(x) if 32 < x < 127 else "" for x in tag_bytes]) def get_pool_type(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: """Returns the enum name for the PoolType value on applicable systems""" # Not applicable until Vista - if hasattr(self, 'PoolType'): + if hasattr(self, "PoolType"): if not self.pool_type_lookup: self._generate_pool_type_lookup() - return self.pool_type_lookup.get(self.PoolType, f"Unknown choice {self.PoolType}") + return self.pool_type_lookup.get( + self.PoolType, f"Unknown choice {self.PoolType}" + ) else: return renderers.NotApplicableValue() @@ -262,16 +326,21 @@ class ExecutiveObject(interfaces.objects.ObjectInterface): """This is used as a "mixin" that provides all kernel executive objects with a means of finding their own object header.""" - def get_object_header(self) -> 'OBJECT_HEADER': + def get_object_header(self) -> "OBJECT_HEADER": if constants.BANG not in self.vol.type_name: - raise ValueError(f"Invalid symbol table name syntax (no {constants.BANG} found)") + raise ValueError( + f"Invalid symbol table name syntax (no {constants.BANG} found)" + ) symbol_table_name = self.vol.type_name.split(constants.BANG)[0] - body_offset = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + - "_OBJECT_HEADER").relative_child_offset("Body") - return self._context.object(symbol_table_name + constants.BANG + "_OBJECT_HEADER", - layer_name = self.vol.layer_name, - offset = self.vol.offset - body_offset, - native_layer_name = self.vol.native_layer_name) + body_offset = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "_OBJECT_HEADER" + ).relative_child_offset("Body") + return self._context.object( + symbol_table_name + constants.BANG + "_OBJECT_HEADER", + layer_name=self.vol.layer_name, + offset=self.vol.offset - body_offset, + native_layer_name=self.vol.native_layer_name, + ) class OBJECT_HEADER(objects.StructType): @@ -292,7 +361,9 @@ class OBJECT_HEADER(objects.StructType): return True - def get_object_type(self, type_map: Dict[int, str], cookie: int = None) -> Optional[str]: + def get_object_type( + self, type_map: Dict[int, str], cookie: int = None + ) -> Optional[str]: """Across all Windows versions, the _OBJECT_HEADER embeds details on the type of object (i.e. process, file) but the way its embedded differs between versions. @@ -300,12 +371,12 @@ class OBJECT_HEADER(objects.StructType): This API abstracts away those details. """ - if self.vol.get('object_header_object_type', None) is not None: + if self.vol.get("object_header_object_type", None) is not None: return self.vol.object_header_object_type try: # vista and earlier have a Type member - self._vol['object_header_object_type'] = self.Type.Name.String + self._vol["object_header_object_type"] = self.Type.Name.String except AttributeError: # windows 7 and later have a TypeIndex, but windows 10 # further encodes the index value with nt1!ObHeaderCookie @@ -314,13 +385,15 @@ class OBJECT_HEADER(objects.StructType): except (AttributeError, TypeError): type_index = self.TypeIndex - self._vol['object_header_object_type'] = type_map.get(type_index) + self._vol["object_header_object_type"] = type_map.get(type_index) return self.vol.object_header_object_type @property def NameInfo(self) -> interfaces.objects.ObjectInterface: if constants.BANG not in self.vol.type_name: - raise ValueError(f"Invalid symbol table name syntax (no {constants.BANG} found)") + raise ValueError( + f"Invalid symbol table name syntax (no {constants.BANG} found)" + ) symbol_table_name = self.vol.type_name.split(constants.BANG)[0] @@ -334,22 +407,33 @@ class OBJECT_HEADER(objects.StructType): kvo = layer.config.get("kernel_virtual_offset", None) if kvo is None: - raise AttributeError(f"Could not find kernel_virtual_offset for layer: {self.vol.layer_name}") + raise AttributeError( + f"Could not find kernel_virtual_offset for layer: {self.vol.layer_name}" + ) - ntkrnlmp = self._context.module(symbol_table_name, layer_name = self.vol.layer_name, offset = kvo) + ntkrnlmp = self._context.module( + symbol_table_name, layer_name=self.vol.layer_name, offset=kvo + ) address = ntkrnlmp.get_symbol("ObpInfoMaskToOffset").address calculated_index = self.InfoMask & (name_info_bit | (name_info_bit - 1)) - header_offset = self._context.object(symbol_table_name + constants.BANG + "unsigned char", - layer_name = self.vol.native_layer_name, - offset = kvo + address + calculated_index) + header_offset = self._context.object( + symbol_table_name + constants.BANG + "unsigned char", + layer_name=self.vol.native_layer_name, + offset=kvo + address + calculated_index, + ) if header_offset == 0: - raise ValueError("Could not find _OBJECT_HEADER_NAME_INFO for object at {} of layer {}".format( - self.vol.offset, self.vol.layer_name)) + raise ValueError( + "Could not find _OBJECT_HEADER_NAME_INFO for object at {} of layer {}".format( + self.vol.offset, self.vol.layer_name + ) + ) - header = self._context.object(symbol_table_name + constants.BANG + "_OBJECT_HEADER_NAME_INFO", - layer_name = self.vol.layer_name, - offset = self.vol.offset - header_offset, - native_layer_name = self.vol.native_layer_name) + header = self._context.object( + symbol_table_name + constants.BANG + "_OBJECT_HEADER_NAME_INFO", + layer_name=self.vol.layer_name, + offset=self.vol.offset - header_offset, + native_layer_name=self.vol.native_layer_name, + ) return header diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index c71fcf49b..fbd3ead8e 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -8,11 +8,15 @@ import struct from typing import Iterable, Optional, Union from volatility3.framework import constants, exceptions, interfaces, objects -from volatility3.framework.layers.registry import RegistryFormatException, RegistryHive, RegistryInvalidIndex +from volatility3.framework.layers.registry import ( + RegistryFormatException, + RegistryHive, + RegistryInvalidIndex, +) vollog = logging.getLogger(__name__) -BIG_DATA_MAXLEN = 0x3fd8 +BIG_DATA_MAXLEN = 0x3FD8 class RegValueTypes(enum.Enum): @@ -49,20 +53,20 @@ class RegKeyFlags(enum.IntEnum): class HMAP_ENTRY(objects.StructType): - def get_block_offset(self) -> int: try: - return (self.PermanentBinAddress ^ (self.PermanentBinAddress & 0xf)) + self.BlockOffset + return ( + self.PermanentBinAddress ^ (self.PermanentBinAddress & 0xF) + ) + self.BlockOffset except AttributeError: return self.BlockAddress class CMHIVE(objects.StructType): - def is_valid(self) -> bool: """Determine if the object is valid.""" try: - return self.Hive.Signature == 0xbee0bee0 + return self.Hive.Signature == 0xBEE0BEE0 except exceptions.InvalidAddressException: return False @@ -75,7 +79,9 @@ class CMHIVE(objects.StructType): """ for attr in ["FileFullPath", "FileUserName", "HiveRootPath"]: - with contextlib.suppress(AttributeError, exceptions.InvalidAddressException): + with contextlib.suppress( + AttributeError, exceptions.InvalidAddressException + ): name = getattr(self, attr) if name.Length > 0: return name.get_string() @@ -94,7 +100,10 @@ class CM_KEY_BODY(objects.StructType): checking for Flags that contain KEY_HIVE_ENTRY.""" # _CM_KEY_BODY.Trans introduced in Win10 14393 - if hasattr(self, "Trans") and RegKeyFlags.KEY_HIVE_ENTRY & kcb_flags == RegKeyFlags.KEY_HIVE_ENTRY: + if ( + hasattr(self, "Trans") + and RegKeyFlags.KEY_HIVE_ENTRY & kcb_flags == RegKeyFlags.KEY_HIVE_ENTRY + ): return True return False @@ -112,10 +121,13 @@ class CM_KEY_BODY(objects.StructType): break output.append( - kcb.NameBlock.Name.cast("string", - encoding = "utf8", - max_length = kcb.NameBlock.NameLength, - errors = "replace")) + kcb.NameBlock.Name.cast( + "string", + encoding="utf8", + max_length=kcb.NameBlock.NameLength, + errors="replace", + ) + ) kcb = kcb.ParentKcb return "\\".join(reversed(output)) @@ -125,7 +137,9 @@ class CM_KEY_NODE(objects.StructType): def get_volatile(self) -> bool: if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): - raise ValueError("Cannot determine volatility of registry key without an offset in a RegistryHive layer") + raise ValueError( + "Cannot determine volatility of registry key without an offset in a RegistryHive layer" + ) return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterable[interfaces.objects.ObjectInterface]: @@ -139,40 +153,50 @@ class CM_KEY_NODE(objects.StructType): yield from self._get_subkeys_recursive(hive, subkey_node) def _get_subkeys_recursive( - self, hive: RegistryHive, - node: interfaces.objects.ObjectInterface) -> Iterable[interfaces.objects.ObjectInterface]: + self, hive: RegistryHive, node: interfaces.objects.ObjectInterface + ) -> Iterable[interfaces.objects.ObjectInterface]: """Recursively descend a node returning subkeys.""" # The keylist appears to include 4 bytes of key name after each value # We can either double the list and only use the even items, or # We could change the array type to a struct with both parts try: - signature = node.cast('string', max_length = 2, encoding = 'latin-1') + signature = node.cast("string", max_length=2, encoding="latin-1") except (exceptions.InvalidAddressException, RegistryFormatException): return listjump = None - if signature == 'ri': + if signature == "ri": listjump = 1 - elif signature == 'lh' or signature == 'lf': + elif signature == "lh" or signature == "lf": listjump = 2 elif node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"): yield node else: - vollog.debug("Unexpected node type encountered when traversing subkeys: {}, signature: {}".format( - node.vol.type_name, signature)) + vollog.debug( + "Unexpected node type encountered when traversing subkeys: {}, signature: {}".format( + node.vol.type_name, signature + ) + ) if listjump: node.List.count = node.Count * listjump for subnode_offset in node.List[::listjump]: - if (subnode_offset & 0x7fffffff) > hive.maximum_address: - vollog.log(constants.LOGLEVEL_VVV, - f"Node found with address outside the valid Hive size: {hex(subnode_offset)}") + if (subnode_offset & 0x7FFFFFFF) > hive.maximum_address: + vollog.log( + constants.LOGLEVEL_VVV, + f"Node found with address outside the valid Hive size: {hex(subnode_offset)}", + ) else: try: subnode = hive.get_node(subnode_offset) - except (exceptions.InvalidAddressException, RegistryFormatException): - vollog.log(constants.LOGLEVEL_VVV, - f"Failed to get node at {hex(subnode_offset)}, skipping") + except ( + exceptions.InvalidAddressException, + RegistryFormatException, + ): + vollog.log( + constants.LOGLEVEL_VVV, + f"Failed to get node at {hex(subnode_offset)}, skipping", + ) continue yield from self._get_subkeys_recursive(hive, subnode) @@ -192,7 +216,7 @@ class CM_KEY_NODE(objects.StructType): except (RegistryInvalidIndex, RegistryFormatException) as excp: vollog.debug(f"Invalid address {excp}") continue - if node.vol.type_name.endswith(constants.BANG + '_CM_KEY_VALUE'): + if node.vol.type_name.endswith(constants.BANG + "_CM_KEY_VALUE"): yield node except (exceptions.InvalidAddressException, RegistryFormatException) as excp: vollog.debug(f"Invalid address in get_values iteration: {excp}") @@ -202,7 +226,7 @@ class CM_KEY_NODE(objects.StructType): """Gets the name for the current key node""" namelength = self.NameLength self.Name.count = namelength - return self.Name.cast("string", max_length = namelength, encoding = "latin-1") + return self.Name.cast("string", max_length=namelength, encoding="latin-1") def get_key_path(self) -> str: reg = self._context.layers[self.vol.layer_name] @@ -212,8 +236,8 @@ class CM_KEY_NODE(objects.StructType): # if self.vol.offset == reg.get_node(reg.root_cell_offset).vol.offset: if self.vol.offset == reg.root_cell_offset + 4: # return the last part of the hive name for the root entry - return reg.get_name().split('\\')[-1] - return reg.get_node(self.Parent).get_key_path() + '\\' + self.get_name() + return reg.get_name().split("\\")[-1] + return reg.get_node(self.Parent).get_key_path() + "\\" + self.get_name() class CM_KEY_VALUE(objects.StructType): @@ -223,7 +247,7 @@ class CM_KEY_VALUE(objects.StructType): """Gets the name for the current key value""" namelength = self.NameLength self.Name.count = namelength - return self.Name.cast("string", max_length = namelength, encoding = "latin-1") + return self.Name.cast("string", max_length=namelength, encoding="latin-1") def decode_data(self) -> Union[int, bytes]: """Properly decodes the data associated with the value node""" @@ -238,9 +262,11 @@ class CM_KEY_VALUE(objects.StructType): # If the high-bit is set if datalen & 0x80000000: # Remove the high bit - datalen = datalen & 0x7fffffff - if (0 > datalen or datalen > 4): - raise ValueError(f"Unable to read inline registry value with excessive length: {datalen}") + datalen = datalen & 0x7FFFFFFF + if 0 > datalen or datalen > 4: + raise ValueError( + f"Unable to read inline registry value with excessive length: {datalen}" + ) else: data = layer.read(self.Data.vol.offset, datalen) elif layer.hive.Version == 5 and datalen > 0x4000: @@ -250,10 +276,17 @@ class CM_KEY_VALUE(objects.StructType): for i in range(big_data.Count): # The value 4 should actually be unsigned-int.size, but since it's a file format that shouldn't change # the direct value 4 can be used instead - block_offset = layer.get_cell(big_data.List + (i * 4)).cast("unsigned int") - if isinstance(block_offset, int) and block_offset < layer.maximum_address: + block_offset = layer.get_cell(big_data.List + (i * 4)).cast( + "unsigned int" + ) + if ( + isinstance(block_offset, int) + and block_offset < layer.maximum_address + ): amount = min(BIG_DATA_MAXLEN, datalen) - data += layer.read(offset = layer.get_cell(block_offset).vol.offset, length = amount) + data += layer.read( + offset=layer.get_cell(block_offset).vol.offset, length=amount + ) datalen -= amount else: # Suspect Data actually points to a Cell, @@ -263,27 +296,38 @@ class CM_KEY_VALUE(objects.StructType): self_type = RegValueTypes(self.Type) if self_type == RegValueTypes.REG_DWORD: if len(data) != struct.calcsize("L"): - raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}") - res, = struct.unpack(">L", data) + raise ValueError( + f"Size of data does not match the type of registry value {self.get_name()}" + ) + (res,) = struct.unpack(">L", data) return res if self_type == RegValueTypes.REG_QWORD: if len(data) != struct.calcsize(" Union[int, interfaces.renderers.BaseAbsentValue]: """Return the pid of the process, if any.""" - if self.State.description != "SERVICE_RUNNING" or "PROCESS" not in self.get_type(): + if ( + self.State.description != "SERVICE_RUNNING" + or "PROCESS" not in self.get_type() + ): return renderers.NotApplicableValue() try: @@ -44,35 +47,31 @@ class SERVICE_RECORD(objects.StructType): # or kernel driver, the binary path is stored differently try: if "PROCESS" in self.get_type(): - return self.ServiceProcess.BinaryPath.dereference().cast("string", - encoding = "utf-16", - errors = "replace", - max_length = 512) + return self.ServiceProcess.BinaryPath.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) else: - return self.DriverName.dereference().cast("string", - encoding = "utf-16", - errors = "replace", - max_length = 512) + return self.DriverName.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) except exceptions.InvalidAddressException: return renderers.UnreadableValue() def get_name(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: """Returns the service name.""" try: - return self.ServiceName.dereference().cast("string", - encoding = "utf-16", - errors = "replace", - max_length = 512) + return self.ServiceName.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) except exceptions.InvalidAddressException: return renderers.UnreadableValue() def get_display(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: """Returns the service display.""" try: - return self.DisplayName.dereference().cast("string", - encoding = "utf-16", - errors = "replace", - max_length = 512) + return self.DisplayName.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) except exceptions.InvalidAddressException: return renderers.UnreadableValue() @@ -80,16 +79,16 @@ class SERVICE_RECORD(objects.StructType): """Returns the binary types.""" SERVICE_TYPE_FLAGS = { - 'SERVICE_KERNEL_DRIVER': 1, - 'SERVICE_FILE_SYSTEM_DRIVER': 2, - 'SERVICE_ADAPTOR': 4, - 'SERVICE_RECOGNIZER_DRIVER': 8, - 'SERVICE_WIN32_OWN_PROCESS': 16, - 'SERVICE_WIN32_SHARE_PROCESS': 32, - 'SERVICE_INTERACTIVE_PROCESS': 256 + "SERVICE_KERNEL_DRIVER": 1, + "SERVICE_FILE_SYSTEM_DRIVER": 2, + "SERVICE_ADAPTOR": 4, + "SERVICE_RECOGNIZER_DRIVER": 8, + "SERVICE_WIN32_OWN_PROCESS": 16, + "SERVICE_WIN32_SHARE_PROCESS": 32, + "SERVICE_INTERACTIVE_PROCESS": 256, } - type_flags = Flags(choices = SERVICE_TYPE_FLAGS) + type_flags = Flags(choices=SERVICE_TYPE_FLAGS) return "|".join(type_flags(self.Type)) def traverse(self): @@ -125,4 +124,4 @@ class SERVICE_HEADER(objects.StructType): return False -class_types = {'_SERVICE_RECORD': SERVICE_RECORD, '_SERVICE_HEADER': SERVICE_HEADER} +class_types = {"_SERVICE_RECORD": SERVICE_RECORD, "_SERVICE_HEADER": SERVICE_HEADER} diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index 4809aafcf..82ec31ccb 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -19,220 +19,79 @@ from volatility3.framework.layers import physical, msf, resources vollog = logging.getLogger(__name__) primitives = { - 0x03: ("void", { - "endian": "little", - "kind": "void", - "signed": True, - "size": 0 - }), - 0x08: ("HRESULT", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 4 - }), - 0x10: ("char", { - "endian": "little", - "kind": "char", - "signed": True, - "size": 1 - }), - 0x20: ("unsigned char", { - "endian": "little", - "kind": "char", - "signed": False, - "size": 1 - }), - 0x68: ("int8", { - "endian": "little", - "kind": "int", - "signed": True, - "size": 1 - }), - 0x69: ("uint8", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 1 - }), - 0x70: ("char", { - "endian": "little", - "kind": "char", - "signed": True, - "size": 1 - }), - 0x71: ("wchar", { - "endian": "little", - "kind": "int", - "signed": True, - "size": 2 - }), + 0x03: ("void", {"endian": "little", "kind": "void", "signed": True, "size": 0}), + 0x08: ("HRESULT", {"endian": "little", "kind": "int", "signed": False, "size": 4}), + 0x10: ("char", {"endian": "little", "kind": "char", "signed": True, "size": 1}), + 0x20: ( + "unsigned char", + {"endian": "little", "kind": "char", "signed": False, "size": 1}, + ), + 0x68: ("int8", {"endian": "little", "kind": "int", "signed": True, "size": 1}), + 0x69: ("uint8", {"endian": "little", "kind": "int", "signed": False, "size": 1}), + 0x70: ("char", {"endian": "little", "kind": "char", "signed": True, "size": 1}), + 0x71: ("wchar", {"endian": "little", "kind": "int", "signed": True, "size": 2}), # 0x7a: ("rchar16", {}), # 0x7b: ("rchar32", {}), - 0x11: ("short", { - "endian": "little", - "kind": "int", - "signed": True, - "size": 2 - }), - 0x21: ("unsigned short", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 2 - }), - 0x72: ("short", { - "endian": "little", - "kind": "int", - "signed": True, - "size": 2 - }), - 0x73: ("unsigned short", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 2 - }), - 0x12: ("long", { - "endian": "little", - "kind": "int", - "signed": True, - "size": 4 - }), - 0x22: ("unsigned long", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 4 - }), - 0x74: ("int", { - "endian": "little", - "kind": "int", - "signed": True, - "size": 4 - }), - 0x75: ("unsigned int", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 4 - }), - 0x13: ("long long", { - "endian": "little", - "kind": "int", - "signed": True, - "size": 8 - }), - 0x23: ("unsigned long long", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 8 - }), - 0x76: ("long long", { - "endian": "little", - "kind": "int", - "signed": True, - "size": 8 - }), - 0x77: ("unsigned long long", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 8 - }), - 0x14: ("int128", { - "endian": "little", - "kind": "int", - "signed": True, - "size": 16 - }), - 0x24: ("uint128", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 16 - }), - 0x78: ("int128", { - "endian": "little", - "kind": "int", - "signed": True, - "size": 16 - }), - 0x79: ("uint128", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 16 - }), - 0x46: ("f16", { - "endian": "little", - "kind": "float", - "signed": True, - "size": 2 - }), - 0x40: ("f32", { - "endian": "little", - "kind": "float", - "signed": True, - "size": 4 - }), - 0x45: ("f32pp", { - "endian": "little", - "kind": "float", - "signed": True, - "size": 4 - }), - 0x44: ("f48", { - "endian": "little", - "kind": "float", - "signed": True, - "size": 6 - }), - 0x41: ("double", { - "endian": "little", - "kind": "float", - "signed": True, - "size": 8 - }), - 0x42: ("f80", { - "endian": "little", - "kind": "float", - "signed": True, - "size": 10 - }), - 0x43: ("f128", { - "endian": "little", - "kind": "float", - "signed": True, - "size": 16 - }) + 0x11: ("short", {"endian": "little", "kind": "int", "signed": True, "size": 2}), + 0x21: ( + "unsigned short", + {"endian": "little", "kind": "int", "signed": False, "size": 2}, + ), + 0x72: ("short", {"endian": "little", "kind": "int", "signed": True, "size": 2}), + 0x73: ( + "unsigned short", + {"endian": "little", "kind": "int", "signed": False, "size": 2}, + ), + 0x12: ("long", {"endian": "little", "kind": "int", "signed": True, "size": 4}), + 0x22: ( + "unsigned long", + {"endian": "little", "kind": "int", "signed": False, "size": 4}, + ), + 0x74: ("int", {"endian": "little", "kind": "int", "signed": True, "size": 4}), + 0x75: ( + "unsigned int", + {"endian": "little", "kind": "int", "signed": False, "size": 4}, + ), + 0x13: ("long long", {"endian": "little", "kind": "int", "signed": True, "size": 8}), + 0x23: ( + "unsigned long long", + {"endian": "little", "kind": "int", "signed": False, "size": 8}, + ), + 0x76: ("long long", {"endian": "little", "kind": "int", "signed": True, "size": 8}), + 0x77: ( + "unsigned long long", + {"endian": "little", "kind": "int", "signed": False, "size": 8}, + ), + 0x14: ("int128", {"endian": "little", "kind": "int", "signed": True, "size": 16}), + 0x24: ("uint128", {"endian": "little", "kind": "int", "signed": False, "size": 16}), + 0x78: ("int128", {"endian": "little", "kind": "int", "signed": True, "size": 16}), + 0x79: ("uint128", {"endian": "little", "kind": "int", "signed": False, "size": 16}), + 0x46: ("f16", {"endian": "little", "kind": "float", "signed": True, "size": 2}), + 0x40: ("f32", {"endian": "little", "kind": "float", "signed": True, "size": 4}), + 0x45: ("f32pp", {"endian": "little", "kind": "float", "signed": True, "size": 4}), + 0x44: ("f48", {"endian": "little", "kind": "float", "signed": True, "size": 6}), + 0x41: ("double", {"endian": "little", "kind": "float", "signed": True, "size": 8}), + 0x42: ("f80", {"endian": "little", "kind": "float", "signed": True, "size": 10}), + 0x43: ("f128", {"endian": "little", "kind": "float", "signed": True, "size": 16}), } indirections = { - 0x100: ("pointer16", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 2 - }), - 0x400: ("pointer32", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 4 - }), - 0x600: ("pointer64", { - "endian": "little", - "kind": "int", - "signed": False, - "size": 8 - }) + 0x100: ( + "pointer16", + {"endian": "little", "kind": "int", "signed": False, "size": 2}, + ), + 0x400: ( + "pointer32", + {"endian": "little", "kind": "int", "signed": False, "size": 4}, + ), + 0x600: ( + "pointer64", + {"endian": "little", "kind": "int", "signed": False, "size": 8}, + ), } class ForwardArrayCount: - def __init__(self, size, element_type): self.element_type = element_type self.size = size @@ -259,19 +118,25 @@ class PdbReader: sized data following. """ - def __init__(self, - context: interfaces.context.ContextInterface, - location: str, - database_name: Optional[str] = None, - progress_callback: constants.ProgressCallback = None) -> None: + def __init__( + self, + context: interfaces.context.ContextInterface, + location: str, + database_name: Optional[str] = None, + progress_callback: constants.ProgressCallback = None, + ) -> None: self._layer_name, self._context = self.load_pdb_layer(context, location) self._dbiheader: Optional[interfaces.objects.ObjectInterface] = None if not progress_callback: progress_callback = lambda x, y: None self._progress_callback = progress_callback self.types: List[ - Tuple[interfaces.objects.ObjectInterface, Optional[str], interfaces.objects.ObjectInterface]] = [ - ] + Tuple[ + interfaces.objects.ObjectInterface, + Optional[str], + interfaces.objects.ObjectInterface, + ] + ] = [] self.bases: Dict[str, Any] = {} self.user_types: Dict[str, Any] = {} self.enumerations: Dict[str, Any] = {} @@ -290,29 +155,42 @@ class PdbReader: return self._layer_name @classmethod - def load_pdb_layer(cls, context: interfaces.context.ContextInterface, - location: str) -> Tuple[str, interfaces.context.ContextInterface]: + def load_pdb_layer( + cls, context: interfaces.context.ContextInterface, location: str + ) -> Tuple[str, interfaces.context.ContextInterface]: """Loads a PDB file into a layer within the context and returns the name of the new layer. Note: the context may be changed by this method """ physical_layer_name = context.layers.free_layer_name("FileLayer") - physical_config_path = interfaces.configuration.path_join("pdbreader", physical_layer_name) + physical_config_path = interfaces.configuration.path_join( + "pdbreader", physical_layer_name + ) # Create the file layer # This must be specific to get us started, setup the config and run new_context = context.clone() - new_context.config[interfaces.configuration.path_join(physical_config_path, "location")] = location + new_context.config[ + interfaces.configuration.path_join(physical_config_path, "location") + ] = location - physical_layer = physical.FileLayer(new_context, physical_config_path, physical_layer_name) + physical_layer = physical.FileLayer( + new_context, physical_config_path, physical_layer_name + ) new_context.add_layer(physical_layer) # Add on the MSF format layer msf_layer_name = context.layers.free_layer_name("MSFLayer") - msf_config_path = interfaces.configuration.path_join("pdbreader", msf_layer_name) - new_context.config[interfaces.configuration.path_join(msf_config_path, "base_layer")] = physical_layer_name - msf_layer = msf.PdbMultiStreamFormat(new_context, msf_config_path, msf_layer_name) + msf_config_path = interfaces.configuration.path_join( + "pdbreader", msf_layer_name + ) + new_context.config[ + interfaces.configuration.path_join(msf_config_path, "base_layer") + ] = physical_layer_name + msf_layer = msf.PdbMultiStreamFormat( + new_context, msf_config_path, msf_layer_name + ) new_context.add_layer(msf_layer) msf_layer.read_streams() @@ -330,7 +208,7 @@ class PdbReader: def read_necessary_streams(self): """Read streams to populate the various internal components for a PDB table.""" - if not self.metadata['windows'].get('pdb', None): + if not self.metadata["windows"].get("pdb", None): self.read_pdb_info_stream() if not self.user_types: self.read_tpi_stream() @@ -358,28 +236,37 @@ class PdbReader: type_references = self._read_info_stream(4, "IPI", ipi_list) for name in type_references.keys(): # This doesn't break, because we want to use the last string/pdbname in the list - if name.endswith('.pdb'): - self._database_name = name.split('\\')[-1] + if name.endswith(".pdb"): + self._database_name = name.split("\\")[-1] except ValueError: return None def _read_info_stream(self, stream_number, stream_name, info_list): vollog.debug(f"Reading {stream_name}") - info_layer = self._context.layers.get(self._layer_name + "_stream" + str(stream_number), None) + info_layer = self._context.layers.get( + self._layer_name + "_stream" + str(stream_number), None + ) if not info_layer: raise ValueError(f"No {stream_name} stream available") - module = self._context.module(module_name = info_layer.pdb_symbol_table, - layer_name = info_layer.name, - offset = 0) - header = module.object(object_type = "TPI_HEADER", offset = 0) + module = self._context.module( + module_name=info_layer.pdb_symbol_table, + layer_name=info_layer.name, + offset=0, + ) + header = module.object(object_type="TPI_HEADER", offset=0) # Check the header if not (56 <= header.header_size < 1024): raise ValueError(f"{stream_name} Stream Header size outside normal bounds") if header.index_min < 4096: - raise ValueError(f"Minimum {stream_name} index is 4096, found: {header.index_min}") + raise ValueError( + f"Minimum {stream_name} index is 4096, found: {header.index_min}" + ) if header.index_max < header.index_min: - raise ValueError("Maximum {} index is smaller than minimum TPI index, found: {} < {} ".format( - stream_name, header.index_max, header.index_min)) + raise ValueError( + "Maximum {} index is smaller than minimum TPI index, found: {} < {} ".format( + stream_name, header.index_max, header.index_min + ) + ) # Reset the state info_references: Dict[str, int] = {} offset = header.header_size @@ -388,16 +275,18 @@ class PdbReader: length_len = module.get_type(length_type).size info_index = 1 while info_layer.maximum_address - offset > 0: - self._progress_callback(offset * 100 / info_layer.maximum_address, "Reading TPI layer") - length = module.object(object_type = length_type, offset = offset) + self._progress_callback( + offset * 100 / info_layer.maximum_address, "Reading TPI layer" + ) + length = module.object(object_type=length_type, offset=offset) if not isinstance(length, int): raise TypeError("Non-integer length provided") offset += length_len output, consumed = self.consume_type(module, offset, length) leaf_type, name, value = output - for tag_type in ['unnamed', 'anonymous']: - if name == f'<{tag_type}-tag>' or name == f'__{tag_type}': - name = f'__{tag_type}_' + hex(len(info_list) + 0x1000)[2:] + for tag_type in ["unnamed", "anonymous"]: + if name == f"<{tag_type}-tag>" or name == f"__{tag_type}": + name = f"__{tag_type}_" + hex(len(info_list) + 0x1000)[2:] if name: info_references[name] = len(info_list) info_list.append((leaf_type, name, value)) @@ -414,47 +303,80 @@ class PdbReader: dbi_layer = self._context.layers.get(self._layer_name + "_stream3", None) if not dbi_layer: raise ValueError("No DBI stream available") - module = self._context.module(module_name = dbi_layer.pdb_symbol_table, layer_name = dbi_layer.name, offset = 0) - self._dbiheader = module.object(object_type = "DBI_HEADER", offset = 0) + module = self._context.module( + module_name=dbi_layer.pdb_symbol_table, layer_name=dbi_layer.name, offset=0 + ) + self._dbiheader = module.object(object_type="DBI_HEADER", offset=0) if not self._dbiheader: raise ValueError("DBI Header could not be read") # Skip past sections we don't care about to get to the DBG header - dbg_hdr_offset = (self._dbiheader.vol.size + self._dbiheader.module_size + self._dbiheader.secconSize + - self._dbiheader.secmapSize + self._dbiheader.filinfSize + self._dbiheader.tsmapSize + - self._dbiheader.ecinfoSize) - self._dbidbgheader = module.object(object_type = "DBI_DBG_HEADER", offset = dbg_hdr_offset) + dbg_hdr_offset = ( + self._dbiheader.vol.size + + self._dbiheader.module_size + + self._dbiheader.secconSize + + self._dbiheader.secmapSize + + self._dbiheader.filinfSize + + self._dbiheader.tsmapSize + + self._dbiheader.ecinfoSize + ) + self._dbidbgheader = module.object( + object_type="DBI_DBG_HEADER", offset=dbg_hdr_offset + ) self._sections = [] self._omap_mapping = [] if self._dbidbgheader.snSectionHdrOrig != -1: - section_orig_layer_name = self._layer_name + "_stream" + str(self._dbidbgheader.snSectionHdrOrig) - consumed, length = 0, self.context.layers[section_orig_layer_name].maximum_address + section_orig_layer_name = ( + self._layer_name + "_stream" + str(self._dbidbgheader.snSectionHdrOrig) + ) + consumed, length = ( + 0, + self.context.layers[section_orig_layer_name].maximum_address, + ) while consumed < length: - section = self.context.object(dbi_layer.pdb_symbol_table + constants.BANG + "IMAGE_SECTION_HEADER", - offset = consumed, - layer_name = section_orig_layer_name) + section = self.context.object( + dbi_layer.pdb_symbol_table + + constants.BANG + + "IMAGE_SECTION_HEADER", + offset=consumed, + layer_name=section_orig_layer_name, + ) self._sections.append(section) consumed += section.vol.size if self._dbidbgheader.snOmapFromSrc != -1: - omap_layer_name = self._layer_name + "_stream" + str(self._dbidbgheader.snOmapFromSrc) + omap_layer_name = ( + self._layer_name + "_stream" + str(self._dbidbgheader.snOmapFromSrc) + ) length = self.context.layers[omap_layer_name].maximum_address data = self.context.layers[omap_layer_name].read(0, length) # For speed we don't use the framework to read this (usually sizeable) data for i in range(0, length, 8): self._omap_mapping.append( - (int.from_bytes(data[i:i + 4], - byteorder = 'little'), int.from_bytes(data[i + 4:i + 8], byteorder = 'little'))) + ( + int.from_bytes(data[i : i + 4], byteorder="little"), + int.from_bytes(data[i + 4 : i + 8], byteorder="little"), + ) + ) elif self._dbidbgheader.snSectionHdr != -1: - section_layer_name = self._layer_name + "_stream" + str(self._dbidbgheader.snSectionHdr) - consumed, length = 0, self.context.layers[section_layer_name].maximum_address + section_layer_name = ( + self._layer_name + "_stream" + str(self._dbidbgheader.snSectionHdr) + ) + consumed, length = ( + 0, + self.context.layers[section_layer_name].maximum_address, + ) while consumed < length: - section = self.context.object(dbi_layer.pdb_symbol_table + constants.BANG + "IMAGE_SECTION_HEADER", - offset = consumed, - layer_name = section_layer_name) + section = self.context.object( + dbi_layer.pdb_symbol_table + + constants.BANG + + "IMAGE_SECTION_HEADER", + offset=consumed, + layer_name=section_layer_name, + ) self._sections.append(section) consumed += section.vol.size @@ -467,31 +389,45 @@ class PdbReader: vollog.debug("Reading Symbols") - symrec_layer = self._context.layers.get(self._layer_name + "_stream" + str(self._dbiheader.symrecStream), None) + symrec_layer = self._context.layers.get( + self._layer_name + "_stream" + str(self._dbiheader.symrecStream), None + ) if not symrec_layer: raise ValueError("No SymRec stream available") - module = self._context.module(module_name = symrec_layer.pdb_symbol_table, - layer_name = symrec_layer.name, - offset = 0) + module = self._context.module( + module_name=symrec_layer.pdb_symbol_table, + layer_name=symrec_layer.name, + offset=0, + ) offset = 0 max_address = symrec_layer.maximum_address while offset < max_address: self._progress_callback(offset * 100 / max_address, "Reading Symbol layer") - sym = module.object(object_type = "GLOBAL_SYMBOL", offset = offset) - leaf_type = module.object(object_type = "unsigned short", offset = sym.leaf_type.vol.offset) + sym = module.object(object_type="GLOBAL_SYMBOL", offset=offset) + leaf_type = module.object( + object_type="unsigned short", offset=sym.leaf_type.vol.offset + ) name = None address = None if sym.segment < len(self._sections): if leaf_type == 0x1009: # v2 symbol (pascal-string) - name = self.parse_string(sym.name, True, sym.length - sym.vol.size + 2) - address = self._sections[sym.segment - 1].VirtualAddress + sym.offset - elif leaf_type == 0x110e or leaf_type == 0x1127: + name = self.parse_string( + sym.name, True, sym.length - sym.vol.size + 2 + ) + address = ( + self._sections[sym.segment - 1].VirtualAddress + sym.offset + ) + elif leaf_type == 0x110E or leaf_type == 0x1127: # v3 symbol (c-string) - name = self.parse_string(sym.name, False, sym.length - sym.vol.size + 2) - address = self._sections[sym.segment - 1].VirtualAddress + sym.offset + name = self.parse_string( + sym.name, False, sym.length - sym.vol.size + 2 + ) + address = ( + self._sections[sym.segment - 1].VirtualAddress + sym.offset + ) else: vollog.debug(f"Only v2 and v3 symbols are supported: {leaf_type:x}") if name: @@ -514,16 +450,18 @@ class PdbReader: pdb_info_layer = self._context.layers.get(self._layer_name + "_stream1", None) if not pdb_info_layer: raise ValueError("No PDB Info Stream available") - module = self._context.module(module_name = pdb_info_layer.pdb_symbol_table, - layer_name = pdb_info_layer.name, - offset = 0) - pdb_info = module.object(object_type = "PDB_INFORMATION", offset = 0) + module = self._context.module( + module_name=pdb_info_layer.pdb_symbol_table, + layer_name=pdb_info_layer.name, + offset=0, + ) + pdb_info = module.object(object_type="PDB_INFORMATION", offset=0) - self.metadata['windows']['pdb'] = { + self.metadata["windows"]["pdb"] = { "GUID": self.convert_bytes_to_guid(pdb_info.GUID), "age": self._dbiheader.age, - "database": self._database_name or 'unknown.pdb', - "machine_type": self._dbiheader.machine + "database": self._database_name or "unknown.pdb", + "machine_type": self._dbiheader.machine, } def convert_bytes_to_guid(self, original: bytes) -> str: @@ -570,7 +508,7 @@ class PdbReader: self.metadata["producer"] = { "datetime": datetime.datetime.now().isoformat(), "name": "volatility3", - "version": constants.PACKAGE_VERSION + "version": constants.PACKAGE_VERSION, } return { @@ -584,45 +522,67 @@ class PdbReader: def get_type_from_index(self, index: int) -> Union[List[Any], Dict[str, Any]]: """Takes a type index and returns appropriate dictionary.""" if index < 0x1000: - base_name, base = primitives[index & 0xff] + base_name, base = primitives[index & 0xFF] self.bases[base_name] = base - result: Union[List[Dict[str, Any]], Dict[str, Any]] = {"kind": "base", "name": base_name} - indirection = (index & 0xf00) + result: Union[List[Dict[str, Any]], Dict[str, Any]] = { + "kind": "base", + "name": base_name, + } + indirection = index & 0xF00 if indirection: pointer_name, pointer_base = indirections[indirection] - if self.bases.get('pointer', None) and self.bases['pointer'] == pointer_base: + if ( + self.bases.get("pointer", None) + and self.bases["pointer"] == pointer_base + ): result = {"kind": "pointer", "subtype": result} else: self.bases[pointer_name] = pointer_base - result = {"kind": "pointer", "base": pointer_name, "subtype": result} + result = { + "kind": "pointer", + "base": pointer_name, + "subtype": result, + } return result else: leaf_type, name, value = self.types[index - 0x1000] result = {"kind": "struct", "name": name} if leaf_type in [leaf_type.LF_MODIFIER]: result = self.get_type_from_index(value.subtype_index) - elif leaf_type in [leaf_type.LF_ARRAY, leaf_type.LF_ARRAY_ST, leaf_type.LF_STRIDED_ARRAY]: + elif leaf_type in [ + leaf_type.LF_ARRAY, + leaf_type.LF_ARRAY_ST, + leaf_type.LF_STRIDED_ARRAY, + ]: result = { "count": ForwardArrayCount(value.size, value.element_type), "kind": "array", - "subtype": self.get_type_from_index(value.element_type) + "subtype": self.get_type_from_index(value.element_type), } elif leaf_type in [leaf_type.LF_BITFIELD]: result = { "kind": "bitfield", "type": self.get_type_from_index(value.underlying_type), "bit_length": value.length, - "bit_position": value.position + "bit_position": value.position, } elif leaf_type in [leaf_type.LF_POINTER]: # Since we use the base['pointer'] to set the size for pointers, update it and check we don't get conflicts size = self.get_size_from_index(index) if self.bases.get("pointer", None) is None: - self.bases['pointer'] = {"endian": "little", "kind": "int", "signed": False, "size": size} + self.bases["pointer"] = { + "endian": "little", + "kind": "int", + "signed": False, + "size": size, + } else: - if size != self.bases['pointer']['size']: + if size != self.bases["pointer"]["size"]: raise ValueError("Native pointers with different sizes!") - result = {"kind": "pointer", "subtype": self.get_type_from_index(value.subtype_index)} + result = { + "kind": "pointer", + "subtype": self.get_type_from_index(value.subtype_index), + } elif leaf_type in [leaf_type.LF_PROCEDURE]: return {"kind": "function"} elif leaf_type in [leaf_type.LF_UNION]: @@ -639,24 +599,38 @@ class PdbReader: """Returns the size of the structure based on the type index provided.""" result = -1 - name: Optional[str] = '' + name: Optional[str] = "" if index < 0x1000: - if (index & 0xf00): - _, base = indirections[index & 0xf00] + if index & 0xF00: + _, base = indirections[index & 0xF00] else: - _, base = primitives[index & 0xff] - result = base['size'] + _, base = primitives[index & 0xFF] + result = base["size"] else: leaf_type, name, value = self.types[index - 0x1000] if leaf_type in [ - leaf_type.LF_UNION, leaf_type.LF_CLASS, leaf_type.LF_CLASS_ST, leaf_type.LF_STRUCTURE, - leaf_type.LF_STRUCTURE_ST, leaf_type.LF_INTERFACE, leaf_type.LF_CLASS_VS19, leaf_type.LF_STRUCTURE_VS19 + leaf_type.LF_UNION, + leaf_type.LF_CLASS, + leaf_type.LF_CLASS_ST, + leaf_type.LF_STRUCTURE, + leaf_type.LF_STRUCTURE_ST, + leaf_type.LF_INTERFACE, + leaf_type.LF_CLASS_VS19, + leaf_type.LF_STRUCTURE_VS19, ]: if not value.properties.forward_reference: result = value.size - elif leaf_type in [leaf_type.LF_ARRAY, leaf_type.LF_ARRAY_ST, leaf_type.LF_STRIDED_ARRAY]: + elif leaf_type in [ + leaf_type.LF_ARRAY, + leaf_type.LF_ARRAY_ST, + leaf_type.LF_STRIDED_ARRAY, + ]: result = value.size - elif leaf_type in [leaf_type.LF_MODIFIER, leaf_type.LF_ENUM, leaf_type.LF_ARGLIST]: + elif leaf_type in [ + leaf_type.LF_MODIFIER, + leaf_type.LF_ENUM, + leaf_type.LF_ARGLIST, + ]: result = self.get_size_from_index(value.subtype_index) elif leaf_type in [leaf_type.LF_MEMBER]: result = self.get_size_from_index(value.field_type) @@ -665,16 +639,18 @@ class PdbReader: elif leaf_type in [leaf_type.LF_POINTER]: result = value.size if not result: - if value.pointer_type == 0x0a: + if value.pointer_type == 0x0A: return 4 - elif value.pointer_type == 0x0c: + elif value.pointer_type == 0x0C: return 8 else: raise ValueError("Pointer size could not be determined") elif leaf_type in [leaf_type.LF_PROCEDURE]: raise ValueError("LF_PROCEDURE size could not be identified") else: - raise ValueError(f"Unable to determine size of leaf_type {leaf_type.lookup()}") + raise ValueError( + f"Unable to determine size of leaf_type {leaf_type.lookup()}" + ) if result <= 0: raise ValueError(f"Invalid size identified: {index} ({name})") return result @@ -694,14 +670,19 @@ class PdbReader: self._progress_callback(index * 100 / max_len, "Processing types") leaf_type, name, value = self.types[index] if leaf_type in [ - leaf_type.LF_CLASS, leaf_type.LF_CLASS_ST, leaf_type.LF_STRUCTURE, leaf_type.LF_STRUCTURE_ST, - leaf_type.LF_INTERFACE, leaf_type.LF_CLASS_VS19, leaf_type.LF_STRUCTURE_VS19 + leaf_type.LF_CLASS, + leaf_type.LF_CLASS_ST, + leaf_type.LF_STRUCTURE, + leaf_type.LF_STRUCTURE_ST, + leaf_type.LF_INTERFACE, + leaf_type.LF_CLASS_VS19, + leaf_type.LF_STRUCTURE_VS19, ]: if not value.properties.forward_reference and name: self.user_types[name] = { "kind": "struct", "size": value.size, - "fields": self.convert_fields(value.fields - 0x1000) + "fields": self.convert_fields(value.fields - 0x1000), } elif leaf_type in [leaf_type.LF_UNION]: if not value.properties.forward_reference and name: @@ -709,7 +690,7 @@ class PdbReader: self.user_types[name] = { "kind": "union", "size": value.size, - "fields": self.convert_fields(value.fields - 0x1000) + "fields": self.convert_fields(value.fields - 0x1000), } elif leaf_type in [leaf_type.LF_ENUM]: if not value.properties.forward_reference and name: @@ -720,89 +701,117 @@ class PdbReader: if not isinstance(constants, list): raise ValueError("Enumeration fields type not a list") self.enumerations[name] = { - 'base': base['name'], - 'size': self.get_size_from_index(value.subtype_index), - 'constants': dict([(name, enum.value) for _, name, enum in constants]) + "base": base["name"], + "size": self.get_size_from_index(value.subtype_index), + "constants": dict( + [(name, enum.value) for _, name, enum in constants] + ), } # Re-run through for ForwardSizeReferences - self.user_types = self.replace_forward_references(self.user_types, type_references) + self.user_types = self.replace_forward_references( + self.user_types, type_references + ) type_handlers = { # Leaf_type: ('Structure', has_name, value_attribute) - 'LF_CLASS': ('LF_STRUCTURE', True, 'size'), - 'LF_CLASS_ST': ('LF_STRUCTURE', True, 'size'), - 'LF_STRUCTURE': ('LF_STRUCTURE', True, 'size'), - 'LF_STRUCTURE_ST': ('LF_STRUCTURE', True, 'size'), - 'LF_INTERFACE': ('LF_STRUCTURE', True, 'size'), - 'LF_CLASS_VS19': ('LF_STRUCTURE_VS19', True, 'size'), - 'LF_STRUCTURE_VS19': ('LF_STRUCTURE_VS19', True, 'size'), - 'LF_MEMBER': ('LF_MEMBER', True, 'offset'), - 'LF_MEMBER_ST': ('LF_MEMBER', True, 'offset'), - 'LF_ARRAY': ('LF_ARRAY', True, 'size'), - 'LF_ARRAY_ST': ('LF_ARRAY', True, 'size'), - 'LF_STRIDED_ARRAY': ('LF_ARRAY', True, 'size'), - 'LF_ENUMERATE': ('LF_ENUMERATE', True, 'value'), - 'LF_ARGLIST': ('LF_ENUM', True, None), - 'LF_ENUM': ('LF_ENUM', True, None), - 'LF_UNION': ('LF_UNION', True, None), - 'LF_STRING_ID': ('LF_STRING_ID', True, None), - 'LF_FUNC_ID': ('LF_FUNC_ID', True, None), - 'LF_MODIFIER': ('LF_MODIFIER', False, None), - 'LF_POINTER': ('LF_POINTER', False, None), - 'LF_PROCEDURE': ('LF_PROCEDURE', False, None), - 'LF_FIELDLIST': ('LF_FIELDLIST', False, None), - 'LF_BITFIELD': ('LF_BITFIELD', False, None), - 'LF_UDT_SRC_LINE': ('LF_UDT_SRC_LINE', False, None), - 'LF_UDT_MOD_SRC_LINE': ('LF_UDT_MOD_SRC_LINE', False, None), - 'LF_BUILDINFO': ('LF_BUILDINFO', False, None) + "LF_CLASS": ("LF_STRUCTURE", True, "size"), + "LF_CLASS_ST": ("LF_STRUCTURE", True, "size"), + "LF_STRUCTURE": ("LF_STRUCTURE", True, "size"), + "LF_STRUCTURE_ST": ("LF_STRUCTURE", True, "size"), + "LF_INTERFACE": ("LF_STRUCTURE", True, "size"), + "LF_CLASS_VS19": ("LF_STRUCTURE_VS19", True, "size"), + "LF_STRUCTURE_VS19": ("LF_STRUCTURE_VS19", True, "size"), + "LF_MEMBER": ("LF_MEMBER", True, "offset"), + "LF_MEMBER_ST": ("LF_MEMBER", True, "offset"), + "LF_ARRAY": ("LF_ARRAY", True, "size"), + "LF_ARRAY_ST": ("LF_ARRAY", True, "size"), + "LF_STRIDED_ARRAY": ("LF_ARRAY", True, "size"), + "LF_ENUMERATE": ("LF_ENUMERATE", True, "value"), + "LF_ARGLIST": ("LF_ENUM", True, None), + "LF_ENUM": ("LF_ENUM", True, None), + "LF_UNION": ("LF_UNION", True, None), + "LF_STRING_ID": ("LF_STRING_ID", True, None), + "LF_FUNC_ID": ("LF_FUNC_ID", True, None), + "LF_MODIFIER": ("LF_MODIFIER", False, None), + "LF_POINTER": ("LF_POINTER", False, None), + "LF_PROCEDURE": ("LF_PROCEDURE", False, None), + "LF_FIELDLIST": ("LF_FIELDLIST", False, None), + "LF_BITFIELD": ("LF_BITFIELD", False, None), + "LF_UDT_SRC_LINE": ("LF_UDT_SRC_LINE", False, None), + "LF_UDT_MOD_SRC_LINE": ("LF_UDT_MOD_SRC_LINE", False, None), + "LF_BUILDINFO": ("LF_BUILDINFO", False, None), } def consume_type( - self, module: interfaces.context.ModuleInterface, offset: int, length: int - ) -> Tuple[Tuple[Optional[interfaces.objects.ObjectInterface], Optional[str], Union[ - None, List, interfaces.objects.ObjectInterface]], int]: + self, module: interfaces.context.ModuleInterface, offset: int, length: int + ) -> Tuple[ + Tuple[ + Optional[interfaces.objects.ObjectInterface], + Optional[str], + Union[None, List, interfaces.objects.ObjectInterface], + ], + int, + ]: """Returns a (leaf_type, name, object) Tuple for a type, and the number of bytes consumed.""" - leaf_type = self.context.object(module.get_enumeration("LEAF_TYPE"), - layer_name = module._layer_name, - offset = offset) + leaf_type = self.context.object( + module.get_enumeration("LEAF_TYPE"), + layer_name=module._layer_name, + offset=offset, + ) consumed = leaf_type.vol.base_type.size remaining = length - consumed - type_handler, has_name, value_attribute = self.type_handlers.get(leaf_type.lookup(), - ('LF_UNKNOWN', False, None)) + type_handler, has_name, value_attribute = self.type_handlers.get( + leaf_type.lookup(), ("LF_UNKNOWN", False, None) + ) - if type_handler in ['LF_FIELDLIST']: + if type_handler in ["LF_FIELDLIST"]: sub_length = remaining sub_offset = offset + consumed fields = [] while length > consumed: - subfield, sub_consumed = self.consume_type(module, sub_offset, sub_length) - sub_consumed += self.consume_padding(module.layer_name, sub_offset + sub_consumed) + subfield, sub_consumed = self.consume_type( + module, sub_offset, sub_length + ) + sub_consumed += self.consume_padding( + module.layer_name, sub_offset + sub_consumed + ) sub_length -= sub_consumed sub_offset += sub_consumed consumed += sub_consumed fields.append(subfield) result = leaf_type, None, fields - elif type_handler in ['LF_BUILDINFO']: - parsed_obj = module.object(object_type = type_handler, offset = offset + consumed) + elif type_handler in ["LF_BUILDINFO"]: + parsed_obj = module.object( + object_type=type_handler, offset=offset + consumed + ) parsed_obj.arguments.count = parsed_obj.count consumed += parsed_obj.arguments.vol.size result = leaf_type, None, parsed_obj elif type_handler in self.type_handlers: - parsed_obj = module.object(object_type = type_handler, offset = offset + consumed) + parsed_obj = module.object( + object_type=type_handler, offset=offset + consumed + ) current_consumed = remaining if has_name: name_offset = parsed_obj.name.vol.offset - parsed_obj.vol.offset if value_attribute: - name, value, excess = self.determine_extended_value(leaf_type, getattr(parsed_obj, value_attribute), - module, remaining - name_offset) + name, value, excess = self.determine_extended_value( + leaf_type, + getattr(parsed_obj, value_attribute), + module, + remaining - name_offset, + ) setattr(parsed_obj, value_attribute, value) current_consumed = parsed_obj.vol.size + len(name) + 1 + excess else: - name = self.parse_string(parsed_obj.name, leaf_type < leaf_type.LF_ST_MAX, - size = remaining - name_offset) + name = self.parse_string( + parsed_obj.name, + leaf_type < leaf_type.LF_ST_MAX, + size=remaining - name_offset, + ) parsed_obj.name = name else: name = None @@ -816,9 +825,9 @@ class PdbReader: def consume_padding(self, layer_name: str, offset: int) -> int: """Returns the amount of padding used between fields.""" val = self.context.layers[layer_name].read(offset, 1) - if not ((val[0] & 0xf0) == 0xf0): + if not ((val[0] & 0xF0) == 0xF0): return 0 - return (int(val[0]) & 0x0f) + return int(val[0]) & 0x0F def convert_fields(self, fields: int) -> Dict[Optional[str], Dict[str, Any]]: """Converts a field list into a list of fields.""" @@ -829,7 +838,10 @@ class PdbReader: return result for field in fields_struct: _, name, member = field - result[name] = {"offset": member.offset, "type": self.get_type_from_index(member.field_type)} + result[name] = { + "offset": member.offset, + "type": self.get_type_from_index(member.field_type), + } return result def replace_forward_references(self, types, type_references): @@ -852,9 +864,13 @@ class PdbReader: if element_type > 0x1000: _, name, toplevel_type = self.types[element_type - 0x1000] # If there's no name, the original size is probably fine as long as we're not indirect (LF_MODIFIER) - if not name and isinstance( - toplevel_type, - interfaces.objects.ObjectInterface) and toplevel_type.vol.type_name.endswith('LF_MODIFIER'): + if ( + not name + and isinstance( + toplevel_type, interfaces.objects.ObjectInterface + ) + and toplevel_type.vol.type_name.endswith("LF_MODIFIER") + ): # We have check they don't point to a forward reference, so we go round again with the subtype element_type = toplevel_type.subtype_index loop = True @@ -867,66 +883,83 @@ class PdbReader: # COMMON CODE @staticmethod - def parse_string(structure: interfaces.objects.ObjectInterface, - parse_as_pascal: bool = False, - size: int = 0) -> str: + def parse_string( + structure: interfaces.objects.ObjectInterface, + parse_as_pascal: bool = False, + size: int = 0, + ) -> str: """Consumes either a c-string or a pascal string depending on the leaf_type.""" if not parse_as_pascal: - name = structure.cast("string", max_length = size, encoding = "latin-1") + name = structure.cast("string", max_length=size, encoding="latin-1") else: name = structure.cast("pascal_string") - name = name.string.cast("string", max_length = name.length, encoding = "latin-1") + name = name.string.cast( + "string", max_length=name.length, encoding="latin-1" + ) return str(name) - def determine_extended_value(self, leaf_type: interfaces.objects.ObjectInterface, - value: interfaces.objects.ObjectInterface, module: interfaces.context.ModuleInterface, - length: int) -> Tuple[str, interfaces.objects.ObjectInterface, int]: + def determine_extended_value( + self, + leaf_type: interfaces.objects.ObjectInterface, + value: interfaces.objects.ObjectInterface, + module: interfaces.context.ModuleInterface, + length: int, + ) -> Tuple[str, interfaces.objects.ObjectInterface, int]: """Reads a value and potentially consumes more data to construct the value.""" excess = 0 if value >= leaf_type.LF_CHAR: - sub_leaf_type = self.context.object(self.context.symbol_space.get_enumeration(leaf_type.vol.type_name), - layer_name = leaf_type.vol.layer_name, - offset = value.vol.offset) + sub_leaf_type = self.context.object( + self.context.symbol_space.get_enumeration(leaf_type.vol.type_name), + layer_name=leaf_type.vol.layer_name, + offset=value.vol.offset, + ) # Set the offset at just after the previous size type offset = value.vol.offset + value.vol.data_format.length if sub_leaf_type in [leaf_type.LF_CHAR]: - value = module.object(object_type = 'char', offset = offset) + value = module.object(object_type="char", offset=offset) elif sub_leaf_type in [leaf_type.LF_SHORT]: - value = module.object(object_type = 'short', offset = offset) + value = module.object(object_type="short", offset=offset) elif sub_leaf_type in [leaf_type.LF_USHORT]: - value = module.object(object_type = 'unsigned short', offset = offset) + value = module.object(object_type="unsigned short", offset=offset) elif sub_leaf_type in [leaf_type.LF_LONG]: - value = module.object(object_type = 'long', offset = offset) + value = module.object(object_type="long", offset=offset) elif sub_leaf_type in [leaf_type.LF_ULONG]: - value = module.object(object_type = 'unsigned long', offset = offset) + value = module.object(object_type="unsigned long", offset=offset) else: raise TypeError("Unexpected extended value type") excess = value.vol.data_format.length # Updated the consume/offset counters - name = module.object(object_type = "string", offset = value.vol.offset + value.vol.data_format.length) - name_str = self.parse_string(name, leaf_type < leaf_type.LF_ST_MAX, size = length - excess) + name = module.object( + object_type="string", offset=value.vol.offset + value.vol.data_format.length + ) + name_str = self.parse_string( + name, leaf_type < leaf_type.LF_ST_MAX, size=length - excess + ) return name_str, value, excess class PdbRetreiver: - - def retreive_pdb(self, - guid: str, - file_name: str, - progress_callback: constants.ProgressCallback = None) -> Optional[str]: + def retreive_pdb( + self, + guid: str, + file_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[str]: vollog.info("Download PDB file...") - file_name = ".".join(file_name.split(".")[:-1] + ['pdb']) - for sym_url in ['http://msdl.microsoft.com/download/symbols']: + file_name = ".".join(file_name.split(".")[:-1] + ["pdb"]) + for sym_url in ["http://msdl.microsoft.com/download/symbols"]: url = sym_url + f"/{file_name}/{guid}/" result = None - for suffix in [file_name, file_name[:-1] + '_']: + for suffix in [file_name, file_name[:-1] + "_"]: try: vollog.debug(f"Attempting to retrieve {url + suffix}") # We have to cache this because the file is opened by a layer and we can't control whether that caches - with resources.ResourceAccessor(progress_callback).open(url + suffix) as fp: + with resources.ResourceAccessor(progress_callback).open( + url + suffix + ) as fp: fp.read(10) result = True except (error.HTTPError, error.URLError) as excp: @@ -940,10 +973,9 @@ class PdbRetreiver: return url + suffix -if __name__ == '__main__': +if __name__ == "__main__": import argparse - class PrintedProgress(object): """A progress handler that prints the progress value and the description onto the command line.""" @@ -962,26 +994,47 @@ if __name__ == '__main__': message = f"\rProgress: {round(progress, 2): 7.2f}\t\t{description or ''}" message_len = len(message) self._max_message_len = max([self._max_message_len, message_len]) - print(message, end = (' ' * (self._max_message_len - message_len)) + '\r') - + print(message, end=(" " * (self._max_message_len - message_len)) + "\r") parser = argparse.ArgumentParser( - description = "Read PDB files and convert to Volatility 3 Intermediate Symbol Format") - parser.add_argument("-o", "--output", metavar = "OUTPUT", help = "Filename for data output", default = None) - file_group = parser.add_argument_group("file", description = "File-based conversion of PDB to ISF") - file_group.add_argument("-f", "--file", metavar = "FILE", help = "PDB file to translate to ISF") - data_group = parser.add_argument_group("data", description = "Convert based on a GUID and filename pattern") - data_group.add_argument("-p", "--pattern", metavar = "PATTERN", help = "Filename pattern to recover PDB file") - data_group.add_argument("-g", - "--guid", - metavar = "GUID", - help = "GUID + Age string for the required PDB file", - default = None) - data_group.add_argument("-k", - "--keep", - action = "store_true", - default = False, - help = "Keep the downloaded PDB file") + description="Read PDB files and convert to Volatility 3 Intermediate Symbol Format" + ) + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT", + help="Filename for data output", + default=None, + ) + file_group = parser.add_argument_group( + "file", description="File-based conversion of PDB to ISF" + ) + file_group.add_argument( + "-f", "--file", metavar="FILE", help="PDB file to translate to ISF" + ) + data_group = parser.add_argument_group( + "data", description="Convert based on a GUID and filename pattern" + ) + data_group.add_argument( + "-p", + "--pattern", + metavar="PATTERN", + help="Filename pattern to recover PDB file", + ) + data_group.add_argument( + "-g", + "--guid", + metavar="GUID", + help="GUID + Age string for the required PDB file", + default=None, + ) + data_group.add_argument( + "-k", + "--keep", + action="store_true", + default=False, + help="Keep the downloaded PDB file", + ) args = parser.parse_args() pg_cb = PrintedProgress() @@ -989,10 +1042,12 @@ if __name__ == '__main__': delfile = False filename = None if args.guid is not None and args.pattern is not None: - filename = PdbRetreiver().retreive_pdb(guid = args.guid, file_name = args.pattern, progress_callback = pg_cb) + filename = PdbRetreiver().retreive_pdb( + guid=args.guid, file_name=args.pattern, progress_callback=pg_cb + ) if filename is None: parser.error("PDB file could not be retrieved from the internet") - if parse.urlparse(filename, 'file').scheme == 'file': + if parse.urlparse(filename, "file").scheme == "file": delfile = True elif args.file: filename = args.file @@ -1003,15 +1058,17 @@ if __name__ == '__main__': parser.error("No suitable filename provided or retrieved") ctx = contexts.Context() - url = parse.urlparse(filename, scheme = 'file') - if url.scheme == 'file': + url = parse.urlparse(filename, scheme="file") + if url.scheme == "file": if not os.path.exists(filename): parser.error(f"File {filename} does not exists") location = "file:" + request.pathname2url(os.path.abspath(filename)) else: location = filename - convertor = PdbReader(ctx, location, database_name = args.pattern, progress_callback = pg_cb) + convertor = PdbReader( + ctx, location, database_name=args.pattern, progress_callback=pg_cb + ) converted_json = convertor.get_json() if args.output is None: @@ -1019,23 +1076,23 @@ if __name__ == '__main__': guid = args.guid[:-1] age = args.guid[-1:] else: - guid = converted_json['metadata']['windows']['pdb']['GUID'] - age = converted_json['metadata']['windows']['pdb']['age'] + guid = converted_json["metadata"]["windows"]["pdb"]["GUID"] + age = converted_json["metadata"]["windows"]["pdb"]["age"] args.output = f"{guid}-{age}.json.xz" output_url = os.path.abspath(args.output) open_method = open - if args.output.endswith('.gz'): + if args.output.endswith(".gz"): open_method = gzip.open - elif args.output.endswith('.bz2'): + elif args.output.endswith(".bz2"): open_method = bz2.open - elif args.output.endswith('.xz'): + elif args.output.endswith(".xz"): open_method = lzma.open with open_method(output_url, "wb") as f: - json_string = json.dumps(converted_json, indent = 2, sort_keys = True) - f.write(bytes(json_string, 'latin-1')) + json_string = json.dumps(converted_json, indent=2, sort_keys=True) + f.write(bytes(json_string, "latin-1")) if args.keep: print(f"Temporary PDB file: {filename}") diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 25911e376..1c3260fed 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -31,13 +31,14 @@ class PDBUtility(interfaces.configuration.VersionableInterface): @classmethod def symbol_table_from_offset( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - offset: int, - symbol_table_class: str = "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path: str = None, - progress_callback: constants.ProgressCallback = None) -> Optional[str]: + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + offset: int, + symbol_table_class: str = "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path: str = None, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[str]: """Produces the name of a symbol table loaded from the offset for an MZ header Args: @@ -56,49 +57,76 @@ class PDBUtility(interfaces.configuration.VersionableInterface): return None guid, age, pdb_name = result if config_path is None: - config_path = interfaces.configuration.path_join('pdbutility', pdb_name.replace('.', '_')) + config_path = interfaces.configuration.path_join( + "pdbutility", pdb_name.replace(".", "_") + ) - return cls.load_windows_symbol_table(context, guid, age, pdb_name, symbol_table_class, config_path, - progress_callback) + return cls.load_windows_symbol_table( + context, + guid, + age, + pdb_name, + symbol_table_class, + config_path, + progress_callback, + ) @classmethod - def load_windows_symbol_table(cls, - context: interfaces.context.ContextInterface, - guid: str, - age: int, - pdb_name: str, - symbol_table_class: str, - config_path: str = 'pdbutility', - progress_callback: constants.ProgressCallback = None): + def load_windows_symbol_table( + cls, + context: interfaces.context.ContextInterface, + guid: str, + age: int, + pdb_name: str, + symbol_table_class: str, + config_path: str = "pdbutility", + progress_callback: constants.ProgressCallback = None, + ): """Loads (downloading if necessary) a windows symbol table""" - filter_string = os.path.join(pdb_name.strip('\x00'), guid.upper() + "-" + str(age)) + filter_string = os.path.join( + pdb_name.strip("\x00"), guid.upper() + "-" + str(age) + ) isf_path = None # Take the first result of search for the intermediate file - if not requirements.VersionRequirement.matches_required((1, 0, 0), symbol_cache.SqliteCache.version): + if not requirements.VersionRequirement.matches_required( + (1, 0, 0), symbol_cache.SqliteCache.version + ): vollog.debug(f"Required version of SQLiteCache not found") return None - identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + identifiers_path = os.path.join( + constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + ) value = symbol_cache.SqliteCache(identifiers_path).find_location( - symbol_cache.WindowsIdentifier.generate(pdb_name.strip('\x00'), guid.upper(), age), 'windows') + symbol_cache.WindowsIdentifier.generate( + pdb_name.strip("\x00"), guid.upper(), age + ), + "windows", + ) if value: isf_path = value else: # If none are found, attempt to download the pdb, convert it and try again - cls.download_pdb_isf(context, guid.upper(), age, pdb_name, progress_callback) + cls.download_pdb_isf( + context, guid.upper(), age, pdb_name, progress_callback + ) # Try again - for value in intermed.IntermediateSymbolTable.file_symbol_url("windows", filter_string): + for value in intermed.IntermediateSymbolTable.file_symbol_url( + "windows", filter_string + ): isf_path = value break if not isf_path: vollog.debug(f"Required symbol library path not found: {filter_string}") - vollog.info("The symbols can be downloaded later using pdbconv.py -p {} -g {}".format( - pdb_name.strip('\x00'), - guid.upper() + str(age))) + vollog.info( + "The symbols can be downloaded later using pdbconv.py -p {} -g {}".format( + pdb_name.strip("\x00"), guid.upper() + str(age) + ) + ) return None vollog.debug(f"Using symbol library: {filter_string}") @@ -111,13 +139,16 @@ class PDBUtility(interfaces.configuration.VersionableInterface): requirement_name = interfaces.configuration.path_head(config_path) # Construct the appropriate symbol table - requirement = SymbolTableRequirement(name = requirement_name, description = "PDBUtility generated symbol table") + requirement = SymbolTableRequirement( + name=requirement_name, description="PDBUtility generated symbol table" + ) requirement.construct(context, parent_config_path) return context.config[config_path] @classmethod - def get_guid_from_mz(cls, context: interfaces.context.ContextInterface, layer_name: str, - offset: int) -> Optional[Tuple[str, int, str]]: + def get_guid_from_mz( + cls, context: interfaces.context.ContextInterface, layer_name: str, offset: int + ) -> Optional[Tuple[str, int, str]]: """Takes the offset to an MZ header, locates any available pdb headers, and extracts the guid, age and pdb_name from them Args: @@ -131,7 +162,9 @@ class PDBUtility(interfaces.configuration.VersionableInterface): try: import pefile except ImportError: - vollog.error("Get_guid_from_mz requires the following python module: pefile") + vollog.error( + "Get_guid_from_mz requires the following python module: pefile" + ) return None layer = context.layers[layer_name] @@ -141,33 +174,39 @@ class PDBUtility(interfaces.configuration.VersionableInterface): if mz_sig != b"MZ": return None - nt_header_start, = struct.unpack(" None: + def download_pdb_isf( + cls, + context: interfaces.context.ContextInterface, + guid: str, + age: int, + pdb_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> None: """Attempts to download the PDB file, convert it to an ISF file and save it to one of the symbol locations.""" # Check for writability @@ -205,32 +251,51 @@ class PDBUtility(interfaces.configuration.VersionableInterface): # Store any temporary files created by downloading PDB files tmp_files = [] - potential_output_filename = os.path.join(path, "windows", filter_string + ".json.xz") + potential_output_filename = os.path.join( + path, "windows", filter_string + ".json.xz" + ) data_written = False try: - os.makedirs(os.path.dirname(potential_output_filename), exist_ok = True) + os.makedirs(os.path.dirname(potential_output_filename), exist_ok=True) with lzma.open(potential_output_filename, "w") as of: # Once we haven't thrown an error, do the computation - filename = pdbconv.PdbRetreiver().retreive_pdb(guid + str(age), - file_name = pdb_name, - progress_callback = progress_callback) + filename = pdbconv.PdbRetreiver().retreive_pdb( + guid + str(age), + file_name=pdb_name, + progress_callback=progress_callback, + ) if filename: - url = parse.urlparse(filename, scheme = 'file') - if url.scheme == 'file' or len(url.scheme) == 1: + url = parse.urlparse(filename, scheme="file") + if url.scheme == "file" or len(url.scheme) == 1: tmp_files.append(filename) - location = "file:" + request.pathname2url(os.path.abspath(tmp_files[-1])) + location = "file:" + request.pathname2url( + os.path.abspath(tmp_files[-1]) + ) else: location = filename - json_output = pdbconv.PdbReader(context, location, pdb_name, progress_callback).get_json() - of.write(bytes(json.dumps(json_output, indent = 2, sort_keys = True), 'utf-8')) + json_output = pdbconv.PdbReader( + context, location, pdb_name, progress_callback + ).get_json() + of.write( + bytes( + json.dumps(json_output, indent=2, sort_keys=True), + "utf-8", + ) + ) # After we've successfully written it out, record the fact so we don't clear it out data_written = True else: - vollog.warning("Symbol file could not be downloaded from remote server" + (" " * 100)) + vollog.warning( + "Symbol file could not be downloaded from remote server" + + (" " * 100) + ) break except PermissionError: - vollog.warning("Cannot write necessary symbol file, please check permissions on {}".format( - potential_output_filename)) + vollog.warning( + "Cannot write necessary symbol file, please check permissions on {}".format( + potential_output_filename + ) + ) continue finally: # If something else failed, removed the symbol file so we don't pick it up in the future @@ -241,21 +306,27 @@ class PDBUtility(interfaces.configuration.VersionableInterface): try: os.remove(filename) except PermissionError: - vollog.warning(f"Temporary file could not be removed: {filename}") + vollog.warning( + f"Temporary file could not be removed: {filename}" + ) else: - vollog.warning("Cannot write downloaded symbols, please add the appropriate symbols" - " or add/modify a symbols directory that is writable") + vollog.warning( + "Cannot write downloaded symbols, please add the appropriate symbols" + " or add/modify a symbols directory that is writable" + ) @classmethod - def pdbname_scan(cls, - ctx: interfaces.context.ContextInterface, - layer_name: str, - page_size: int, - pdb_names: List[bytes], - progress_callback: constants.ProgressCallback = None, - start: Optional[int] = None, - end: Optional[int] = None, - maximum_invalid_count: int = 100) -> Generator[Dict[str, Optional[Union[bytes, str, int]]], None, None]: + def pdbname_scan( + cls, + ctx: interfaces.context.ContextInterface, + layer_name: str, + page_size: int, + pdb_names: List[bytes], + progress_callback: constants.ProgressCallback = None, + start: Optional[int] = None, + end: Optional[int] = None, + maximum_invalid_count: int = 100, + ) -> Generator[Dict[str, Optional[Union[bytes, str, int]]], None, None]: """Scans through `layer_name` at `ctx` looking for RSDS headers that indicate one of four common pdb kernel names (as listed in `self.pdb_names`) and returns the tuple (GUID, age, pdb_name, @@ -281,11 +352,12 @@ class PDBUtility(interfaces.configuration.VersionableInterface): if end is None: end = ctx.layers[layer_name].maximum_address - for (GUID, age, pdb_name, - signature_offset) in ctx.layers[layer_name].scan(ctx, - PdbSignatureScanner(pdb_names), - progress_callback = progress_callback, - sections = [(start, end - start)]): + for (GUID, age, pdb_name, signature_offset) in ctx.layers[layer_name].scan( + ctx, + PdbSignatureScanner(pdb_names), + progress_callback=progress_callback, + sections=[(start, end - start)], + ): mz_offset = None sig_pfn = signature_offset // page_size current_invalid_counter = 0 @@ -299,22 +371,29 @@ class PDBUtility(interfaces.configuration.VersionableInterface): continue data = ctx.layers[layer_name].read(i * page_size, 2) - if data == b'MZ': + if data == b"MZ": mz_offset = i * page_size break min_pfn = sig_pfn yield { - 'GUID': GUID, - 'age': age, - 'pdb_name': str(pdb_name, "utf-8"), - 'signature_offset': signature_offset, - 'mz_offset': mz_offset + "GUID": GUID, + "age": age, + "pdb_name": str(pdb_name, "utf-8"), + "signature_offset": signature_offset, + "mz_offset": mz_offset, } @classmethod - def symbol_table_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, - pdb_name: str, module_offset: int = None, module_size: int = None) -> str: + def symbol_table_from_pdb( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + pdb_name: str, + module_offset: int = None, + module_size: int = None, + ) -> str: """Creates symbol table for a module in the specified layer_name. Searches the memory section of the loaded module for its PDB GUID @@ -330,14 +409,22 @@ class PDBUtility(interfaces.configuration.VersionableInterface): Returns: The name of the constructed and loaded symbol table """ - _, symbol_table_name = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, - module_size) + _, symbol_table_name = cls._modtable_from_pdb( + context, config_path, layer_name, pdb_name, module_offset, module_size + ) return symbol_table_name @classmethod - def _modtable_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, - pdb_name: str, module_offset: int = None, module_size: int = None, - create_module: bool = False) -> Tuple[Optional[str], Optional[str]]: + def _modtable_from_pdb( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + pdb_name: str, + module_offset: int = None, + module_size: int = None, + create_module: bool = False, + ) -> Tuple[Optional[str], Optional[str]]: if module_offset is None: module_offset = context.layers[layer_name].minimum_address @@ -345,40 +432,59 @@ class PDBUtility(interfaces.configuration.VersionableInterface): module_size = context.layers[layer_name].maximum_address - module_offset guids = list( - cls.pdbname_scan(context, - layer_name, - context.layers[layer_name].page_size, [bytes(pdb_name, 'latin-1')], - start = module_offset, - end = module_offset + module_size)) + cls.pdbname_scan( + context, + layer_name, + context.layers[layer_name].page_size, + [bytes(pdb_name, "latin-1")], + start=module_offset, + end=module_offset + module_size, + ) + ) if not guids: raise exceptions.VolatilityException( - f"Did not find GUID of {pdb_name} in module @ 0x{module_offset:x}!") + f"Did not find GUID of {pdb_name} in module @ 0x{module_offset:x}!" + ) guid = guids[0] vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - module_name = guid["pdb_name"].replace('.pdb', '') + module_name = guid["pdb_name"].replace(".pdb", "") - symbol_table_name = cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) + symbol_table_name = cls.load_windows_symbol_table( + context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path=config_path, + ) new_module_name = None if create_module: - new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], - symbol_table_name = symbol_table_name) + new_module = contexts.Module.create( + context, + module_name, + layer_name, + offset=guid["mz_offset"], + symbol_table_name=symbol_table_name, + ) new_module_name = new_module.name return new_module_name, symbol_table_name @classmethod - def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, - pdb_name: str, module_offset: int = None, module_size: int = None) -> str: + def module_from_pdb( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + pdb_name: str, + module_offset: int = None, + module_size: int = None, + ) -> str: """Creates a module in the specified layer_name based on a pdb name. Searches the memory section of the loaded module for its PDB GUID @@ -395,8 +501,15 @@ class PDBUtility(interfaces.configuration.VersionableInterface): The name of the constructed and loaded symbol table """ - module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, - module_size, create_module = True) + module_name, _ = cls._modtable_from_pdb( + context, + config_path, + layer_name, + pdb_name, + module_offset, + module_size, + create_module=True, + ) return module_name @@ -410,6 +523,7 @@ class PdbSignatureScanner(interfaces.layers.ScannerInterface): .. note:: The pdb_names must be a list of byte strings, unicode strs will not match against the data scanned """ + overlap = 0x4000 """The size of overlap needed for the signature to ensure data cannot hide between two scanned chunks""" thread_safe = True @@ -421,16 +535,52 @@ class PdbSignatureScanner(interfaces.layers.ScannerInterface): super().__init__() self._pdb_names = pdb_names - def __call__(self, data: bytes, data_offset: int) -> Generator[Tuple[str, Any, bytes, int], None, None]: - pattern = b'RSDS' + (b'.' * self._RSDS_format.size) + b'(' + b'|'.join( - [re.escape(x) for x in self._pdb_names]) + b')\x00' - for match in re.finditer(pattern, data, flags = re.DOTALL): - pdb_name = data[match.start(0) + 4 + self._RSDS_format.size:match.start(0) + len(match.group()) - 1] + def __call__( + self, data: bytes, data_offset: int + ) -> Generator[Tuple[str, Any, bytes, int], None, None]: + pattern = ( + b"RSDS" + + (b"." * self._RSDS_format.size) + + b"(" + + b"|".join([re.escape(x) for x in self._pdb_names]) + + b")\x00" + ) + for match in re.finditer(pattern, data, flags=re.DOTALL): + pdb_name = data[ + match.start(0) + + 4 + + self._RSDS_format.size : match.start(0) + + len(match.group()) + - 1 + ] if pdb_name in self._pdb_names: ## this ordering is intentional due to mixed endianness in the GUID - (g3, g2, g1, g0, g5, g4, g7, g6, g8, g9, ga, gb, gc, gd, ge, gf, a) = \ - self._RSDS_format.unpack(data[match.start(0) + 4:match.start(0) + 4 + self._RSDS_format.size]) + ( + g3, + g2, + g1, + g0, + g5, + g4, + g7, + g6, + g8, + g9, + ga, + gb, + gc, + gd, + ge, + gf, + a, + ) = self._RSDS_format.unpack( + data[ + match.start(0) + 4 : match.start(0) + 4 + self._RSDS_format.size + ] + ) - guid = (16 * '{:02X}').format(g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, ga, gb, gc, gd, ge, gf) + guid = (16 * "{:02X}").format( + g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, ga, gb, gc, gd, ge, gf + ) if match.start(0) < self.chunk_size: yield (guid, a, pdb_name, data_offset + match.start(0)) diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index d38cdb701..84ce65432 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -31,13 +31,18 @@ class OsDistinguisher: A function that takes a context and a symbol table name and determines whether that symbol table passes the distinguishing checks """ - def __init__(self, version_check: Callable[[Tuple[int, ...]], bool], fallback_checks: List[Tuple[str, Optional[str], - bool]]) -> None: + def __init__( + self, + version_check: Callable[[Tuple[int, ...]], bool], + fallback_checks: List[Tuple[str, Optional[str], bool]], + ) -> None: self._version_check = version_check self._fallback_checks = fallback_checks # try the primary method based on the pe version in the ISF - def __call__(self, context: interfaces.context.ContextInterface, symbol_table: str) -> bool: + def __call__( + self, context: interfaces.context.ContextInterface, symbol_table: str + ) -> bool: """ Args: @@ -53,17 +58,27 @@ class OsDistinguisher: major, minor, revision, build = pe_version return self._version_check((major, minor, revision, build)) except (AttributeError, ValueError, TypeError): - vollog.log(constants.LOGLEVEL_VVV, "Windows PE version data is not available") + vollog.log( + constants.LOGLEVEL_VVV, "Windows PE version data is not available" + ) # fall back to the backup method, if necessary for name, member, response in self._fallback_checks: if member is None: - if (context.symbol_space.has_symbol(symbol_table + constants.BANG + name) - or context.symbol_space.has_type(symbol_table + constants.BANG + name)) != response: + if ( + context.symbol_space.has_symbol( + symbol_table + constants.BANG + name + ) + or context.symbol_space.has_type( + symbol_table + constants.BANG + name + ) + ) != response: return False else: try: - symbol_type = context.symbol_space.get_type(symbol_table + constants.BANG + name) + symbol_type = context.symbol_space.get_type( + symbol_table + constants.BANG + name + ) if symbol_type.has_member(member) != response: return False except exceptions.SymbolError: @@ -73,49 +88,88 @@ class OsDistinguisher: return True -is_windows_8_1_or_later = OsDistinguisher(version_check = lambda x: x >= (6, 3), - fallback_checks = [("_KPRCB", "PendingTickFlags", True)]) +is_windows_8_1_or_later = OsDistinguisher( + version_check=lambda x: x >= (6, 3), + fallback_checks=[("_KPRCB", "PendingTickFlags", True)], +) -is_vista_or_later = OsDistinguisher(version_check = lambda x: x >= (6, 0), - fallback_checks = [("KdCopyDataBlock", None, True)]) +is_vista_or_later = OsDistinguisher( + version_check=lambda x: x >= (6, 0), + fallback_checks=[("KdCopyDataBlock", None, True)], +) -is_win10 = OsDistinguisher(version_check = lambda x: (10, 0) <= x, - fallback_checks = [("ObHeaderCookie", None, True), ("_HANDLE_TABLE", "HandleCount", False)]) +is_win10 = OsDistinguisher( + version_check=lambda x: (10, 0) <= x, + fallback_checks=[ + ("ObHeaderCookie", None, True), + ("_HANDLE_TABLE", "HandleCount", False), + ], +) -is_windows_xp = OsDistinguisher(version_check = lambda x: (5, 1) <= x < (5, 2), - fallback_checks = [("KdCopyDataBlock", None, False), - ("_HANDLE_TABLE", "HandleCount", True)]) +is_windows_xp = OsDistinguisher( + version_check=lambda x: (5, 1) <= x < (5, 2), + fallback_checks=[ + ("KdCopyDataBlock", None, False), + ("_HANDLE_TABLE", "HandleCount", True), + ], +) -is_xp_or_2003 = OsDistinguisher(version_check = lambda x: (5, 1) <= x < (6, 0), - fallback_checks = [("KdCopyDataBlock", None, False), - ("_HANDLE_TABLE", "HandleCount", True)]) +is_xp_or_2003 = OsDistinguisher( + version_check=lambda x: (5, 1) <= x < (6, 0), + fallback_checks=[ + ("KdCopyDataBlock", None, False), + ("_HANDLE_TABLE", "HandleCount", True), + ], +) -is_win10_up_to_15063 = OsDistinguisher(version_check = lambda x: (10, 0) <= x < (10, 0, 15063), - fallback_checks = [("ObHeaderCookie", None, True), - ("_HANDLE_TABLE", "HandleCount", False), - ("_EPROCESS", "KeepAliveCounter", True)]) +is_win10_up_to_15063 = OsDistinguisher( + version_check=lambda x: (10, 0) <= x < (10, 0, 15063), + fallback_checks=[ + ("ObHeaderCookie", None, True), + ("_HANDLE_TABLE", "HandleCount", False), + ("_EPROCESS", "KeepAliveCounter", True), + ], +) -is_win10_15063 = OsDistinguisher(version_check = lambda x: x == (10, 0, 15063), - fallback_checks = [("ObHeaderCookie", None, True), - ("_HANDLE_TABLE", "HandleCount", False), - ("_EPROCESS", "KeepAliveCounter", False), - ("_EPROCESS", "ControlFlowGuardEnabled", True)]) +is_win10_15063 = OsDistinguisher( + version_check=lambda x: x == (10, 0, 15063), + fallback_checks=[ + ("ObHeaderCookie", None, True), + ("_HANDLE_TABLE", "HandleCount", False), + ("_EPROCESS", "KeepAliveCounter", False), + ("_EPROCESS", "ControlFlowGuardEnabled", True), + ], +) -is_win10_16299_or_later = OsDistinguisher(version_check = lambda x: x >= (10, 0, 16299), - fallback_checks = [("ObHeaderCookie", None, True), - ("_HANDLE_TABLE", "HandleCount", False), - ("_EPROCESS", "KeepAliveCounter", False), - ("_EPROCESS", "ControlFlowGuardEnabled", False)]) +is_win10_16299_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 16299), + fallback_checks=[ + ("ObHeaderCookie", None, True), + ("_HANDLE_TABLE", "HandleCount", False), + ("_EPROCESS", "KeepAliveCounter", False), + ("_EPROCESS", "ControlFlowGuardEnabled", False), + ], +) -is_win10_18363_or_later = OsDistinguisher(version_check = lambda x: x >= (10, 0, 18363), - fallback_checks = [("_KQOS_GROUPING_SETS", None, True)]) +is_win10_18363_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 18363), + fallback_checks=[("_KQOS_GROUPING_SETS", None, True)], +) -is_windows_10 = OsDistinguisher(version_check = lambda x: x >= (10, 0), - fallback_checks = [("ObHeaderCookie", None, True)]) +is_windows_10 = OsDistinguisher( + version_check=lambda x: x >= (10, 0), + fallback_checks=[("ObHeaderCookie", None, True)], +) -is_windows_8_or_later = OsDistinguisher(version_check = lambda x: x >= (6, 2), - fallback_checks = [("_HANDLE_TABLE", "HandleCount", False)]) +is_windows_8_or_later = OsDistinguisher( + version_check=lambda x: x >= (6, 2), + fallback_checks=[("_HANDLE_TABLE", "HandleCount", False)], +) # Technically, this is win7 or less -is_windows_7 = OsDistinguisher(version_check = lambda x: x == (6, 1), - fallback_checks = [("_OBJECT_HEADER", "TypeIndex", True), - ("_HANDLE_TABLE", "HandleCount", True)]) +is_windows_7 = OsDistinguisher( + version_check=lambda x: x == (6, 1), + fallback_checks=[ + ("_OBJECT_HEADER", "TypeIndex", True), + ("_HANDLE_TABLE", "HandleCount", True), + ], +) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 94962bc81..3212cb465 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -10,6 +10,7 @@ from volatility3.plugins.windows.registry import hivelist, printkey vollog = logging.getLogger(__name__) + class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" @@ -18,14 +19,23 @@ class Certificates(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)), - requirements.BooleanRequirement(name = 'dump', - description = "Extract listed certificates", - default = False, - optional = True) + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="printkey", plugin=printkey.PrintKey, version=(1, 0, 0) + ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed certificates", + default=False, + optional=True, + ), ] def parse_data(self, data: bytes) -> Tuple[str, bytes]: @@ -33,18 +43,22 @@ class Certificates(interfaces.plugins.PluginInterface): certificate_data = renderers.NotAvailableValue() while len(data) > 12: ctype, clength = struct.unpack(" \ - Optional[interfaces.plugins.FileHandlerInterface]: + def dump_certificate( + cls, + certificate_data: bytes, + hive_offset: int, + reg_section: str, + key_hash: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + ) -> Optional[interfaces.plugins.FileHandlerInterface]: try: dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) file_handle = open_method(dump_name) @@ -54,37 +68,69 @@ class Certificates(interfaces.plugins.PluginInterface): vollog.debug(f"Unable to dump certificate file at {hive_offset:#x}") return None - def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: - kernel = self.context.modules[self.config['kernel']] + kernel = self.context.modules[self.config["kernel"]] - for hive in hivelist.HiveList.list_hives(self.context, - base_config_path = self.config_path, - layer_name = kernel.layer_name, - symbol_table = kernel.symbol_table_name): + for hive in hivelist.HiveList.list_hives( + self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ): for top_key in [ - "Microsoft\\SystemCertificates", - "Software\\Microsoft\\SystemCertificates", + "Microsoft\\SystemCertificates", + "Software\\Microsoft\\SystemCertificates", ]: with contextlib.suppress(KeyError, exceptions.InvalidAddressException): # Walk it - node_path = hive.get_key(top_key, return_list = True) - for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): + node_path = hive.get_key(top_key, return_list=True) + for ( + _depth, + is_key, + _last_write_time, + key_path, + _volatility, + node, + ) in printkey.PrintKey.key_iterator(hive, node_path, recurse=True): if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) - unique_key_offset = key_path.casefold().index(top_key.casefold()) + len(top_key) + 1 - reg_section = key_path[unique_key_offset:key_path.index("\\", unique_key_offset)] - key_hash = key_path[key_path.rindex("\\") + 1:] + unique_key_offset = ( + key_path.casefold().index(top_key.casefold()) + + len(top_key) + + 1 + ) + reg_section = key_path[ + unique_key_offset : key_path.index( + "\\", unique_key_offset + ) + ] + key_hash = key_path[key_path.rindex("\\") + 1 :] - if self.config['dump']: - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - file_handle = self.dump_certificate(certificate_data, hive.hive_offset, reg_section, key_hash, self.open) + if self.config["dump"]: + if not isinstance( + certificate_data, + interfaces.renderers.BaseAbsentValue, + ): + file_handle = self.dump_certificate( + certificate_data, + hive.hive_offset, + reg_section, + key_hash, + self.open, + ) if file_handle: file_handle.close() - + yield (0, (top_key, reg_section, key_hash, name)) def run(self) -> renderers.TreeGrid: - return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), - ("Certificate name", str)], self._generator()) + return renderers.TreeGrid( + [ + ("Certificate path", str), + ("Certificate section", str), + ("Certificate ID", str), + ("Certificate name", str), + ], + self._generator(), + ) diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index e6f2016ed..e921b3565 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -18,16 +18,24 @@ class Statistics(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]) + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ) ] def _generator(self): # Do mass mapping and determine the number of different layers and how many pages go to each one - layer = self.context.layers[self.config['primary']] + layer = self.context.layers[self.config["primary"]] - page_count = swap_count = invalid_page_count = large_page_count = large_swap_count = large_invalid_count = other_invalid = 0 + page_count = ( + swap_count + ) = ( + invalid_page_count + ) = ( + large_page_count + ) = large_swap_count = large_invalid_count = other_invalid = 0 if isinstance(layer, intel.Intel): page_addr = 0 @@ -35,8 +43,10 @@ class Statistics(plugins.PluginInterface): while page_addr < layer.maximum_address: try: - _, _, _, page_size, layer_name = list(layer.mapping(page_addr, 2 * expected_page_size))[0] - if layer_name != layer.config['memory_layer']: + _, _, _, page_size, layer_name = list( + layer.mapping(page_addr, 2 * expected_page_size) + )[0] + if layer_name != layer.config["memory_layer"]: swap_count += 1 else: page_count += 1 @@ -44,28 +54,51 @@ class Statistics(plugins.PluginInterface): large_page_count += 1 except exceptions.SwappedInvalidAddressException as excp: swap_count += 1 - page_size = (1 << excp.invalid_bits) + page_size = 1 << excp.invalid_bits if page_size != expected_page_size: large_swap_count += 1 except exceptions.PagedInvalidAddressException as excp: invalid_page_count += 1 - page_size = (1 << excp.invalid_bits) + page_size = 1 << excp.invalid_bits if page_size != expected_page_size: large_invalid_count += 1 except exceptions.InvalidAddressException as excp: other_invalid += 1 page_size = expected_page_size - vollog.debug("A non-page lookup invalid address exception occurred at: {} in layer {}".format( - hex(excp.invalid_address), excp.layer_name)) + vollog.debug( + "A non-page lookup invalid address exception occurred at: {} in layer {}".format( + hex(excp.invalid_address), excp.layer_name + ) + ) page_addr += page_size - self._progress_callback((page_addr * 100) / layer.maximum_address, "Reading memory") + self._progress_callback( + (page_addr * 100) / layer.maximum_address, "Reading memory" + ) - yield (0, (page_count, large_page_count, swap_count, large_swap_count, invalid_page_count, large_invalid_count, - other_invalid)) + yield ( + 0, + ( + page_count, + large_page_count, + swap_count, + large_swap_count, + invalid_page_count, + large_invalid_count, + other_invalid, + ), + ) def run(self): - return renderers.TreeGrid([("Valid pages (all)", int), ("Valid pages (large)", int), - ("Swapped Pages (all)", int), ("Swapped Pages (large)", int), - ("Invalid Pages (all)", int), ("Invalid Pages (large)", int), - ("Other Invalid Pages (all)", int)], self._generator()) + return renderers.TreeGrid( + [ + ("Valid pages (all)", int), + ("Valid pages (large)", int), + ("Swapped Pages (all)", int), + ("Swapped Pages (large)", int), + ("Invalid Pages (all)", int), + ("Invalid Pages (large)", int), + ("Other Invalid Pages (all)", int), + ], + self._generator(), + ) diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 8666680b3..be120f2af 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -37,39 +37,45 @@ cached_validations = load_cached_validations() def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: """Validates an input JSON file based upon.""" - format = input.get('metadata', {}).get('format', None) + format = input.get("metadata", {}).get("format", None) if not format: vollog.debug("No schema format defined") return False basepath = os.path.abspath(os.path.dirname(__file__)) - schema_path = os.path.join(basepath, 'schema-' + format + '.json') + schema_path = os.path.join(basepath, "schema-" + format + ".json") if not os.path.exists(schema_path): vollog.debug(f"Schema for format not found: {schema_path}") return False - with open(schema_path, 'r') as s: + with open(schema_path, "r") as s: schema = json.load(s) return valid(input, schema, use_cache) -def create_json_hash(input: Dict[str, Any], schema: Optional[Dict[str, Any]] = None) -> Optional[str]: +def create_json_hash( + input: Dict[str, Any], schema: Optional[Dict[str, Any]] = None +) -> Optional[str]: """Constructs the hash of the input and schema to create a unique identifier for a particular JSON file.""" if schema is None: - format = input.get('metadata', {}).get('format', None) + format = input.get("metadata", {}).get("format", None) if not format: vollog.debug("No schema format defined") return None basepath = os.path.abspath(os.path.dirname(__file__)) - schema_path = os.path.join(basepath, 'schema-' + format + '.json') + schema_path = os.path.join(basepath, "schema-" + format + ".json") if not os.path.exists(schema_path): vollog.debug(f"Schema for format not found: {schema_path}") return None - with open(schema_path, 'r') as s: + with open(schema_path, "r") as s: schema = json.load(s) - return hashlib.sha1(bytes(json.dumps((input, schema), sort_keys = True), 'utf-8')).hexdigest() + return hashlib.sha1( + bytes(json.dumps((input, schema), sort_keys=True), "utf-8") + ).hexdigest() -def valid(input: Dict[str, Any], schema: Dict[str, Any], use_cache: bool = True) -> bool: +def valid( + input: Dict[str, Any], schema: Dict[str, Any], use_cache: bool = True +) -> bool: """Validates a json schema.""" input_hash = create_json_hash(input, schema) if input_hash in cached_validations and use_cache: @@ -87,7 +93,7 @@ def valid(input: Dict[str, Any], schema: Dict[str, Any], use_cache: bool = True) cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") except jsonschema.exceptions.SchemaError: - vollog.debug("Schema validation error", exc_info = True) + vollog.debug("Schema validation error", exc_info=True) return False record_cached_validations(cached_validations) diff --git a/volshell.py b/volshell.py index 590994324..71d35a47c 100644 --- a/volshell.py +++ b/volshell.py @@ -6,5 +6,5 @@ from volatility3.cli import volshell -if __name__ == '__main__': +if __name__ == "__main__": volshell.main() From 66d63676edc47c84119e8e4a127aa2561706284f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 7 Dec 2022 20:54:52 +0000 Subject: [PATCH 249/526] Core: Fix issue in constants code based on CodeQL result --- volatility3/framework/constants/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 9eb7bbdb0..8f1163fe1 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -120,10 +120,10 @@ REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' # DEPRECATED VALUES ### -_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, 'linux_banners.cache') +_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, "linux_banners.cache") """This value is deprecated and is no longer used within volatility""" -_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, 'mac_banners.cache') +_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") """This value is deprecated and is no longer used within volatility""" _deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) @@ -131,7 +131,10 @@ _deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) def __getattr__(name): - deprecated_tag = '_deprecated_' - if name in [x[len(deprecated_tag):] for x in globals() if x.startswith(deprecated_tag)]: + deprecated_tag = "_deprecated_" + if name in [ + x[len(deprecated_tag) :] for x in globals() if x.startswith(deprecated_tag) + ]: warnings.warn(f"{name} is deprecated", FutureWarning) return globals()[f"{deprecated_tag}{name}"] + return None From 441be4535cc2b5d21b85cd89e1cc6df5d832201c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 7 Dec 2022 21:09:32 +0000 Subject: [PATCH 250/526] Core: Fix up black workflow action --- .github/workflows/black.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 6bd6fd1af..ff16e5297 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,4 +1,13 @@ -- uses: psf/black@stable - with: - options: "--check --diff --verbose" - src: "./volatility3" +name: Black python linter + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: psf/black@stable + with: + options: "--check --diff --verbose" + src: "./volatility3" From 12109c6f7218bc02b555966da706062b5a03f77a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 7 Dec 2022 21:11:22 +0000 Subject: [PATCH 251/526] Core: Fix up lint issue from pull request --- volatility3/framework/layers/resources.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index b9eae6634..3d47e6616 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -140,7 +140,9 @@ class ResourceAccessor(object): except error.URLError as excp: if excp.args: if isinstance(excp.args[0], ssl.SSLCertVerificationError): - vollog.warning("SSL certificate verification failed: attempting UNVERIFIED retrieval") + vollog.warning( + "SSL certificate verification failed: attempting UNVERIFIED retrieval" + ) non_verifying_ctx = ssl.SSLContext() non_verifying_ctx.check_hostname = False non_verifying_ctx.verify_mode = ssl.CERT_NONE From 4cf0a185e397ecbfec0d8921fde0035a9f683a2b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 7 Dec 2022 21:12:49 +0000 Subject: [PATCH 252/526] Core: Bump minimum python 3 version for pypi build --- .github/workflows/build-pypi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 061c8359a..939621422 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -15,10 +15,10 @@ on: jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: - python-version: ["3.6"] + python-version: ["3.7"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From e67bac4080bb8a782d0003076342a92d162b0e21 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 7 Dec 2022 21:25:22 +0000 Subject: [PATCH 253/526] Testing: Fix unclosed file open --- test/test_volatility.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index fb4fece12..aaad615bc 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -120,9 +120,8 @@ def test_windows_hivelist(image, volatility, python): def test_windows_dumpfiles(image, volatility, python): - json_file = open("./test/known_files.json") - - known_files = json.load(json_file) + with open("./test/known_files.json") as json_file: + known_files = json.load(json_file) failed_chksms = 0 From 5f4e2716dbf43f03421d880d7a678879ef6ca7af Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 8 Dec 2022 16:26:54 +0900 Subject: [PATCH 254/526] Fix: ubuntu-latest version --- .github/workflows/build-pypi.yml | 2 +- .github/workflows/test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 939621422..073bcda3b 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -15,7 +15,7 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: matrix: python-version: ["3.7"] diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b1a9dd31b..acb5f1afa 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: python-version: ["3.7"] From 632fe0b6715119dc1be4e585e8cf7a7311524777 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 9 Dec 2022 01:00:16 +0900 Subject: [PATCH 255/526] Fix: ubuntu version to 20.04 --- .github/workflows/black.yml | 2 +- .github/workflows/build-pypi.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/test.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index ff16e5297..dba5b5b80 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: lint: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: psf/black@stable diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 073bcda3b..f21898971 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -15,7 +15,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: python-version: ["3.7"] diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 72bba07aa..078af2abf 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -23,7 +23,7 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 permissions: actions: read contents: read diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index acb5f1afa..b1a9dd31b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: python-version: ["3.7"] From 92c7b3e5500b03fa68d8893204b31f50cef7b4dc Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 9 Dec 2022 15:37:20 +0000 Subject: [PATCH 256/526] add linux envars --- volatility3/framework/plugins/linux/envars.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 volatility3/framework/plugins/linux/envars.py diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py new file mode 100644 index 000000000..b62400c45 --- /dev/null +++ b/volatility3/framework/plugins/linux/envars.py @@ -0,0 +1,110 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from volatility3.framework import exceptions, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + +class Envars(plugins.PluginInterface): + """Lists processes with their environment variables""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + ] + + def _generator(self, tasks): + """Generates a listing of processes along with environment variables""" + + # walk the process list and return the envars + for task in tasks: + pid = task.pid + + # get process name as string + name = utility.array_to_string(task.comm) + + # try and get task parent + try: + ppid = task.parent.pid + except exceptions.InvalidAddressException: + vollog.debug(f"Unable to read parent pid for task {pid} {name}, setting ppid to 0.") + ppid = 0 + + # kernel threads never have an mm as they do not have userland mappings + try: + mm = task.mm + except exceptions.InvalidAddressException: + # no mm so cannot get envars + vollog.debug(f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars.") + mm = None + continue + + # if mm exists attempt to get envars + if mm: + + # get process layer to read envars from + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + vollog.debug(f"Unable to construct process layer for task {pid} {name}, will not extract any envars.") + continue + proc_layer = self.context.layers[proc_layer_name] + + + # get the size of the envars with sanity checking + envars_size = task.mm.env_end - task.mm.env_start + if not (0 < envars_size <= 8192): + vollog.debug(f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars.") + continue + + # attempt to read all envars data + try: + envar_data = proc_layer.read(task.mm.env_start, envars_size) + except exceptions.InvalidAddressException: + vollog.debug(f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars.") + continue + + # parse envar data, envars are null terminated, keys and values are separated by '=' + envar_data = envar_data.rstrip(b'\x00') + for envar_pair in envar_data.split(b'\x00'): + try: + key, value = envar_pair.decode().split('=', 1) + except ValueError: + vollog.debug(f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated") + continue + yield (0, (pid, ppid, name, key, value)) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + return renderers.TreeGrid( + [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), + ) From 53b24d33e0d3c63fec59d23bf553f35bfff92580 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 13 Dec 2022 09:59:09 +0000 Subject: [PATCH 257/526] First attempt at adding a --dump option to linux.proc, aim to be similar to windows.vadinfo --dump --- volatility3/framework/plugins/linux/proc.py | 143 +++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 9d8af482e..6d182ff9e 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -4,18 +4,23 @@ """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" -from volatility3.framework import renderers +import logging +from typing import Callable, List, Generator, Iterable, Type, Optional + +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) + MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod def get_requirements(cls): @@ -35,16 +40,138 @@ class Maps(plugins.PluginInterface): element_type=int, optional=True, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed memory segments", + default=False, + optional=True, + ), + requirements.ListRequirement( + name="address", + description="Process virtual memory address to include " + "(all other address ranges are excluded). This must be " + "a base address, not an address within the desired range.", + element_type=int, + optional=True, + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size for dumped VMA sections " + "(all the bigger sections will be ignored)", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), ] + @classmethod + def list_vmas( + cls, + task: interfaces.objects.ObjectInterface, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: False, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Lists the Virtual Memory Areas of a specific process. + + Args: + task: task object from which to list the vma + filter_func: Function to take a vma and return True if it should be filtered out + + Returns: + A list of vmas based on the task and filtered based on the filter function + """ + if task.mm: + for vma in task.mm.get_mmap_iter(): + if not filter_func(vma): + yield vma + + @classmethod + def vma_dump( + cls, + context: interfaces.context.ContextInterface, + task: interfaces.objects.ObjectInterface, + vma: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + maxsize: int = MAXSIZE_DEFAULT, + ) -> Optional[interfaces.plugins.FileHandlerInterface]: + """Extracts the complete data for VMA as a FileInterface. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + task: an task_struct instance + vma: The suspected VMA to extract (ObjectInterface) + open_method: class to provide context manager for opening the file + maxsize: Max size of VMA section (default MAXSIZE_DEFAULT) + + Returns: + An open FileInterface object containing the complete data for the task or None in the case of failure + """ + try: + vm_start = vma.vm_start + vm_end = vma.vm_end + except AttributeError: + vollog.debug("Unable to find the vm_start and vm_end") + return None + + vm_size = vm_end - vm_start + if 0 < maxsize < vm_size: + vollog.debug( + f"Skip virtual memory dump {vm_start:#x}-{vm_end:#x} due to maxsize limit" + ) + return None + + pid = "Unknown" + try: + pid = task.tgid + proc_layer_name = task.add_process_layer() + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + pid, excp.invalid_address, excp.layer_name + ) + ) + return None + + proc_layer = context.layers[proc_layer_name] + file_name = f"pid.{pid}.vma.{vm_start:#x}-{vm_end:#x}.dmp" + try: + file_handle = open_method(file_name) + chunk_size = 1024 * 1024 * 10 + offset = vm_start + while offset < vm_start + vm_size: + to_read = min(chunk_size, vm_start + vm_size - offset) + data = proc_layer.read(offset, to_read, pad=True) + if not data: + break + file_handle.write(data) + offset += to_read + + except Exception as excp: + vollog.debug(f"Unable to dump virtual memory {file_name}: {excp}") + return None + + return file_handle + def _generator(self, tasks): + # build filter for addresses if required + address_list = self.config.get("address", []) + if address_list == []: + # do not filter as no address_list was supplied + filter_func = lambda _: False + else: + # filter for any vm_start that matches the supplied address config + def filter_function(x: interfaces.objects.ObjectInterface) -> bool: + return x.vm_start not in address_list + + filter_func = filter_function + for task in tasks: if not task.mm: continue name = utility.array_to_string(task.comm) - for vma in task.mm.get_mmap_iter(): + for vma in self.list_vmas(task, filter_func=filter_func): flags = vma.get_protection() page_offset = vma.get_page_offset() major = 0 @@ -61,6 +188,16 @@ class Maps(plugins.PluginInterface): path = vma.get_name(self.context, task) + file_output = "Disabled" + if self.config["dump"]: + file_handle = self.vma_dump( + self.context, task, vma, self.open, self.config["maxsize"] + ) + file_output = "Error outputting file" + if file_handle: + file_handle.close() + file_output = file_handle.preferred_filename + yield ( 0, ( @@ -74,6 +211,7 @@ class Maps(plugins.PluginInterface): minor, inode, path, + file_output, ), ) @@ -92,6 +230,7 @@ class Maps(plugins.PluginInterface): ("Minor", int), ("Inode", int), ("File Path", str), + ("File output", str), ], self._generator( pslist.PsList.list_tasks( From a68d56e19109a81098ae208d33ae94a679ff0824 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 13 Dec 2022 16:09:57 +0000 Subject: [PATCH 258/526] add linux.vmayarascan based on windows.vadtarascan --- .../framework/plugins/linux/vmayarascan.py | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 volatility3/framework/plugins/linux/vmayarascan.py diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py new file mode 100644 index 000000000..e9482089a --- /dev/null +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -0,0 +1,121 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from typing import Iterable, List, Tuple + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins import yarascan +from volatility3.plugins.linux import pslist + +class VmaYaraScan(interfaces.plugins.PluginInterface): + """Scans all virtual memory areas for tasks using yara.""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.BooleanRequirement( + name="wide", + description="Match wide (unicode) strings", + default=False, + optional=True, + ), + requirements.StringRequirement( + name="yara_rules", description="Yara rules (as a string)", optional=True + ), + requirements.URIRequirement( + name="yara_file", description="Yara rules (as a file)", optional=True + ), + # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code + # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later + requirements.URIRequirement( + name="yara_compiled_file", + description="Yara compiled rules (as a file)", + optional=True, + ), + requirements.IntRequirement( + name="max_size", + default=0x40000000, + description="Set the maximum size (default is 1GB)", + optional=True, + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), + ] + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + rules = yarascan.YaraScan.process_yara_options(dict(self.config)) + + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + for task in pslist.PsList.list_tasks( + context=self.context, + vmlinux_module_name=self.config["kernel"], + filter_func=filter_func, + ): + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + continue + + proc_layer = self.context.layers[proc_layer_name] + for offset, rule_name, name, value in proc_layer.scan( + context=self.context, + scanner=yarascan.YaraScanner(rules=rules), + sections=self.get_vma_maps(task), + ): + yield 0, ( + format_hints.Hex(offset), + task.tgid, + rule_name, + name, + value, + ) + + @staticmethod + def get_vma_maps( + task: interfaces.objects.ObjectInterface, + ) -> Iterable[Tuple[int, int]]: + """Creates a map of start/end addresses for each virtual memory area in a task. + + Args: + task: The task object of which to read the vmas from + + Returns: + An iterable of tuples containing start and end addresses for each descriptor + """ + if task.mm: + for vma in task.mm.get_mmap_iter(): + vm_size = vma.vm_end - vma.vm_start + yield (vma.vm_start, vm_size) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("PID", int), + ("Rule", str), + ("Component", str), + ("Value", bytes), + ], + self._generator(), + ) From 92ece08c5f709777564efda9111a352a886ccda4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 14 Dec 2022 18:50:49 +0000 Subject: [PATCH 259/526] Core: Fix up file close issue --- volatility3/framework/layers/resources.py | 24 +++++++++++------------ 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 3d47e6616..a64fa7d7a 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -179,20 +179,18 @@ class ResourceAccessor(object): except AttributeError: # If our fp doesn't have an info member, carry on gracefully content_length = -1 - cache_file = open(temp_filename, "wb") - - count = 0 - block = fp.read(block_size) - while block: - count += len(block) - if self._progress_callback: - self._progress_callback( - count * 100 / max(count, int(content_length)), - f"Reading file {url}", - ) - cache_file.write(block) + with open(temp_filename, "wb") as cache_file: + count = 0 block = fp.read(block_size) - cache_file.close() + while block: + count += len(block) + if self._progress_callback: + self._progress_callback( + count * 100 / max(count, int(content_length)), + f"Reading file {url}", + ) + cache_file.write(block) + block = fp.read(block_size) else: vollog.debug(f"Using already cached file at: {temp_filename}") # Re-open the cache with a different mode From a553a69efde143183c0580910dcc089cf0e060ed Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 21 Dec 2022 06:42:02 +0000 Subject: [PATCH 260/526] fix linting issues --- volatility3/framework/plugins/linux/envars.py | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index b62400c45..028eb2a57 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -12,6 +12,7 @@ from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) + class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" @@ -51,7 +52,9 @@ class Envars(plugins.PluginInterface): try: ppid = task.parent.pid except exceptions.InvalidAddressException: - vollog.debug(f"Unable to read parent pid for task {pid} {name}, setting ppid to 0.") + vollog.debug( + f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." + ) ppid = 0 # kernel threads never have an mm as they do not have userland mappings @@ -59,7 +62,9 @@ class Envars(plugins.PluginInterface): mm = task.mm except exceptions.InvalidAddressException: # no mm so cannot get envars - vollog.debug(f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars.") + vollog.debug( + f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." + ) mm = None continue @@ -69,31 +74,38 @@ class Envars(plugins.PluginInterface): # get process layer to read envars from proc_layer_name = task.add_process_layer() if proc_layer_name is None: - vollog.debug(f"Unable to construct process layer for task {pid} {name}, will not extract any envars.") + vollog.debug( + f"Unable to construct process layer for task {pid} {name}, will not extract any envars." + ) continue proc_layer = self.context.layers[proc_layer_name] - # get the size of the envars with sanity checking envars_size = task.mm.env_end - task.mm.env_start if not (0 < envars_size <= 8192): - vollog.debug(f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars.") + vollog.debug( + f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." + ) continue # attempt to read all envars data try: envar_data = proc_layer.read(task.mm.env_start, envars_size) except exceptions.InvalidAddressException: - vollog.debug(f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars.") + vollog.debug( + f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." + ) continue # parse envar data, envars are null terminated, keys and values are separated by '=' - envar_data = envar_data.rstrip(b'\x00') - for envar_pair in envar_data.split(b'\x00'): + envar_data = envar_data.rstrip(b"\x00") + for envar_pair in envar_data.split(b"\x00"): try: - key, value = envar_pair.decode().split('=', 1) + key, value = envar_pair.decode().split("=", 1) except ValueError: - vollog.debug(f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated") + vollog.debug( + f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" + ) continue yield (0, (pid, ppid, name, key, value)) From f82a3f520facdc4e9ba973ddf8316473bbd190fe Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 4 Jan 2023 09:32:39 +0000 Subject: [PATCH 261/526] liniting for linux.vmayarascan --- volatility3/framework/plugins/linux/vmayarascan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index e9482089a..3efbd2ae1 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -10,6 +10,7 @@ from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan from volatility3.plugins.linux import pslist + class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" From 6f82e3d8cf7b173f07c51634016413b5ef2243c6 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 4 Jan 2023 09:36:20 +0000 Subject: [PATCH 262/526] remove unused variable in linux.vmayarascan --- volatility3/framework/plugins/linux/vmayarascan.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 3efbd2ae1..c7a48cc14 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -64,8 +64,6 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): ] def _generator(self): - kernel = self.context.modules[self.config["kernel"]] - rules = yarascan.YaraScan.process_yara_options(dict(self.config)) filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) From 4c0a0b923bc1b7c13e753e90f84db44d95b95368 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 4 Jan 2023 16:05:00 +0000 Subject: [PATCH 263/526] Update linux.proc --dump changes based on comments from ikelos --- volatility3/framework/plugins/linux/proc.py | 64 ++++++++++++--------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 6d182ff9e..d8a17ae38 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -5,7 +5,7 @@ found in Linux's /proc file system.""" import logging -from typing import Callable, List, Generator, Iterable, Type, Optional +from typing import Callable, Generator, Type, Optional from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements @@ -16,6 +16,7 @@ from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) + class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" @@ -48,9 +49,9 @@ class Maps(plugins.PluginInterface): ), requirements.ListRequirement( name="address", - description="Process virtual memory address to include " - "(all other address ranges are excluded). This must be " - "a base address, not an address within the desired range.", + description="Process virtual memory addresses to include " + "(all other VMA sections are excluded). This can be any " + "virtual address within the VMA section.", element_type=int, optional=True, ), @@ -69,21 +70,25 @@ class Maps(plugins.PluginInterface): task: interfaces.objects.ObjectInterface, filter_func: Callable[ [interfaces.objects.ObjectInterface], bool - ] = lambda _: False, + ] = lambda _: True, ) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Lists the Virtual Memory Areas of a specific process. Args: task: task object from which to list the vma - filter_func: Function to take a vma and return True if it should be filtered out + filter_func: Function to take a vma and return False if it should be filtered out Returns: - A list of vmas based on the task and filtered based on the filter function + Yields vmas based on the task and filtered based on the filter function """ if task.mm: for vma in task.mm.get_mmap_iter(): - if not filter_func(vma): + if filter_func(vma): yield vma + else: + vollog.debug( + f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.pid} due to filter_func" + ) @classmethod def vma_dump( @@ -106,23 +111,15 @@ class Maps(plugins.PluginInterface): Returns: An open FileInterface object containing the complete data for the task or None in the case of failure """ + pid = task.pid try: vm_start = vma.vm_start vm_end = vma.vm_end except AttributeError: - vollog.debug("Unable to find the vm_start and vm_end") - return None - - vm_size = vm_end - vm_start - if 0 < maxsize < vm_size: - vollog.debug( - f"Skip virtual memory dump {vm_start:#x}-{vm_end:#x} due to maxsize limit" - ) + vollog.debug(f"Unable to find the vm_start and vm_end for pid {pid}") return None - pid = "Unknown" try: - pid = task.tgid proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( @@ -132,6 +129,13 @@ class Maps(plugins.PluginInterface): ) return None + vm_size = vm_end - vm_start + if 0 < maxsize < vm_size: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is larger than maxsize limit of {maxsize}" + ) + return None + proc_layer = context.layers[proc_layer_name] file_name = f"pid.{pid}.vma.{vm_start:#x}-{vm_end:#x}.dmp" try: @@ -141,8 +145,6 @@ class Maps(plugins.PluginInterface): while offset < vm_start + vm_size: to_read = min(chunk_size, vm_start + vm_size - offset) data = proc_layer.read(offset, to_read, pad=True) - if not data: - break file_handle.write(data) offset += to_read @@ -154,16 +156,24 @@ class Maps(plugins.PluginInterface): def _generator(self, tasks): # build filter for addresses if required - address_list = self.config.get("address", []) - if address_list == []: + address_list = self.config.get("address", None) + if not address_list: # do not filter as no address_list was supplied - filter_func = lambda _: False + vma_filter_func = lambda _: True else: # filter for any vm_start that matches the supplied address config - def filter_function(x: interfaces.objects.ObjectInterface) -> bool: - return x.vm_start not in address_list + def vma_filter_function(x: interfaces.objects.ObjectInterface) -> bool: + addrs_in_vma = [ + addr for addr in address_list if x.vm_start <= addr <= x.vm_end + ] - filter_func = filter_function + # if any of the user supplied addresses would fall within this vma return true + if addrs_in_vma: + return True + else: + return False + + vma_filter_func = vma_filter_function for task in tasks: if not task.mm: @@ -171,7 +181,7 @@ class Maps(plugins.PluginInterface): name = utility.array_to_string(task.comm) - for vma in self.list_vmas(task, filter_func=filter_func): + for vma in self.list_vmas(task, filter_func=vma_filter_func): flags = vma.get_protection() page_offset = vma.get_page_offset() major = 0 From c1e425217bf8ea0e4b62a56ddaff5db29d87b4f0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 5 Jan 2023 10:20:19 +0000 Subject: [PATCH 264/526] Initial canonical helper addition The intel 64-bit 4-page paging mechanism allows for 48-bit virtual addresses. They introduced a convention that the higher bits must be set a particular way to avoid operating system developers abusing those bits and creating problems that would be difficult to resolve in the future. Volatility requires that addresses for mapping or translation fit within the available bounds of the virtual address space. This unfortunately means that addresses that have the protections against abuse in place can may live outside this range. This provides two function (canonicalize and decanonicalize) which will either set the appropriate sign extension or remove it. The decanonicalize function will return an adress outside of the address range if the original value was not canonical. --- volatility3/framework/layers/intel.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index dce207fd5..ecfb6bf11 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -56,6 +56,11 @@ class Intel(linear.LinearlyMappedLayer): ) self._entry_size = struct.calcsize(self._entry_format) self._entry_number = self.page_size // self._entry_size + self._canonical_prefix = self._mask( + (1 << self._bits_per_register) - 1, + self._bits_per_register, + self._maxvirtaddr, + ) # These can vary depending on the type of space self._index_shift = int( @@ -106,6 +111,23 @@ class Intel(linear.LinearlyMappedLayer): """Returns whether a particular page is valid based on its entry.""" return bool(entry & 1) + def canonicalize(self, addr: int) -> int: + """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" + if self._bits_per_register <= self._maxvirtaddr: + return addr & self.address_mask + elif addr < (1 << self._maxvirtaddr - 1): + return addr + return self._mask(addr, self._maxvirtaddr, 0) + self._canonical_prefix + + def decanonicalize(self, addr: int) -> int: + """Removes canonicalization to ensure an adress fits within the correct range if it has been canonicalized + + This will produce an address outside the range if the canonicalization is incorrect + """ + if addr < (1 << self._maxvirtaddr - 1): + return addr + return addr ^ self._canonical_prefix + def _translate(self, offset: int) -> Tuple[int, int, str]: """Translates a specific offset based on paging tables. From 0163f0b9e67258d2a433766d0027ffc25d0b6d07 Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 5 Jan 2023 12:17:36 +0000 Subject: [PATCH 265/526] add linux.iomem plugin based on vol2 plugin by atcuno --- volatility3/framework/plugins/linux/iomem.py | 139 +++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 volatility3/framework/plugins/linux/iomem.py diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py new file mode 100644 index 000000000..6b0469d60 --- /dev/null +++ b/volatility3/framework/plugins/linux/iomem.py @@ -0,0 +1,139 @@ +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import List + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints + +vollog = logging.getLogger(__name__) + + +class IOMem(interfaces.plugins.PluginInterface): + """Generates an output similar to /proc/iomem on a running system.""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ) + ] + + @classmethod + def parse_resource( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + resource_offset: int, + seen: set = set(), + depth: int = 0, + ): + """Recursively parse from a root resource to find details about all related resources. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + resource_offset: The offset to the resouce to be parsed + seen: The set of resource offsets that have already been parsed + depth: How deep into the resource structure we are + + Yields: + Each row of output + """ + # create the resource object + vmlinux = context.modules[vmlinux_module_name] + resource = vmlinux.object("resource", resource_offset) + + # extract the information required for this resource + name = utility.pointer_to_string(resource.name, 128) + start = format_hints.Hex(resource.start) + end = format_hints.Hex(resource.end) + + # mark this resource as seen in the seen set. Normally this should not be needed but will protect + # against possible infinite loops. Warn the user if an infinite loop would have happened. + if resource_offset in seen: + vollog.warning( + f"The resource object at {resource_offset:#x} '{name}' has already been processed, " + "this should not normally occur. No further results from related resources will be " + "displayed to protect against infinite loops." + ) + return None + else: + seen.add(resource_offset) + + # yield information on this resource + yield depth, (name, start, end) + + # process child resource if this exists + if resource.child != 0: + yield from cls.parse_resource( + context, + vmlinux_module_name, + resource.child, + seen, + depth + 1, + ) + + # process sibling resource if this exists + if resource.sibling != 0: + yield from cls.parse_resource( + context, + vmlinux_module_name, + resource.sibling, + seen, + depth, + ) + + def _generator(self): + """Generates an output similar to /proc/iomem on a running system + + Args: + None + + Yields: + Each row of output using the parse_resource function + """ + + # get the kernel module from the current context + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] + + # check that the iomem_resource symbol exists + # normally exported in /kernel/resource.c + try: + iomem_root_offset = vmlinux.get_absolute_symbol_address("iomem_resource") + except exceptions.SymbolError: + iomem_root_offset = None + + # error if 'iomem_resource' is not found + if not iomem_root_offset: + raise TypeError( + "This plugin requires the iomem_resource structure. This structure is not present in the supplied symbol table. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." + ) + + # error if type 'resource' is not found + if not vmlinux.has_type("resource"): + raise TypeError( + "This plugin requires the resource type. This type is not present in the supplied symbol table. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." + ) + + # recursively parse the resources starting from the root resource at 'iomem_resource' + yield from self.parse_resource( + self.context, vmlinux_module_name, iomem_root_offset + ) + + def run(self): + columns = [ + ("NAME", str), + ("START", format_hints.Hex), + ("END", format_hints.Hex), + ] + return renderers.TreeGrid(columns, self._generator()) From af3b70320ecbbfc38c8ce51cc2aacf7ccd584580 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 6 Jan 2023 10:18:17 +0000 Subject: [PATCH 266/526] linux: Apply black linting to outstanding files --- .../framework/constants/linux/__init__.py | 20 +- volatility3/framework/plugins/linux/lsof.py | 27 +-- .../framework/plugins/linux/sockstat.py | 192 +++++++++++++----- .../framework/symbols/linux/__init__.py | 60 +++--- .../symbols/linux/extensions/__init__.py | 29 ++- 5 files changed, 219 insertions(+), 109 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 0c4d3c376..1b133eb42 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -13,7 +13,7 @@ PAGE_SHIFT = 12 """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" # include/linux/sched.h -PF_KTHREAD = 0x00200000 # I'm a kernel thread +PF_KTHREAD = 0x00200000 # I'm a kernel thread # Standard well-defined IP protocols. # ref: include/uapi/linux/in.h @@ -139,13 +139,7 @@ SOCK_FAMILY = ( # Socket states # ref: include/uapi/linux/net.h -SOCKET_STATES = ( - "FREE", - "UNCONNECTED", - "CONNECTING", - "CONNECTED", - "DISCONNECTING" -) +SOCKET_STATES = ("FREE", "UNCONNECTED", "CONNECTING", "CONNECTED", "DISCONNECTING") # Netlink protocols # ref: include/uapi/linux/netlink.h @@ -188,17 +182,17 @@ ETH_PROTOCOLS = { 0x0007: "ETH_P_WAN_PPP", 0x0008: "ETH_P_PPP_MP", 0x0009: "ETH_P_LOCALTALK", - 0x000c: "ETH_P_CAN", - 0x000f: "ETH_P_CANFD", + 0x000C: "ETH_P_CAN", + 0x000F: "ETH_P_CANFD", 0x0010: "ETH_P_PPPTALK", 0x0011: "ETH_P_TR_802_2", 0x0016: "ETH_P_CONTROL", 0x0017: "ETH_P_IRDA", 0x0018: "ETH_P_ECONET", 0x0019: "ETH_P_HDLC", - 0x001a: "ETH_P_ARCNET", - 0x001b: "ETH_P_DSA", - 0x001c: "ETH_P_TRAILER", + 0x001A: "ETH_P_ARCNET", + 0x001B: "ETH_P_DSA", + 0x001C: "ETH_P_TRAILER", 0x0060: "ETH_P_LOOP", 0x00F6: "ETH_P_IEEE802154", 0x00F7: "ETH_P_CAIF", diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 920aaf7f1..62bade1f1 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -46,10 +46,12 @@ class Lsof(plugins.PluginInterface): ] @classmethod - def list_fds(cls, - context: interfaces.context.ContextInterface, - symbol_table: str, - filter_func: Callable[[int], bool] = lambda _: False): + def list_fds( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + filter_func: Callable[[int], bool] = lambda _: False, + ): linuxutils_symbol_table = None # type: ignore for task in pslist.PsList.list_tasks(context, symbol_table, filter_func): @@ -62,18 +64,17 @@ class Lsof(plugins.PluginInterface): pid = int(task.pid) fd_generator = linux.LinuxUtilities.files_descriptors_for_process( - context, - linuxutils_symbol_table, - task) + context, linuxutils_symbol_table, task + ) for fd_fields in fd_generator: yield pid, task_comm, task, fd_fields def _generator(self, pids, symbol_table): filter_func = pslist.PsList.create_pid_filter(pids) - fds_generator = self.list_fds(self.context, - symbol_table, - filter_func=filter_func) + fds_generator = self.list_fds( + self.context, symbol_table, filter_func=filter_func + ) for pid, task_comm, _task, fd_fields in fds_generator: fd_num, _filp, full_path = fd_fields @@ -82,8 +83,8 @@ class Lsof(plugins.PluginInterface): yield (0, fields) def run(self): - pids = self.config.get('pid', None) - symbol_table = self.config['kernel'] + pids = self.config.get("pid", None) + symbol_table = self.config["kernel"] tree_grid_args = [("PID", int), ("Process", str), ("FD", int), ("Path", str)] - return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) \ No newline at end of file + return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index ad3eee01f..f03a2ad8e 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -16,6 +16,7 @@ from volatility3.plugins.linux import lsof vollog = logging.getLogger(__name__) + class SockHandlers(interfaces.configuration.VersionableInterface): """Handles several socket families extracting the sockets information.""" @@ -56,7 +57,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): nethead = self._vmlinux.object_from_symbol(symbol_name="net_namespace_list") net_symname = self._vmlinux.symbol_table_name + constants.BANG + "net" for net in nethead.to_list(net_symname, "list"): - net_device_symname = self._vmlinux.symbol_table_name + constants.BANG + "net_device" + net_device_symname = ( + self._vmlinux.symbol_table_name + constants.BANG + "net_device" + ) for net_dev in net.dev_base_head.to_list(net_device_symname, "dev_list"): if net.get_inode() != netns_id: continue @@ -64,7 +67,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): netdevices_map[net_dev.ifindex] = dev_name return netdevices_map - def process_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str], Dict]: + def process_sock( + self, sock: objects.StructType + ) -> Tuple[objects.StructType, Tuple[str, str, str], Dict]: """Takes a kernel generic `sock` object and processes it with its respective socket family Args: @@ -86,7 +91,12 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return unix_sock, sock_stat, socket_filter except exceptions.SymbolError as e: # Cannot finds the *_sock type in the symbols - vollog.log(constants.LOGLEVEL_V, "Error processing socket family '%s': %s", family, e) + vollog.log( + constants.LOGLEVEL_V, + "Error processing socket family '%s': %s", + family, + e, + ) else: vollog.log(constants.LOGLEVEL_V, "Unsupported family '%s'", family) @@ -100,7 +110,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return sock, sock_stat, socket_filter - def _update_socket_filters_info(self, sock: objects.Pointer, socket_filter: dict) -> None: + def _update_socket_filters_info( + self, sock: objects.Pointer, socket_filter: dict + ) -> None: """Get information from the socket and reuseport filters Args: @@ -117,7 +129,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): socket_filter["filter_type"] = "reuseport_filter" self._extract_socket_filter_info(sock_reuseport_cb, socket_filter) - def _extract_socket_filter_info(self, sock_filter: objects.Pointer, socket_filter: dict) -> None: + def _extract_socket_filter_info( + self, sock_filter: objects.Pointer, socket_filter: dict + ) -> None: """Get specific information for each type of filter Args: @@ -146,7 +160,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if bpfprog_name: socket_filter["bpf_filter_name"] = bpfprog_name - def _unix_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + def _unix_sock( + self, sock: objects.StructType + ) -> Tuple[objects.StructType, Tuple[str, str, str]]: """Handles the AF_UNIX socket family Args: @@ -171,7 +187,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = src_addr, src_port, dst_addr, dst_port, state return unix_sock, sock_stat - def _inet_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + def _inet_sock( + self, sock: objects.StructType + ) -> Tuple[objects.StructType, Tuple[str, str, str]]: """Handles the AF_INET/6 socket families Args: @@ -191,7 +209,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = src_addr, src_port, dst_addr, dst_port, state return inet_sock, sock_stat - def _netlink_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + def _netlink_sock( + self, sock: objects.StructType + ) -> Tuple[objects.StructType, Tuple[str, str, str]]: """Handles the AF_NETLINK socket family Args: @@ -221,7 +241,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = src_addr, src_port, dst_addr, dst_port, state return netlink_sock, sock_stat - def _vsock_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + def _vsock_sock( + self, sock: objects.StructType + ) -> Tuple[objects.StructType, Tuple[str, str, str]]: """Handles the AF_VSOCK socket family Args: @@ -241,7 +263,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = src_addr, src_port, dst_addr, dst_port, state return vsock_sock, sock_stat - def _packet_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + def _packet_sock( + self, sock: objects.StructType + ) -> Tuple[objects.StructType, Tuple[str, str, str]]: """Handles the AF_PACKET socket family Args: @@ -262,7 +286,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = src_addr, src_port, dst_addr, dst_port, state return packet_sock, sock_stat - def _xdp_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + def _xdp_sock( + self, sock: objects.StructType + ) -> Tuple[objects.StructType, Tuple[str, str, str]]: """Handles the AF_XDP socket family Args: @@ -304,7 +330,9 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock_stat = src_addr, src_port, dst_addr, dst_port, state return xdp_sock, sock_stat - def _bluetooth_sock(self, sock: objects.StructType) -> Tuple[objects.StructType, Tuple[str, str, str]]: + def _bluetooth_sock( + self, sock: objects.StructType + ) -> Tuple[objects.StructType, Tuple[str, str, str]]: """Handles the AF_BLUETOOTH socket family Args: @@ -324,11 +352,17 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if bt_protocol == "HCI": if self._vmlinux.has_type("hci_pinfo"): pinfo = bt_sock.cast("hci_pinfo") - if pinfo.has_member("hdev") and self._vmlinux.has_type("hci_dev") \ - and pinfo.hdev.has_member("dev_name"): + if ( + pinfo.has_member("hdev") + and self._vmlinux.has_type("hci_dev") + and pinfo.hdev.has_member("dev_name") + ): src_addr = utility.array_to_string(pinfo.hdev.dev_name) else: - vollog.log(constants.LOGLEVEL_V, "Type definition for 'hci_pinfo' is not available in the symbols") + vollog.log( + constants.LOGLEVEL_V, + "Type definition for 'hci_pinfo' is not available in the symbols", + ) elif bt_protocol == "L2CAP": if self._vmlinux.has_type("l2cap_pinfo"): pinfo = bt_sock.cast("l2cap_pinfo") @@ -337,7 +371,10 @@ class SockHandlers(interfaces.configuration.VersionableInterface): src_port = pinfo.chan.sport dst_port = pinfo.chan.psm else: - vollog.log(constants.LOGLEVEL_V, "Type definition for 'l2cap_pinfo' is not available in the symbols") + vollog.log( + constants.LOGLEVEL_V, + "Type definition for 'l2cap_pinfo' is not available in the symbols", + ) elif bt_protocol == "RFCOMM": if self._vmlinux.has_type("rfcomm_pinfo"): pinfo = bt_sock.cast("rfcomm_pinfo") @@ -345,22 +382,31 @@ class SockHandlers(interfaces.configuration.VersionableInterface): dst_addr = bt_addr(pinfo.dst) src_port = pinfo.channel else: - vollog.log(constants.LOGLEVEL_V, "Type definition for 'rfcomm_pinfo' is not available in the symbols") + vollog.log( + constants.LOGLEVEL_V, + "Type definition for 'rfcomm_pinfo' is not available in the symbols", + ) elif bt_protocol == "SCO": if self._vmlinux.has_type("sco_pinfo"): pinfo = bt_sock.cast("sco_pinfo") src_addr = bt_addr(pinfo.src) dst_addr = bt_addr(pinfo.dst) else: - vollog.log(constants.LOGLEVEL_V, "Type definition for 'sco_pinfo' is not available in the symbols") + vollog.log( + constants.LOGLEVEL_V, + "Type definition for 'sco_pinfo' is not available in the symbols", + ) else: - vollog.log(constants.LOGLEVEL_V, "Unsupported bluetooth protocol '%s'", bt_protocol) + vollog.log( + constants.LOGLEVEL_V, "Unsupported bluetooth protocol '%s'", bt_protocol + ) state = bt_sock.get_state() sock_stat = src_addr, src_port, dst_addr, dst_port, state return bt_sock, sock_stat + class Sockstat(plugins.PluginInterface): """Lists all network connections for all processes.""" @@ -371,31 +417,48 @@ class Sockstat(plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.ModuleRequirement(name="kernel", description="Linux kernel", - architectures=["Intel32", "Intel64"]), - requirements.VersionRequirement(name="SockHandlers", component=SockHandlers, version=(1, 0, 0)), - requirements.PluginRequirement(name="lsof", plugin=lsof.Lsof, version=(1, 1, 0)), - requirements.VersionRequirement(name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0)), - requirements.BooleanRequirement(name="unix", - description=("Show UNIX domain Sockets only"), - default=False, - optional=True), - requirements.ListRequirement(name="pids", - description="Filter results by process IDs. " - "It takes the root PID namespace identifiers.", - element_type=int, - optional=True), - requirements.IntRequirement(name="netns", - description="Filter results by network namespace. " - "Otherwise, all of them are shown.", - optional=True), + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="SockHandlers", component=SockHandlers, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="lsof", plugin=lsof.Lsof, version=(1, 1, 0) + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) + ), + requirements.BooleanRequirement( + name="unix", + description=("Show UNIX domain Sockets only"), + default=False, + optional=True, + ), + requirements.ListRequirement( + name="pids", + description="Filter results by process IDs. " + "It takes the root PID namespace identifiers.", + element_type=int, + optional=True, + ), + requirements.IntRequirement( + name="netns", + description="Filter results by network namespace. " + "Otherwise, all of them are shown.", + optional=True, + ), ] @classmethod - def list_sockets(cls, - context: interfaces.context.ContextInterface, - symbol_table: str, - filter_func: Callable[[int], bool] = lambda _: False): + def list_sockets( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + filter_func: Callable[[int], bool] = lambda _: False, + ): """Returns every single socket descriptor Args: @@ -433,7 +496,9 @@ class Sockstat(plugins.PluginInterface): if not d_inode: continue - socket_alloc = linux.LinuxUtilities.container_of(d_inode, "socket_alloc", "vfs_inode", vmlinux) + socket_alloc = linux.LinuxUtilities.container_of( + d_inode, "socket_alloc", "vfs_inode", vmlinux + ) socket = socket_alloc.socket if not (socket and socket.sk): @@ -466,7 +531,9 @@ class Sockstat(plugins.PluginInterface): Returns: `sock_stat` and `protocol` formatted. """ - sock_stat = [NotAvailableValue() if field is None else str(field) for field in sock_stat] + sock_stat = [ + NotAvailableValue() if field is None else str(field) for field in sock_stat + ] if protocol is None: protocol = NotAvailableValue() @@ -494,26 +561,49 @@ class Sockstat(plugins.PluginInterface): extended information such as socket filters, bpf info, etc. """ filter_func = lsof.pslist.PsList.create_pid_filter(pids) - socket_generator = self.list_sockets(self.context, symbol_table, filter_func=filter_func) + socket_generator = self.list_sockets( + self.context, symbol_table, filter_func=filter_func + ) - for task, netns_id, fd_num, family, sock_type, protocol, sock_fields in socket_generator: + for ( + task, + netns_id, + fd_num, + family, + sock_type, + protocol, + sock_fields, + ) in socket_generator: if netns_id_arg and netns_id_arg != netns_id: continue sock, sock_stat, extended = sock_fields sock_stat, protocol = self._format_fields(sock_stat, protocol) - socket_filter_str = ",".join(f"{k}={v}" for k, v in extended.items()) if extended else NotAvailableValue() + socket_filter_str = ( + ",".join(f"{k}={v}" for k, v in extended.items()) + if extended + else NotAvailableValue() + ) - fields = (netns_id, task.pid, fd_num, format_hints.Hex(sock.vol.offset), - family, sock_type, protocol, *sock_stat, socket_filter_str) + fields = ( + netns_id, + task.pid, + fd_num, + format_hints.Hex(sock.vol.offset), + family, + sock_type, + protocol, + *sock_stat, + socket_filter_str, + ) yield (0, fields) def run(self): - pids = self.config.get('pids') - netns_id = self.config['netns'] - symbol_table = self.config['kernel'] + pids = self.config.get("pids") + netns_id = self.config["netns"] + symbol_table = self.config["kernel"] tree_grid_args = [ ("NetNS", int), diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 1d1419ae9..0d7cbb7e4 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -17,38 +17,38 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): super().__init__(*args, **kwargs) # Set-up Linux specific types - self.set_type_class('file', extensions.struct_file) - self.set_type_class('list_head', extensions.list_head) - self.set_type_class('mm_struct', extensions.mm_struct) - self.set_type_class('super_block', extensions.super_block) - self.set_type_class('task_struct', extensions.task_struct) - self.set_type_class('vm_area_struct', extensions.vm_area_struct) - self.set_type_class('qstr', extensions.qstr) - self.set_type_class('dentry', extensions.dentry) - self.set_type_class('fs_struct', extensions.fs_struct) - self.set_type_class('files_struct', extensions.files_struct) - self.set_type_class('kobject', extensions.kobject) + self.set_type_class("file", extensions.struct_file) + self.set_type_class("list_head", extensions.list_head) + self.set_type_class("mm_struct", extensions.mm_struct) + self.set_type_class("super_block", extensions.super_block) + self.set_type_class("task_struct", extensions.task_struct) + self.set_type_class("vm_area_struct", extensions.vm_area_struct) + self.set_type_class("qstr", extensions.qstr) + self.set_type_class("dentry", extensions.dentry) + self.set_type_class("fs_struct", extensions.fs_struct) + self.set_type_class("files_struct", extensions.files_struct) + self.set_type_class("kobject", extensions.kobject) # Might not exist in the current symbols - self.optional_set_type_class('module', extensions.module) + self.optional_set_type_class("module", extensions.module) # Mount - self.set_type_class('vfsmount', extensions.vfsmount) + self.set_type_class("vfsmount", extensions.vfsmount) # Might not exist in older kernels or the current symbols - self.optional_set_type_class('mount', extensions.mount) - self.optional_set_type_class('mnt_namespace', extensions.mnt_namespace) + self.optional_set_type_class("mount", extensions.mount) + self.optional_set_type_class("mnt_namespace", extensions.mnt_namespace) # Network - self.set_type_class('net', extensions.net) - self.set_type_class('socket', extensions.socket) - self.set_type_class('sock', extensions.sock) - self.set_type_class('inet_sock', extensions.inet_sock) - self.set_type_class('unix_sock', extensions.unix_sock) + self.set_type_class("net", extensions.net) + self.set_type_class("socket", extensions.socket) + self.set_type_class("sock", extensions.sock) + self.set_type_class("inet_sock", extensions.inet_sock) + self.set_type_class("unix_sock", extensions.unix_sock) # Might not exist in older kernels or the current symbols - self.optional_set_type_class('netlink_sock', extensions.netlink_sock) - self.optional_set_type_class('vsock_sock', extensions.vsock_sock) - self.optional_set_type_class('packet_sock', extensions.packet_sock) - self.optional_set_type_class('bt_sock', extensions.bt_sock) - self.optional_set_type_class('xdp_sock', extensions.xdp_sock) + self.optional_set_type_class("netlink_sock", extensions.netlink_sock) + self.optional_set_type_class("vsock_sock", extensions.vsock_sock) + self.optional_set_type_class("packet_sock", extensions.packet_sock) + self.optional_set_type_class("bt_sock", extensions.bt_sock) + self.optional_set_type_class("xdp_sock", extensions.xdp_sock) class LinuxUtilities(interfaces.configuration.VersionableInterface): @@ -322,7 +322,11 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): @classmethod def container_of( - cls, addr: int, type_name: str, member_name: str, vmlinux: interfaces.context.ModuleInterface + cls, + addr: int, + type_name: str, + member_name: str, + vmlinux: interfaces.context.ModuleInterface, ) -> Optional[interfaces.objects.ObjectInterface]: """Cast a member of a structure out to the containing structure. It mimicks the Linux kernel macro container_of() see include/linux.kernel.h @@ -343,4 +347,6 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): type_dec = vmlinux.get_type(type_name) member_offset = type_dec.relative_child_offset(member_name) container_addr = addr - member_offset - return vmlinux.object(object_type=type_name, offset=container_addr, absolute=True) + return vmlinux.object( + object_type=type_name, offset=container_addr, absolute=True + ) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0ea61a3bb..55f139730 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -812,6 +812,7 @@ class vfsmount(objects.StructType): def get_mnt_root(self): return self.mnt_root + class kobject(objects.StructType): def reference_count(self): refcnt = self.kref.refcount @@ -842,6 +843,7 @@ class mnt_namespace(objects.StructType): for mount in self.list.to_list(mnt_type, "mnt_list"): yield mount + class net(objects.StructType): def get_inode(self): if self.has_member("proc_inum"): @@ -851,12 +853,15 @@ class net(objects.StructType): else: raise AttributeError("Unable to find net_namespace inode") + class socket(objects.StructType): def _get_vol_kernel(self): symbol_table_arr = self.vol.type_name.split("!", 1) symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None - module_names = list(self._context.modules.get_modules_by_symbol_tables(symbol_table)) + module_names = list( + self._context.modules.get_modules_by_symbol_tables(symbol_table) + ) if not module_names: raise ValueError(f"No module using the symbol table {symbol_table}") @@ -870,7 +875,9 @@ class socket(objects.StructType): except ValueError: return 0 - socket_alloc = linux.LinuxUtilities.container_of(self.vol.offset, "socket_alloc", "socket", kernel) + socket_alloc = linux.LinuxUtilities.container_of( + self.vol.offset, "socket_alloc", "socket", kernel + ) vfs_inode = socket_alloc.vfs_inode return vfs_inode.i_ino @@ -880,6 +887,7 @@ class socket(objects.StructType): if 0 <= socket_state_idx < len(SOCKET_STATES): return SOCKET_STATES[socket_state_idx] + class sock(objects.StructType): def get_family(self): family_idx = self.__sk_common.skc_family @@ -905,6 +913,7 @@ class sock(objects.StructType): return self.sk_socket.get_state() + class unix_sock(objects.StructType): def get_name(self): if not self.addr: @@ -932,6 +941,7 @@ class unix_sock(objects.StructType): def get_inode(self): return self.sk.get_inode() + class inet_sock(objects.StructType): def get_family(self): family_idx = self.sk.__sk_common.skc_family @@ -966,7 +976,7 @@ class inet_sock(objects.StructType): def get_dst_port(self): sk_common = self.sk.__sk_common if hasattr(sk_common, "skc_portpair"): - dport_le = sk_common.skc_portpair & 0xffff + dport_le = sk_common.skc_portpair & 0xFFFF elif hasattr(self, "dport"): dport_le = self.dport elif hasattr(self, "inet_dport"): @@ -999,7 +1009,9 @@ class inet_sock(objects.StructType): try: addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) except exceptions.InvalidAddressException: - vollog.debug(f"Unable to read socket src address from {saddr.vol.offset:#x}") + vollog.debug( + f"Unable to read socket src address from {saddr.vol.offset:#x}" + ) return return socket_module.inet_ntop(family, addr_bytes) @@ -1028,11 +1040,14 @@ class inet_sock(objects.StructType): try: addr_bytes = parent_layer.read(daddr.vol.offset, addr_size) except exceptions.InvalidAddressException: - vollog.debug(f"Unable to read socket dst address from {daddr.vol.offset:#x}") + vollog.debug( + f"Unable to read socket dst address from {daddr.vol.offset:#x}" + ) return return socket_module.inet_ntop(family, addr_bytes) + class netlink_sock(objects.StructType): def get_protocol(self): protocol_idx = self.sk.sk_protocol @@ -1043,6 +1058,7 @@ class netlink_sock(objects.StructType): # Return the generic socket state return self.sk.sk_socket.get_state() + class vsock_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for vsocks @@ -1052,6 +1068,7 @@ class vsock_sock(objects.StructType): # Return the generic socket state return self.sk.sk_socket.get_state() + class packet_sock(objects.StructType): def get_protocol(self): eth_proto = socket_module.htons(self.num) @@ -1066,6 +1083,7 @@ class packet_sock(objects.StructType): # Return the generic socket state return self.sk.sk_socket.get_state() + class bt_sock(objects.StructType): def get_protocol(self): type_idx = self.sk.sk_protocol @@ -1077,6 +1095,7 @@ class bt_sock(objects.StructType): if 0 <= state_idx < len(BLUETOOTH_STATES): return BLUETOOTH_STATES[state_idx] + class xdp_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for xdp_sock From cd89e39ee053f3ca9b3a8f93628da2526accc99f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 6 Jan 2023 22:10:50 +0000 Subject: [PATCH 267/526] Layers: Fix QEMU layer cutting off the last byte of the config --- volatility3/framework/layers/qemu.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 829354987..501b8655e 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -117,13 +117,15 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): chunk_size = 4096 data = b"" for i in range( - base_layer.maximum_address, base_layer.minimum_address, -chunk_size + base_layer.maximum_address + 1, base_layer.minimum_address, -chunk_size ): - if i != base_layer.maximum_address: + # Since we're going backwards, we need to include one extra byte so the tail doesn't get chopped off + if i != base_layer.maximum_address + 1: data = (base_layer.read(i, chunk_size) + data).rstrip(b"\x00") if b"\x00" in data: last_null_byte = data.rfind(b"\x00") start_of_json = data.find(b"{", last_null_byte) + if start_of_json >= 0: data = data[start_of_json:] return json.loads(data) From 560569e03e5e8f7f6f29d6695b57745b11e2afee Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 6 Jan 2023 22:12:20 +0000 Subject: [PATCH 268/526] update linux.proc --dump so that vma object is not passed to dump func --- volatility3/framework/plugins/linux/proc.py | 41 ++++++++++++++------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index d8a17ae38..99c8761c1 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -95,7 +95,8 @@ class Maps(plugins.PluginInterface): cls, context: interfaces.context.ContextInterface, task: interfaces.objects.ObjectInterface, - vma: interfaces.objects.ObjectInterface, + vm_start: int, + vm_end: int, open_method: Type[interfaces.plugins.FileHandlerInterface], maxsize: int = MAXSIZE_DEFAULT, ) -> Optional[interfaces.plugins.FileHandlerInterface]: @@ -105,6 +106,8 @@ class Maps(plugins.PluginInterface): context: The context to retrieve required elements (layers, symbol tables) from task: an task_struct instance vma: The suspected VMA to extract (ObjectInterface) + vm_start: The start virtual address from the vma to dump + vm_end: The end virtual address from the vma to dump open_method: class to provide context manager for opening the file maxsize: Max size of VMA section (default MAXSIZE_DEFAULT) @@ -112,12 +115,6 @@ class Maps(plugins.PluginInterface): An open FileInterface object containing the complete data for the task or None in the case of failure """ pid = task.pid - try: - vm_start = vma.vm_start - vm_end = vma.vm_end - except AttributeError: - vollog.debug(f"Unable to find the vm_start and vm_end for pid {pid}") - return None try: proc_layer_name = task.add_process_layer() @@ -200,13 +197,31 @@ class Maps(plugins.PluginInterface): file_output = "Disabled" if self.config["dump"]: - file_handle = self.vma_dump( - self.context, task, vma, self.open, self.config["maxsize"] - ) file_output = "Error outputting file" - if file_handle: - file_handle.close() - file_output = file_handle.preferred_filename + try: + vm_start = vma.vm_start + vm_end = vma.vm_end + except AttributeError: + vollog.debug( + f"Unable to find the vm_start and vm_end for vma at {vma.vol.offset:#x} for pid {pid}" + ) + vm_start = None + vm_end = None + + if vm_start and vm_end: + # only attempt to dump the memory if we have vm_start and vm_end + file_handle = self.vma_dump( + self.context, + task, + vm_start, + vm_end, + self.open, + self.config["maxsize"], + ) + + if file_handle: + file_handle.close() + file_output = file_handle.preferred_filename yield ( 0, From a1eeecf6de088888c0509fb2165b7947e87dd868 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 9 Jan 2023 21:32:19 +0000 Subject: [PATCH 269/526] Layers: Fix uncaught exception in Elf layer --- volatility3/framework/layers/elf.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index bcafa9aed..a10d36592 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -119,4 +119,8 @@ class Elf64Stacker(interfaces.automagic.StackerLayerInterface): interfaces.configuration.path_join(new_name, "base_layer") ] = layer_name - return Elf64Layer(context, new_name, new_name) + try: + return Elf64Layer(context, new_name, new_name) + except ElfFormatException as excp: + vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") + return None From a60b91b6e18cb5979f2872ec61dc60af4a10faa1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 9 Jan 2023 21:33:07 +0000 Subject: [PATCH 270/526] Layers: Add in Xen CoreDump format support --- volatility3/framework/layers/xen.py | 171 +++++++++++++++++++ volatility3/framework/symbols/linux/xen.json | 115 +++++++++++++ 2 files changed, 286 insertions(+) create mode 100644 volatility3/framework/layers/xen.py create mode 100644 volatility3/framework/symbols/linux/xen.json diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py new file mode 100644 index 000000000..33050ea4b --- /dev/null +++ b/volatility3/framework/layers/xen.py @@ -0,0 +1,171 @@ +import logging +import struct +from typing import Optional + +from volatility3.framework import constants, interfaces, exceptions +from volatility3.framework.layers import elf +from volatility3.framework.symbols import intermed + +vollog = logging.getLogger(__name__) + + +class XenCoreDumpLayer(elf.Elf64Layer): + """A layer that supports the Xen Dump-Core format as documented at: https://xenbits.xen.org/docs/4.6-testing/misc/dump-core-format.txt""" + + _header_struct = struct.Struct(" None: + # Create a custom SymbolSpace + self._elf_table_name = intermed.IntermediateSymbolTable.create( + context, config_path, "linux", "elf" + ) + self._xen_table_name = intermed.IntermediateSymbolTable.create( + context, config_path, "linux", "xen" + ) + + super().__init__(context, config_path, name) + + def _load_segments(self) -> None: + """Load the segments from based on the PT_LOAD segments of the Elf64 format""" + ehdr = self.context.object( + self._elf_table_name + constants.BANG + "Elf64_Ehdr", + layer_name=self._base_layer, + offset=0, + ) + + segments = [] + segment_headers = [] + + for sindex in range(ehdr.e_shnum): + shdr = self.context.object( + self._elf_table_name + constants.BANG + "Elf64_Shdr", + layer_name=self._base_layer, + offset=ehdr.e_shoff + (sindex * ehdr.e_shentsize), + ) + + segment_headers.append(shdr) + + if sindex == ehdr.e_shstrndx: + segment_names = self.context.layers[self._base_layer].read( + shdr.sh_offset, shdr.sh_size + ) + segment_names = segment_names.split(b"\x00") + + if not segment_names: + raise elf.ElfFormatException("No segment names, not a Xen Core Dump") + + p2m_data = None + pfn_data = None + + for varname, pattern, outvar in [ + ("xen_p2m", b".xen_p2m", p2m_data), + ("xen_pfn", b".xen_pfn", pfn_data), + ]: + if pattern in segment_names: + hdr = segment_headers[segment_names.index(pattern)] + result = self.context.object( + self._xen_table_name + constants.BANG + varname, + layer_name=self._base_layer, + offset=hdr.sh_offset, + size=hdr.sh_size, + ) + result.entries.count = hdr.sh_size // result.entries.vol.subtype.size + outvar = result + + pages_hdr = segment_headers[segment_names.index(b".xen_pages")] + page_size = 0x1000 + + if pfn_data and not p2m_data: + for entry_index in range(len(pfn_data.entries)): + entry = pfn_data.entries[entry_index] + # TODO: Don't hardcode the maximum value here + if entry and entry != 0xFFFFFFFF: + segments.append( + ( + entry * page_size, + pages_hdr.sh_offset + (entry_index * page_size), + page_size, + page_size, + ) + ) + elif p2m_data and not pfn_data: + for entry_index in range(len(p2m_data.entries)): + entry = p2m_data.entries[entry_index] + # TODO: Don't hardcode the maximum value here + if entry.pfn != 0xFFFFFFFF: + segments.append( + ( + entry.pfn * page_size, + pages_hdr.sh_offset + (entry_index * page_size), + page_size, + page_size, + ) + ) + elif p2m_data and pfn_data: + raise elf.ElfFormatException( + self.name, f"Both P2M and PFN in Xen Core Dump" + ) + else: + raise elf.ElfFormatException( + self.name, f"Neither P2M nor PFN in Xen Core Dump" + ) + + if len(segments) == 0: + raise elf.ElfFormatException( + self.name, f"No ELF segments defined in {self._base_layer}" + ) + + self._segments = segments + + @classmethod + def _check_header( + cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0 + ) -> bool: + try: + header_data = base_layer.read(offset, cls._header_struct.size) + except exceptions.InvalidAddressException: + raise elf.ElfFormatException( + base_layer.name, + f"Offset 0x{offset:0x} does not exist within the base layer", + ) + (magic, elf_class, elf_data_encoding, elf_version) = cls._header_struct.unpack( + header_data + ) + if magic != cls.MAGIC: + raise elf.ElfFormatException( + base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}" + ) + if elf_class != cls.ELF_CLASS: + raise elf.ElfFormatException( + base_layer.name, f"ELF class is not 64-bit (2): {elf_class:d}" + ) + # Virtualbox uses an ELF version of 0, which isn't to specification, but is ok to deal with + return True + + +class XenCoreDumpStacker(elf.Elf64Stacker): + stack_order = 10 + + @classmethod + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: + try: + if not XenCoreDumpLayer._check_header(context.layers[layer_name]): + return None + except elf.ElfFormatException as excp: + vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") + return None + new_name = context.layers.free_layer_name("XenCoreDumpLayer") + context.config[ + interfaces.configuration.path_join(new_name, "base_layer") + ] = layer_name + + return XenCoreDumpLayer(context, new_name, new_name) diff --git a/volatility3/framework/symbols/linux/xen.json b/volatility3/framework/symbols/linux/xen.json new file mode 100644 index 000000000..8e843e728 --- /dev/null +++ b/volatility3/framework/symbols/linux/xen.json @@ -0,0 +1,115 @@ +{ + "symbols": { + }, + "user_types": { + "xen_p2m": { + "fields":{ + "entries": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned long long" + } + } + } + }, + "kind": "struct", + "size": 8 + }, + "xen_pfn":{ + "fields":{ + "entries": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned long long" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "xen_pfn_entry":{ + "fields":{ + "pfn":{ + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "gmfn":{ + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + } + }, + "kind": "struct", + "size": 16 + + } + }, + "enums": { + }, + "base_types": { + "unsigned char": { + "endian": "little", + "kind": "char", + "signed": false, + "size": 1 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + }, + "long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 4 + }, + "char": { + "endian": "little", + "kind": "char", + "signed": true, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "ikelos-by-hand", + "datetime": "2023-01-09T00:51:00" + }, + "format": "6.1.0" + } +} From d963aa3afb702980552653e40d3b090c57bf06bf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 10 Jan 2023 16:39:43 +0000 Subject: [PATCH 271/526] Layers: Fix trying to be too clever with Xen --- volatility3/framework/layers/xen.py | 49 +++++++++++++++++------------ 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index 33050ea4b..f7881a091 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -26,9 +26,23 @@ class XenCoreDumpLayer(elf.Elf64Layer): self._xen_table_name = intermed.IntermediateSymbolTable.create( context, config_path, "linux", "xen" ) + self._segment_headers = {} super().__init__(context, config_path, name) + def _extract_result_array( + self, varname: str, segment_index: int + ) -> interfaces.objects.ObjectInterface: + hdr = self._segment_headers[segment_index] + result = self.context.object( + self._xen_table_name + constants.BANG + varname, + layer_name=self._base_layer, + offset=hdr.sh_offset, + size=hdr.sh_size, + ) + result.entries.count = hdr.sh_size // result.entries.vol.subtype.size + return result + def _load_segments(self) -> None: """Load the segments from based on the PT_LOAD segments of the Elf64 format""" ehdr = self.context.object( @@ -38,7 +52,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): ) segments = [] - segment_headers = [] + self._segment_headers = [] for sindex in range(ehdr.e_shnum): shdr = self.context.object( @@ -47,7 +61,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): offset=ehdr.e_shoff + (sindex * ehdr.e_shentsize), ) - segment_headers.append(shdr) + self._segment_headers.append(shdr) if sindex == ehdr.e_shstrndx: segment_names = self.context.layers[self._base_layer].read( @@ -58,25 +72,20 @@ class XenCoreDumpLayer(elf.Elf64Layer): if not segment_names: raise elf.ElfFormatException("No segment names, not a Xen Core Dump") - p2m_data = None - pfn_data = None + try: + p2m_data = self._extract_result_array( + "xen_p2m", segment_names.index(b".xen_p2m") + ) + except ValueError: + p2m_data = None + try: + pfn_data = self._extract_result_array( + "xen_pfn", segment_names.index(b".xen_pfn") + ) + except ValueError: + pfn_data = None - for varname, pattern, outvar in [ - ("xen_p2m", b".xen_p2m", p2m_data), - ("xen_pfn", b".xen_pfn", pfn_data), - ]: - if pattern in segment_names: - hdr = segment_headers[segment_names.index(pattern)] - result = self.context.object( - self._xen_table_name + constants.BANG + varname, - layer_name=self._base_layer, - offset=hdr.sh_offset, - size=hdr.sh_size, - ) - result.entries.count = hdr.sh_size // result.entries.vol.subtype.size - outvar = result - - pages_hdr = segment_headers[segment_names.index(b".xen_pages")] + pages_hdr = self._segment_headers[segment_names.index(b".xen_pages")] page_size = 0x1000 if pfn_data and not p2m_data: From 1641a6e4c43aaf8ff8baf993319c61fb6163206c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 12 Jan 2023 19:58:51 +0000 Subject: [PATCH 272/526] Windows: Fix up black issue with vadinfo --- volatility3/framework/plugins/windows/vadinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 6cd453550..3214c7134 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -206,7 +206,7 @@ class VadInfo(interfaces.plugins.PluginInterface): if self.config.get("address", None) is not None: def filter_function(x: interfaces.objects.ObjectInterface) -> bool: - return not (x.get_start() <= self.config['address'] <= x.get_end()) + return not (x.get_start() <= self.config["address"] <= x.get_end()) filter_func = filter_function From 558b31cbdc9002dd8b0dacc2af8c5296ee3bcba5 Mon Sep 17 00:00:00 2001 From: cstation Date: Fri, 13 Jan 2023 11:58:11 +0100 Subject: [PATCH 273/526] Dump ELFs to file --- volatility3/framework/plugins/linux/elfs.py | 113 +++++++++++++++++- volatility3/framework/plugins/linux/pslist.py | 101 +++++++++++----- 2 files changed, 180 insertions(+), 34 deletions(-) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 822a69dd6..f56438f4f 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -4,20 +4,26 @@ """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" -from typing import List +import logging +from typing import List, Optional, Type -from volatility3.framework import renderers, interfaces +from volatility3.framework import constants, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.linux.extensions import elf from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) + class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -36,9 +42,95 @@ class Elfs(plugins.PluginInterface): element_type=int, optional=True, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed processes", + default=False, + optional=True, + ), ] + @classmethod + def elf_dump( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + elf_table_name: str, + vma: interfaces.objects.ObjectInterface, + task: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + ) -> Optional[interfaces.plugins.FileHandlerInterface]: + """Extracts an ELF as a FileHandlerInterface + Args: + context: the context to operate upon + layer_name: The name of the layer on which to operate + elf_table_name: the name for the symbol table containing the symbols for ELF-files + vma: virtual memory allocation of ELF + task: the task object whose memory should be output + open_method: class to provide context manager for opening the file + Returns: + An open FileHandlerInterface object containing the complete data for the task or None in the case of failure + """ + + proc_layer = context.layers[layer_name] + file_handle = None + + try: + elf_object = context.object( + elf_table_name + constants.BANG + "Elf", + offset=vma.vm_start, + layer_name=layer_name, + ) + + if not elf_object.is_valid(): + return None + + sections = {} + # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? + for phdr in elf_object.get_program_headers(): + if phdr.p_type != 1: # PT_LOAD = 1 + continue + + start = phdr.p_vaddr + size = phdr.p_memsz + end = start + size + + # Use complete memory pages for dumping + # If start isn't a multiple of 4096, stick to the highest multiple < start + # If end isn't a multiple of 4096, stick to the lowest multiple > end + if start % 4096: + start = start & ~0xFFF + + if end % 4096: + end = (end & ~0xFFF) + 4096 + + real_size = end - start + + if real_size < 0 or real_size > 100000000: + continue + + sections[start] = real_size + + elf_data = b"" + for section_start in sorted(sections.keys()): + read_size = sections[section_start] + + buf = proc_layer.read(vma.vm_start + section_start, read_size, pad=True) + elf_data = elf_data + buf + + file_handle = open_method( + f"pid.{task.pid}.{utility.array_to_string(task.comm)}.{vma.vm_start:#x}.dmp" + ) + file_handle.write(elf_data) + except Exception as e: + vollog.debug(f"Unable to dump ELF with pid {task.pid}: {e}") + + return file_handle + def _generator(self, tasks): + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "linux", "elf", class_types=elf.class_types + ) for task in tasks: proc_layer_name = task.add_process_layer() if not proc_layer_name: @@ -60,6 +152,21 @@ class Elfs(plugins.PluginInterface): path = vma.get_name(self.context, task) + file_output = "Disabled" + if self.config["dump"]: + file_handle = self.elf_dump( + self.context, + proc_layer_name, + elf_table_name, + vma, + task, + self.open, + ) + file_output = "Error outputting file" + if file_handle: + file_handle.close() + file_output = str(file_handle.preferred_filename) + yield ( 0, ( @@ -68,6 +175,7 @@ class Elfs(plugins.PluginInterface): format_hints.Hex(vma.vm_start), format_hints.Hex(vma.vm_end), path, + file_output, ), ) @@ -81,6 +189,7 @@ class Elfs(plugins.PluginInterface): ("Start", format_hints.Hex), ("End", format_hints.Hex), ("File Path", str), + ("File Output", str), ], self._generator( pslist.PsList.list_tasks( diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index af260a772..16e370b6e 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -1,12 +1,15 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Callable, Iterable, List, Any, Tuple +from typing import Any, Callable, Iterable, List -from volatility3.framework import renderers, interfaces +from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.linux.extensions import elf +from volatility3.plugins.linux import elfs class PsList(interfaces.plugins.PluginInterface): @@ -24,6 +27,9 @@ class PsList(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.PluginRequirement( + name="elfs", plugin=elfs.Elfs, version=(2, 0, 0) + ), requirements.ListRequirement( name="pid", description="Filter on specific process IDs", @@ -42,6 +48,12 @@ class PsList(interfaces.plugins.PluginInterface): optional=True, default=False, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed processes", + optional=True, + default=False, + ), ] @classmethod @@ -66,38 +78,12 @@ class PsList(interfaces.plugins.PluginInterface): else: return lambda _: False - def _get_task_fields( - self, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False - ) -> Tuple[int, int, int, str]: - """Extract the fields needed for the final output - - Args: - task: A task object from where to get the fields. - decorate_comm: If True, it decorates the comm string of - - User threads: in curly brackets, - - Kernel threads: in square brackets - Defaults to False. - Returns: - A tuple with the fields to show in the plugin output. - """ - pid = task.tgid - tid = task.pid - ppid = task.parent.tgid if task.parent else 0 - name = utility.array_to_string(task.comm) - if decorate_comm: - if task.is_kernel_thread: - name = f"[{name}]" - elif task.is_user_thread: - name = f"{{{name}}}" - - task_fields = (format_hints.Hex(task.vol.offset), pid, tid, ppid, name) - return task_fields - def _generator( self, pid_filter: Callable[[Any], bool], include_threads: bool = False, decorate_comm: bool = False, + dump: bool = False, ): """Generates the tasks list. @@ -110,14 +96,63 @@ class PsList(interfaces.plugins.PluginInterface): - User threads: in curly brackets, - Kernel threads: in square brackets Defaults to False. + dump: If True, the main executable of the process is written to a file + Defaults to False. Yields: Each rows """ for task in self.list_tasks( self.context, self.config["kernel"], pid_filter, include_threads ): - row = self._get_task_fields(task, decorate_comm) - yield (0, row) + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "linux", + "elf", + class_types=elf.class_types, + ) + file_output = "Disabled" + if dump: + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + continue + + # Find the vma that belongs to the main ELF of the process + file_output = "Error outputting file" + + for v in task.mm.get_mmap_iter(): + if v.vm_start == task.mm.start_code: + file_handle = elfs.Elfs.elf_dump( + self.context, + proc_layer_name, + elf_table_name, + v, + task, + self.open, + ) + if file_handle: + file_output = str(file_handle.preferred_filename) + file_handle.close() + break + + pid = task.tgid + tid = task.pid + ppid = task.parent.tgid if task.parent else 0 + name = utility.array_to_string(task.comm) + if decorate_comm: + if task.is_kernel_thread: + name = f"[{name}]" + elif task.is_user_thread: + name = f"{{{name}}}" + + yield 0, ( + format_hints.Hex(task.vol.offset), + pid, + tid, + ppid, + name, + file_output, + ) @classmethod def list_tasks( @@ -155,6 +190,7 @@ class PsList(interfaces.plugins.PluginInterface): pids = self.config.get("pid") include_threads = self.config.get("threads") decorate_comm = self.config.get("decorate_comm") + dump = self.config.get("dump") filter_func = self.create_pid_filter(pids) columns = [ @@ -163,7 +199,8 @@ class PsList(interfaces.plugins.PluginInterface): ("TID", int), ("PPID", int), ("COMM", str), + ("File output", str), ] return renderers.TreeGrid( - columns, self._generator(filter_func, include_threads, decorate_comm) + columns, self._generator(filter_func, include_threads, decorate_comm, dump) ) From bd291c43d5bc31409cb6ac920f6135c29b63c58f Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 16 Jan 2023 10:15:36 +0200 Subject: [PATCH 274/526] added debug message --- volatility3/framework/symbols/windows/pdbutil.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 1c3260fed..74fd0e4e8 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -54,6 +54,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): """ result = cls.get_guid_from_mz(context, layer_name, offset) if result is None: + vollog.debug(f"Could not get GUID for {hex(offset)}") return None guid, age, pdb_name = result if config_path is None: From 5aca49388eb83d20316d24add538dc79d8e34dee Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 16 Jan 2023 12:01:45 +0200 Subject: [PATCH 275/526] fix smearing --- volatility3/framework/plugins/windows/callbacks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 3bde95cf3..98e09d925 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -203,7 +203,10 @@ class Callbacks(interfaces.plugins.PluginInterface): callback_list = ntkrnlmp.object(object_type="_LIST_ENTRY", offset=symbol_offset) for callback in callback_list.to_list(full_type_name, "Link"): - yield "CmRegisterCallbackEx", callback.Function, f"Altitude: {callback.Altitude.String}" + altitude = "-" + with contextlib.suppress(exceptions.InvalidAddressException): + altitude = callback.Altitude.String + yield "CmRegisterCallbackEx", callback.Function, f"Altitude: {altitude}" @classmethod def list_registry_callbacks( From 5e33a98481e019083f1f4a41d8a145e2e31742e0 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 16 Jan 2023 12:39:08 +0200 Subject: [PATCH 276/526] change default value to None --- volatility3/framework/plugins/windows/callbacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 98e09d925..a1283d394 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -203,7 +203,7 @@ class Callbacks(interfaces.plugins.PluginInterface): callback_list = ntkrnlmp.object(object_type="_LIST_ENTRY", offset=symbol_offset) for callback in callback_list.to_list(full_type_name, "Link"): - altitude = "-" + altitude = None with contextlib.suppress(exceptions.InvalidAddressException): altitude = callback.Altitude.String yield "CmRegisterCallbackEx", callback.Function, f"Altitude: {altitude}" From 4cafc982f4972a8dace64589a0adcd7a02518d83 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 16 Jan 2023 10:47:09 +0000 Subject: [PATCH 277/526] Windows: Fix up callbacks typos and typing info --- volatility3/framework/plugins/windows/callbacks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index a1283d394..6898935fc 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -82,7 +82,7 @@ class Callbacks(interfaces.plugins.PluginInterface): context: The context to retrieve required elements (layers, symbol tables) from layer_name: The name of the layer on which to operate symbol_table: The name of the table containing the kernel symbols - callback_table_name: The nae of the table containing the callback symbols + callback_table_name: The name of the table containing the callback symbols Yields: A name, location and optional detail string @@ -182,7 +182,7 @@ class Callbacks(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, callback_table_name: str, - ) -> Iterable[Tuple[str, int, None]]: + ) -> Iterable[Tuple[str, int, Optional[str]]]: """ Lists all registry callbacks via the CallbackListHead. """ @@ -215,14 +215,14 @@ class Callbacks(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, callback_table_name: str, - ) -> Iterable[Tuple[str, int, None]]: + ) -> Iterable[Tuple[str, int, Optional[str]]]: """Lists all registry callbacks. Args: context: The context to retrieve required elements (layers, symbol tables) from layer_name: The name of the layer on which to operate symbol_table: The name of the table containing the kernel symbols - callback_table_name: The nae of the table containing the callback symbols + callback_table_name: The name of the table containing the callback symbols Yields: A name, location and optional detail string @@ -272,7 +272,7 @@ class Callbacks(interfaces.plugins.PluginInterface): context: The context to retrieve required elements (layers, symbol tables) from layer_name: The name of the layer on which to operate symbol_table: The name of the table containing the kernel symbols - callback_table_name: The nae of the table containing the callback symbols + callback_table_name: The name of the table containing the callback symbols Yields: A name, location and optional detail string @@ -330,7 +330,7 @@ class Callbacks(interfaces.plugins.PluginInterface): context: The context to retrieve required elements (layers, symbol tables) from layer_name: The name of the layer on which to operate symbol_table: The name of the table containing the kernel symbols - callback_table_name: The nae of the table containing the callback symbols + callback_table_name: The name of the table containing the callback symbols Yields: A name, location and optional detail string From f036acdeb8181a0cec6f4a73611080c8e14b5349 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 16 Jan 2023 13:02:20 +0200 Subject: [PATCH 278/526] missing import --- volatility3/framework/plugins/windows/callbacks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 6898935fc..56609a73a 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -3,6 +3,7 @@ # import logging +import contextlib from typing import List, Iterable, Tuple, Optional, Union from volatility3.framework import constants, exceptions, renderers, interfaces, symbols From e16887414e97cb9b2ef414a9ce15071fa9ac18f4 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Tue, 17 Jan 2023 10:15:03 +0200 Subject: [PATCH 279/526] add escapechar --- volatility3/cli/text_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 5df378d08..bb0f41ca3 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -241,7 +241,7 @@ class CSVRenderer(CLIRenderer): # Ignore the type because namedtuples don't realize they have accessible attributes header_list.append(f"{column.name}") - writer = csv.DictWriter(outfd, header_list, lineterminator="\n") + writer = csv.DictWriter(outfd, header_list, lineterminator="\n", escapechar='\\') writer.writeheader() def visitor(node: interfaces.renderers.TreeNode, accumulator): From 728e8b608ab59e4fa43353f4f2dea9f378b0e06a Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Tue, 17 Jan 2023 16:11:08 +0200 Subject: [PATCH 280/526] black reformat --- volatility3/cli/text_renderer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index bb0f41ca3..7e2167b6b 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -241,7 +241,9 @@ class CSVRenderer(CLIRenderer): # Ignore the type because namedtuples don't realize they have accessible attributes header_list.append(f"{column.name}") - writer = csv.DictWriter(outfd, header_list, lineterminator="\n", escapechar='\\') + writer = csv.DictWriter( + outfd, header_list, lineterminator="\n", escapechar="\\" + ) writer.writeheader() def visitor(node: interfaces.renderers.TreeNode, accumulator): From 1d4aa72c85c07f4bdad05de8abe1ecc8c053e760 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 27 Jan 2023 11:07:05 +0000 Subject: [PATCH 281/526] update linux.iomem with basic smear protection --- volatility3/framework/plugins/linux/iomem.py | 23 +++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index 6b0469d60..08a61bb46 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -48,15 +48,32 @@ class IOMem(interfaces.plugins.PluginInterface): Yields: Each row of output """ - # create the resource object vmlinux = context.modules[vmlinux_module_name] - resource = vmlinux.object("resource", resource_offset) + + # create the resource object with protection against memory smear + try: + resource = vmlinux.object("resource", resource_offset) + except exceptions.InvalidAddressException: + vollog.warning( + f"Unable to create resource object at {resource_offset:#x}. This resource, " + "its sibling, and any of it's childern and will be missing from the output." + ) + return None # extract the information required for this resource - name = utility.pointer_to_string(resource.name, 128) start = format_hints.Hex(resource.start) end = format_hints.Hex(resource.end) + # get name with protection against smear as following a pointer + try: + name = utility.pointer_to_string(resource.name, 128) + except exceptions.InvalidAddressException: + vollog.warning( + "Unable to follow pointer to name for resource object at {resource_offset:#x}, " + "replaced with UnreadableValue" + ) + name = renderers.UnreadableValue() + # mark this resource as seen in the seen set. Normally this should not be needed but will protect # against possible infinite loops. Warn the user if an infinite loop would have happened. if resource_offset in seen: From c2a1afb0ba8305b34bed11defdd21b9d83b76deb Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 Jan 2023 12:42:17 +0000 Subject: [PATCH 282/526] Core: Update codeql action to only run once a week --- .github/workflows/codeql.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 078af2abf..b9300251a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -12,11 +12,6 @@ name: "CodeQL" on: - push: - branches: [ "develop" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "develop" ] schedule: - cron: '16 8 * * 0' From 3297ba02e7cd2d24dfabac35c24996f36ca86891 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 Jan 2023 12:46:06 +0000 Subject: [PATCH 283/526] Core: Rather than scheduling it daily, only do it on commits --- .github/workflows/codeql.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b9300251a..fa9bd7ef6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -12,8 +12,13 @@ name: "CodeQL" on: - schedule: - - cron: '16 8 * * 0' + push: + branches: [ "develop" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "develop" ] +# schedule: +# - cron: '16 8 * * 0' jobs: analyze: From b375c6f71d0a90d8b3d632edbde9d25f9c72c266 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 1 Feb 2023 09:43:02 +0000 Subject: [PATCH 284/526] Update linux.iomem --- volatility3/framework/plugins/linux/iomem.py | 44 +++++++++++--------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index 08a61bb46..fddea4668 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -16,6 +16,7 @@ class IOMem(interfaces.plugins.PluginInterface): """Generates an output similar to /proc/iomem on a running system.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -60,10 +61,6 @@ class IOMem(interfaces.plugins.PluginInterface): ) return None - # extract the information required for this resource - start = format_hints.Hex(resource.start) - end = format_hints.Hex(resource.end) - # get name with protection against smear as following a pointer try: name = utility.pointer_to_string(resource.name, 128) @@ -87,7 +84,7 @@ class IOMem(interfaces.plugins.PluginInterface): seen.add(resource_offset) # yield information on this resource - yield depth, (name, start, end) + yield depth, (name, resource.start, resource.end) # process child resource if this exists if resource.child != 0: @@ -123,17 +120,32 @@ class IOMem(interfaces.plugins.PluginInterface): vmlinux_module_name = self.config["kernel"] vmlinux = self.context.modules[vmlinux_module_name] - # check that the iomem_resource symbol exists - # normally exported in /kernel/resource.c + # get the address for the iomem_resource try: iomem_root_offset = vmlinux.get_absolute_symbol_address("iomem_resource") except exceptions.SymbolError: iomem_root_offset = None - # error if 'iomem_resource' is not found - if not iomem_root_offset: + # only continue if iomem_root address was located + if iomem_root_offset is not None: + + # recursively parse the resources starting from the root resource at 'iomem_resource' + for depth, (name, start, end) in self.parse_resource( + self.context, vmlinux_module_name, iomem_root_offset + ): + # use format_hints to format start and end addresses for the renderers + yield depth, (name, format_hints.Hex(start), format_hints.Hex(end)) + + def run(self): + # get the kernel module from the current context + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] + + # check that the iomem_resource symbol exists + # normally exported in /kernel/resource.c + if not vmlinux.has_symbol("iomem_resource"): raise TypeError( - "This plugin requires the iomem_resource structure. This structure is not present in the supplied symbol table. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." + "This plugin requires the iomem_resource symbol. This symbol is not present in the supplied symbol table. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." ) # error if type 'resource' is not found @@ -142,15 +154,9 @@ class IOMem(interfaces.plugins.PluginInterface): "This plugin requires the resource type. This type is not present in the supplied symbol table. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." ) - # recursively parse the resources starting from the root resource at 'iomem_resource' - yield from self.parse_resource( - self.context, vmlinux_module_name, iomem_root_offset - ) - - def run(self): columns = [ - ("NAME", str), - ("START", format_hints.Hex), - ("END", format_hints.Hex), + ("Name", str), + ("Start", format_hints.Hex), + ("End", format_hints.Hex), ] return renderers.TreeGrid(columns, self._generator()) From 3b1e4ce0e2635db5ac860cf30bb5c5524d55632e Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 1 Feb 2023 11:27:44 +0000 Subject: [PATCH 285/526] Update linux.vmayarascan to pull requirements from the generic yarascan plugin --- .../framework/plugins/linux/vmayarascan.py | 79 ++++++++++--------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index c7a48cc14..f0d42f6e3 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # @@ -15,68 +15,71 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [ - requirements.ModuleRequirement( - name="kernel", - description="Linux kernel", - architectures=["Intel32", "Intel64"], - ), - requirements.BooleanRequirement( - name="wide", - description="Match wide (unicode) strings", - default=False, - optional=True, - ), - requirements.StringRequirement( - name="yara_rules", description="Yara rules (as a string)", optional=True - ), - requirements.URIRequirement( - name="yara_file", description="Yara rules (as a file)", optional=True - ), - # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code - # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later - requirements.URIRequirement( - name="yara_compiled_file", - description="Yara compiled rules (as a file)", - optional=True, - ), - requirements.IntRequirement( - name="max_size", - default=0x40000000, - description="Set the maximum size (default is 1GB)", - optional=True, - ), - requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) - ), - requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) - ), + # create a list of requirements for vmayarascan + vmayarascan_requirements = [ requirements.ListRequirement( name="pid", element_type=int, description="Process IDs to include (all other processes are excluded)", optional=True, ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(1, 1, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), ] + # get base yarascan requirements + yarascan_requirements = yarascan.YaraScan.get_requirements() + + # remove TranslationLayerRequirement from the base yarascan requirements + # if this is not removed automagic will not find both the TranslationLayerRequirement + # for YaraScan and the ModuleRequirement for VmaYaraScan + yarascan_requirements = [ + requirement + for requirement in yarascan_requirements + if not isinstance(requirement, requirements.TranslationLayerRequirement) + ] + + # return the combined requirements + return yarascan_requirements + vmayarascan_requirements + def _generator(self): + # use yarascan to parse the yara options provided and create the rules rules = yarascan.YaraScan.process_yara_options(dict(self.config)) + # filter based on the pid option if provided filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for task in pslist.PsList.list_tasks( context=self.context, vmlinux_module_name=self.config["kernel"], filter_func=filter_func, ): + + # attempt to create a process layer for each task and skip those + # that cannot (e.g. kernel threads) proc_layer_name = task.add_process_layer() if not proc_layer_name: continue + # get the proc_layer object from the context proc_layer = self.context.layers[proc_layer_name] + + # scan the process layer with the yarascanner for offset, rule_name, name, value in proc_layer.scan( context=self.context, scanner=yarascan.YaraScanner(rules=rules), From 43a17384c32da91d711f9f186a3036bab43a3954 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 3 Feb 2023 00:35:49 +0000 Subject: [PATCH 286/526] Core: Update to black 23.1.0 which removes many blank lines and parentheses --- volatility3/cli/__init__.py | 2 +- volatility3/cli/text_renderer.py | 2 +- volatility3/cli/volargparse.py | 1 - volatility3/framework/automagic/construct_layers.py | 1 - volatility3/framework/automagic/mac.py | 1 - volatility3/framework/automagic/symbol_cache.py | 3 ++- volatility3/framework/automagic/symbol_finder.py | 4 ++-- volatility3/framework/interfaces/layers.py | 5 ++--- volatility3/framework/layers/avml.py | 2 +- volatility3/framework/layers/crash.py | 1 - volatility3/framework/layers/intel.py | 2 +- volatility3/framework/layers/leechcore.py | 1 - volatility3/framework/layers/linear.py | 4 ++-- volatility3/framework/layers/registry.py | 1 - volatility3/framework/plugins/linux/check_afinfo.py | 4 +--- volatility3/framework/plugins/linux/check_creds.py | 3 +-- volatility3/framework/plugins/linux/check_modules.py | 2 -- volatility3/framework/plugins/linux/check_syscall.py | 7 +++---- volatility3/framework/plugins/linux/lsmod.py | 1 - volatility3/framework/plugins/linux/lsof.py | 1 - volatility3/framework/plugins/linux/mountinfo.py | 1 - volatility3/framework/plugins/linux/tty_check.py | 2 -- volatility3/framework/plugins/mac/check_syscall.py | 2 +- volatility3/framework/plugins/mac/kauth_listeners.py | 1 - volatility3/framework/plugins/mac/kauth_scopes.py | 1 - volatility3/framework/plugins/mac/kevents.py | 1 - volatility3/framework/plugins/mac/list_files.py | 3 --- volatility3/framework/plugins/mac/lsmod.py | 2 -- volatility3/framework/plugins/mac/netstat.py | 2 -- volatility3/framework/plugins/mac/pslist.py | 1 - volatility3/framework/plugins/timeliner.py | 2 +- volatility3/framework/plugins/windows/bigpools.py | 1 - volatility3/framework/plugins/windows/cachedump.py | 1 - volatility3/framework/plugins/windows/callbacks.py | 5 ----- volatility3/framework/plugins/windows/devicetree.py | 2 +- volatility3/framework/plugins/windows/dlllist.py | 2 -- volatility3/framework/plugins/windows/driverirp.py | 2 -- volatility3/framework/plugins/windows/drivermodule.py | 1 - volatility3/framework/plugins/windows/driverscan.py | 1 - volatility3/framework/plugins/windows/dumpfiles.py | 1 - volatility3/framework/plugins/windows/envars.py | 1 - volatility3/framework/plugins/windows/filescan.py | 2 -- volatility3/framework/plugins/windows/getservicesids.py | 1 - volatility3/framework/plugins/windows/getsids.py | 3 --- volatility3/framework/plugins/windows/handles.py | 8 ++------ volatility3/framework/plugins/windows/hashdump.py | 1 - volatility3/framework/plugins/windows/info.py | 3 --- volatility3/framework/plugins/windows/joblinks.py | 2 +- volatility3/framework/plugins/windows/ldrmodules.py | 1 - volatility3/framework/plugins/windows/lsadump.py | 6 ------ volatility3/framework/plugins/windows/malfind.py | 1 - volatility3/framework/plugins/windows/mbrscan.py | 2 -- volatility3/framework/plugins/windows/modscan.py | 3 --- volatility3/framework/plugins/windows/modules.py | 1 - volatility3/framework/plugins/windows/mutantscan.py | 2 -- volatility3/framework/plugins/windows/netscan.py | 2 -- volatility3/framework/plugins/windows/netstat.py | 2 -- volatility3/framework/plugins/windows/poolscanner.py | 2 -- volatility3/framework/plugins/windows/privileges.py | 2 -- volatility3/framework/plugins/windows/pslist.py | 1 - volatility3/framework/plugins/windows/psscan.py | 2 -- .../framework/plugins/windows/registry/hivelist.py | 1 - .../framework/plugins/windows/registry/hivescan.py | 1 - .../framework/plugins/windows/registry/printkey.py | 4 +--- .../framework/plugins/windows/registry/userassist.py | 2 -- volatility3/framework/plugins/windows/sessions.py | 2 -- .../framework/plugins/windows/skeleton_key_check.py | 2 -- volatility3/framework/plugins/windows/ssdt.py | 3 --- volatility3/framework/plugins/windows/svcscan.py | 2 -- volatility3/framework/plugins/windows/symlinkscan.py | 2 -- volatility3/framework/plugins/windows/vadinfo.py | 1 - volatility3/framework/plugins/windows/verinfo.py | 1 - volatility3/framework/plugins/windows/virtmap.py | 2 +- volatility3/framework/renderers/__init__.py | 4 ++-- volatility3/framework/renderers/format_hints.py | 1 - volatility3/framework/symbols/__init__.py | 2 +- volatility3/framework/symbols/linux/__init__.py | 4 +--- .../framework/symbols/linux/extensions/__init__.py | 3 --- volatility3/framework/symbols/linux/extensions/elf.py | 1 - volatility3/framework/symbols/mac/__init__.py | 4 ---- volatility3/framework/symbols/mac/extensions/__init__.py | 4 ++-- .../framework/symbols/windows/extensions/__init__.py | 8 -------- .../framework/symbols/windows/extensions/network.py | 4 ---- volatility3/framework/symbols/windows/extensions/pe.py | 1 - volatility3/framework/symbols/windows/pdbutil.py | 4 +--- volatility3/plugins/windows/registry/certificates.py | 1 - 86 files changed, 32 insertions(+), 162 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index fb124a3c9..336902d50 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -443,7 +443,7 @@ class CommandLine: # Construct and run the plugin if constructed: renderers[args.renderer]().render(constructed.run()) - except (exceptions.VolatilityException) as excp: + except exceptions.VolatilityException as excp: self.process_exceptions(excp) @classmethod diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 5df378d08..4df04e2a3 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -346,7 +346,7 @@ class PrettyTextRenderer(CLIRenderer): column_titles = [""] + [column.name for column in grid.columns] outfd.write(format_string.format(*column_titles)) - for (depth, line) in final_output: + for depth, line in final_output: nums_line = max([len(line[column]) for column in line]) for column in line: line[column] = line[column] + ([""] * (nums_line - len(line[column]))) diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index dd89a64fd..3048a0885 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -31,7 +31,6 @@ class HelpfulSubparserAction(argparse._SubParsersAction): values: Union[str, Sequence[Any], None], option_string: Optional[str] = None, ) -> None: - parser_name = "" arg_strings = [] # type: List[str] if values is not None: diff --git a/volatility3/framework/automagic/construct_layers.py b/volatility3/framework/automagic/construct_layers.py index ceed2fe50..239f0cfb6 100644 --- a/volatility3/framework/automagic/construct_layers.py +++ b/volatility3/framework/automagic/construct_layers.py @@ -36,7 +36,6 @@ class ConstructionMagic(interfaces.automagic.AutomagicInterface): progress_callback=None, optional=False, ) -> List[str]: - # Make sure we import the layers, so they can reconstructed framework.import_files(sys.modules["volatility3.framework.layers"]) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 3ca0b4ea2..aa75fbc3d 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -251,7 +251,6 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): context=context, progress_callback=progress_callback, ): - banner = context.layers[layer_name].read(offset, 128) idx = banner.find(b"\x00") diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 44a76506c..63c6fc7fa 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -161,7 +161,8 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns ISF statistics based on the location Returns: - A tuple of base_types, types, enums, symbols, or None is location not found""" + A tuple of base_types, types, enums, symbols, or None is location not found + """ def get_hash(self, location: str) -> Optional[str]: """Returns the hash of the JSON from within a location ISF""" diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 0143e74b1..f30dff456 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -82,13 +82,13 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): shortcut=False, ) - for (sub_path, requirement) in self._requirements: + for sub_path, requirement in self._requirements: parent_path = interfaces.configuration.parent_path(sub_path) if isinstance( requirement, requirements.SymbolTableRequirement ) and requirement.unsatisfied(context, parent_path): - for (tl_sub_path, tl_requirement) in self._requirements: + for tl_sub_path, tl_requirement in self._requirements: tl_parent_path = interfaces.configuration.parent_path(tl_sub_path) # Find the TranslationLayer sibling to the SymbolTableRequirement if ( diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index a3c31a953..68592f8cb 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -294,7 +294,7 @@ class DataLayerInterface( sections.""" result: List[Tuple[int, int]] = [] position = 0 - for (start, length) in sorted(sections): + for start, length in sorted(sections): if result and start <= position: initial_start, _ = result.pop() result.append((initial_start, (start + length) - initial_start)) @@ -375,7 +375,6 @@ class DataLayerInterface( def _scan_metric( self, _scanner: "ScannerInterface", sections: List[Tuple[int, int]] ) -> Callable[[int], float]: - if not sections: raise ValueError("Sections have no size, nothing to scan") last_section, last_length = sections[-1] @@ -551,7 +550,7 @@ class TranslationLayerInterface(DataLayerInterface, metaclass=ABCMeta): scanner.chunk_size + scanner.overlap DataLayers by default are assumed to have no holes """ - for (section_start, section_length) in sections: + for section_start, section_length in sections: output: List[Tuple[str, int, int]] = [] # Hold the offsets of each chunk (including how much has been filled) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index b12fdd01c..66f3f0e4f 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -73,7 +73,7 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer): ) segments, consumed = self._read_snappy_frames(chunk_data, end - start) # The returned segments are accurate the chunk_data that was passed in, but needs shifting - for (thing, mapped_offset, size, mapped_size, compressed) in segments: + for thing, mapped_offset, size, mapped_size, compressed in segments: self._segments.append( ( thing + start, diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 64166cfba..8efd4f7c7 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -39,7 +39,6 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): def __init__( self, context: interfaces.context.ContextInterface, config_path: str, name: str ) -> None: - # Construct these so we can use self.config self._context = context self._config_path = config_path diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index ecfb6bf11..478eb168f 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -173,7 +173,7 @@ class Intel(linear.LinearlyMappedLayer): ) # Run through the offset in various chunks - for (name, size, large_page) in self._structure: + for name, size, large_page in self._structure: # Check we're valid if not self._page_is_valid(entry): raise exceptions.PagedInvalidAddressException( diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index 73700dd3c..542fd6ca2 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -91,7 +91,6 @@ if HAS_LEECHCORE: chunk_size = size output = [] for entry in self.handle.memmap: - if ( entry["base"] + entry["size"] <= chunk_start or entry["base"] >= chunk_start + chunk_size diff --git a/volatility3/framework/layers/linear.py b/volatility3/framework/layers/linear.py index 19203eb66..47170df7b 100644 --- a/volatility3/framework/layers/linear.py +++ b/volatility3/framework/layers/linear.py @@ -42,7 +42,7 @@ class LinearlyMappedLayer(interfaces.layers.TranslationLayerInterface): length size.""" current_offset = offset output: List[bytes] = [] - for (offset, _, mapped_offset, mapped_length, layer) in self.mapping( + for offset, _, mapped_offset, mapped_length, layer in self.mapping( offset, length, ignore_errors=pad ): if not pad and offset > current_offset: @@ -71,7 +71,7 @@ class LinearlyMappedLayer(interfaces.layers.TranslationLayerInterface): underlying mapping.""" current_offset = offset length = len(value) - for (offset, _, mapped_offset, length, layer) in self.mapping(offset, length): + for offset, _, mapped_offset, length, layer in self.mapping(offset, length): if offset > current_offset: raise exceptions.InvalidAddressException( self.name, diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 660e0a299..cc8ce1f4c 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -269,7 +269,6 @@ class RegistryHive(linear.LinearlyMappedLayer): def mapping( self, offset: int, length: int, ignore_errors: bool = False ) -> Iterable[Tuple[int, int, int, int, str]]: - if length < 0: raise ValueError("Mapping length of RegistryHive must be positive or zero") diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index c177ee642..90e714eaa 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -68,7 +68,6 @@ class Check_afinfo(plugins.PluginInterface): yield var_name, "show", var.seq_show def _generator(self): - vmlinux = self.context.modules[self.config["kernel"]] op_members = vmlinux.get_type("file_operations").members @@ -86,7 +85,7 @@ class Check_afinfo(plugins.PluginInterface): ) protocols = [tcp, udp] - for (struct_type, global_vars) in protocols: + for struct_type, global_vars in protocols: for global_var_name in global_vars: # this will lookup fail for the IPv6 protocols on kernels without IPv6 support try: @@ -104,7 +103,6 @@ class Check_afinfo(plugins.PluginInterface): yield 0, (name, member, format_hints.Hex(address)) def run(self): - return renderers.TreeGrid( [ ("Symbol Name", str), diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 6d4e2bc8a..ab6ee4935 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -46,7 +46,6 @@ class Check_creds(interfaces.plugins.PluginInterface): tasks = pslist.PsList.list_tasks(self.context, vmlinux.name) for task in tasks: - cred_addr = task.cred.dereference().vol.offset if cred_addr not in creds: @@ -54,7 +53,7 @@ class Check_creds(interfaces.plugins.PluginInterface): creds[cred_addr].append(task.pid) - for (_, pids) in creds.items(): + for _, pids in creds.items(): if len(pids) > 1: pid_str = "" for pid in pids: diff --git a/volatility3/framework/plugins/linux/check_modules.py b/volatility3/framework/plugins/linux/check_modules.py index 766858888..9b3594c5e 100644 --- a/volatility3/framework/plugins/linux/check_modules.py +++ b/volatility3/framework/plugins/linux/check_modules.py @@ -37,7 +37,6 @@ class Check_modules(plugins.PluginInterface): def get_kset_modules( cls, context: interfaces.context.ContextInterface, vmlinux_name: str ): - vmlinux = context.modules[vmlinux_name] try: @@ -57,7 +56,6 @@ class Check_modules(plugins.PluginInterface): for kobj in module_kset.list.to_list( vmlinux.symbol_table_name + constants.BANG + "kobject", "entry" ): - mod_kobj = vmlinux.object( object_type="module_kobject", offset=kobj.vol.offset - kobj_off, diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 6b11038ec..b1d2919f9 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -110,7 +110,7 @@ class Check_syscall(plugins.PluginInterface): vmlinux = self.context.modules[self.config["kernel"]] data = self.context.layers.read(vmlinux.layer_name, func_addr, 6) - for (address, size, mnemonic, op_str) in md.disasm_lite(data, func_addr): + for address, size, mnemonic, op_str in md.disasm_lite(data, func_addr): if mnemonic == "CMP": table_size = int(op_str.split(",")[1].strip()) & 0xFFFF break @@ -161,7 +161,7 @@ class Check_syscall(plugins.PluginInterface): ia32_info = self._get_table_info(vmlinux, "ia32_sys_call_table", ptr_sz) tables.append(("32bit", ia32_info)) - for (table_name, (tableaddr, tblsz)) in tables: + for table_name, (tableaddr, tblsz) in tables: table = vmlinux.object( object_type="array", subtype=vmlinux.get_type("pointer"), @@ -169,7 +169,7 @@ class Check_syscall(plugins.PluginInterface): count=tblsz, ) - for (i, call_addr) in enumerate(table): + for i, call_addr in enumerate(table): if not call_addr: continue @@ -196,7 +196,6 @@ class Check_syscall(plugins.PluginInterface): ) def run(self): - return renderers.TreeGrid( [ ("Table Address", format_hints.Hex), diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index 1c1e094c3..a65b0d00b 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -60,7 +60,6 @@ class Lsmod(plugins.PluginInterface): def _generator(self): try: for module in self.list_modules(self.context, self.config["kernel"]): - mod_size = module.get_init_size() + module.get_core_size() mod_name = utility.array_to_string(module.name) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 62bade1f1..d970ad8a9 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -52,7 +52,6 @@ class Lsof(plugins.PluginInterface): symbol_table: str, filter_func: Callable[[int], bool] = lambda _: False, ): - linuxutils_symbol_table = None # type: ignore for task in pslist.PsList.list_tasks(context, symbol_table, filter_func): if linuxutils_symbol_table is None: diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index c849d51c6..ebd6e55a0 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -203,7 +203,6 @@ class MountInfo(plugins.PluginInterface): mount_format: bool, per_namespace: bool, ) -> Iterable[Tuple[int, Tuple]]: - for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, per_namespace): if mnt_ns_ids and mnt_ns_id not in mnt_ns_ids: continue diff --git a/volatility3/framework/plugins/linux/tty_check.py b/volatility3/framework/plugins/linux/tty_check.py index dcc9f3e06..45238ef8c 100644 --- a/volatility3/framework/plugins/linux/tty_check.py +++ b/volatility3/framework/plugins/linux/tty_check.py @@ -61,7 +61,6 @@ class tty_check(plugins.PluginInterface): for tty in tty_drivers.to_list( vmlinux.symbol_table_name + constants.BANG + "tty_driver", "tty_drivers" ): - try: ttys = utility.array_of_pointers( tty.ttys.dereference(), @@ -73,7 +72,6 @@ class tty_check(plugins.PluginInterface): continue for tty_dev in ttys: - if tty_dev == 0: continue diff --git a/volatility3/framework/plugins/mac/check_syscall.py b/volatility3/framework/plugins/mac/check_syscall.py index a7a32e9ab..5c22e6463 100644 --- a/volatility3/framework/plugins/mac/check_syscall.py +++ b/volatility3/framework/plugins/mac/check_syscall.py @@ -47,7 +47,7 @@ class Check_syscall(plugins.PluginInterface): table = kernel.object_from_symbol(symbol_name="sysent") - for (i, ent) in enumerate(table): + for i, ent in enumerate(table): try: call_addr = ent.sy_call.dereference().vol.offset except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/mac/kauth_listeners.py b/volatility3/framework/plugins/mac/kauth_listeners.py index 0b945a8fd..ed43bfb42 100644 --- a/volatility3/framework/plugins/mac/kauth_listeners.py +++ b/volatility3/framework/plugins/mac/kauth_listeners.py @@ -49,7 +49,6 @@ class Kauth_listeners(interfaces.plugins.PluginInterface): for scope in kauth_scopes.Kauth_scopes.list_kauth_scopes( self.context, self.config["kernel"] ): - scope_name = utility.pointer_to_string(scope.ks_identifier, 128) for listener in scope.get_listeners(): diff --git a/volatility3/framework/plugins/mac/kauth_scopes.py b/volatility3/framework/plugins/mac/kauth_scopes.py index bfd7216a8..afb320a07 100644 --- a/volatility3/framework/plugins/mac/kauth_scopes.py +++ b/volatility3/framework/plugins/mac/kauth_scopes.py @@ -65,7 +65,6 @@ class Kauth_scopes(interfaces.plugins.PluginInterface): ) for scope in self.list_kauth_scopes(self.context, self.config["kernel"]): - callback = scope.ks_callback if callback == 0: continue diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 74c5f6037..3b996bc0a 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -184,7 +184,6 @@ class Kevents(interfaces.plugins.PluginInterface): for task_name, pid, kn in self.list_kernel_events( self.context, self.config["kernel"], filter_func=filter_func ): - filter_index = kn.kn_kevent.filter * -1 if filter_index in self.event_types: filter_name = self.event_types[filter_index] diff --git a/volatility3/framework/plugins/mac/list_files.py b/volatility3/framework/plugins/mac/list_files.py index ede0fa32b..c18b0b7a2 100644 --- a/volatility3/framework/plugins/mac/list_files.py +++ b/volatility3/framework/plugins/mac/list_files.py @@ -137,7 +137,6 @@ class List_Files(plugins.PluginInterface): def _walk_mounts( cls, context: interfaces.context.ContextInterface, kernel_module_name: str ) -> Iterable[interfaces.objects.ObjectInterface]: - loop_vnodes = {} # iterate each vnode source from each mount @@ -186,7 +185,6 @@ class List_Files(plugins.PluginInterface): def list_files( cls, context: interfaces.context.ContextInterface, kernel_module_name: str ) -> Iterable[interfaces.objects.ObjectInterface]: - vnodes = cls._walk_mounts(context, kernel_module_name) for voff, (vnode_name, parent_offset, vnode) in vnodes.items(): @@ -196,7 +194,6 @@ class List_Files(plugins.PluginInterface): def _generator(self): for vnode, full_path in self.list_files(self.context, self.config["kernel"]): - yield (0, (format_hints.Hex(vnode.vol.offset), full_path)) def run(self): diff --git a/volatility3/framework/plugins/mac/lsmod.py b/volatility3/framework/plugins/mac/lsmod.py index 2cdd5e3de..2979e374b 100644 --- a/volatility3/framework/plugins/mac/lsmod.py +++ b/volatility3/framework/plugins/mac/lsmod.py @@ -63,7 +63,6 @@ class Lsmod(plugins.PluginInterface): seen: Set = set() while kmod != 0 and kmod not in seen and len(seen) < 1024: - kmod_obj = kmod.dereference() if not kernel_layer.is_valid(kmod_obj.vol.offset, kmod_obj.vol.size): @@ -81,7 +80,6 @@ class Lsmod(plugins.PluginInterface): def _generator(self): for module in self.list_modules(self.context, self.config["kernel"]): - mod_name = utility.array_to_string(module.name) mod_size = module.size diff --git a/volatility3/framework/plugins/mac/netstat.py b/volatility3/framework/plugins/mac/netstat.py index 581a9c67f..76bba25f6 100644 --- a/volatility3/framework/plugins/mac/netstat.py +++ b/volatility3/framework/plugins/mac/netstat.py @@ -68,7 +68,6 @@ class Netstat(plugins.PluginInterface): # This is hardcoded, since a change in the default method would change the expected results list_tasks = pslist.PsList.get_list_tasks(pslist.PsList.pslist_methods[0]) for task in list_tasks(context, kernel_module_name, filter_func): - task_name = utility.array_to_string(task.p_comm) pid = task.p_pid @@ -101,7 +100,6 @@ class Netstat(plugins.PluginInterface): for task_name, pid, socket in self.list_sockets( self.context, self.config["kernel"], filter_func=filter_func ): - family = socket.get_family() if family == 1: diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index c2ae71e7e..1d97216bf 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -83,7 +83,6 @@ class PsList(interfaces.plugins.PluginInterface): @classmethod def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[int], bool]: - filter_func = lambda _: False # FIXME: mypy #4973 or #2608 pid_list = pid_list or [] diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 0776b6cc8..d1c1c0f70 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -136,7 +136,7 @@ class Timeliner(interfaces.plugins.PluginInterface): ) try: vollog.log(logging.INFO, f"Running {plugin_name}") - for (item, timestamp_type, timestamp) in plugin.generate_timeline(): + for item, timestamp_type, timestamp in plugin.generate_timeline(): times = self.timeline.get((plugin_name, item), {}) if times.get(timestamp_type, None) is not None: vollog.debug( diff --git a/volatility3/framework/plugins/windows/bigpools.py b/volatility3/framework/plugins/windows/bigpools.py index 1a51a0b81..393c2a417 100644 --- a/volatility3/framework/plugins/windows/bigpools.py +++ b/volatility3/framework/plugins/windows/bigpools.py @@ -141,7 +141,6 @@ class BigPools(interfaces.plugins.PluginInterface): tags=tags, show_free=self.config.get("show-free"), ): - num_bytes = big_pool.get_number_of_bytes() if not isinstance(num_bytes, interfaces.renderers.BaseAbsentValue): num_bytes = format_hints.Hex(num_bytes) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index 7d3093ed7..a9b669add 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -173,7 +173,6 @@ class Cachedump(interfaces.plugins.PluginInterface): kernel.symbol_table_name, hive_offsets=None if offset is None else [offset], ): - if hive.get_name().split("\\")[-1].upper() == "SYSTEM": syshive = hive if hive.get_name().split("\\")[-1].upper() == "SECURITY": diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 56609a73a..48b2e7c62 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -104,7 +104,6 @@ class Callbacks(interfaces.plugins.PluginInterface): ] for symbol_name, extended_list in symbol_names: - try: symbol_offset = ntkrnlmp.get_symbol(symbol_name).address except exceptions.SymbolError: @@ -354,7 +353,6 @@ class Callbacks(interfaces.plugins.PluginInterface): ) for callback in callback_record.Entry: - if not context.layers[layer_name].is_valid(callback.CallbackRoutine, 64): continue @@ -372,7 +370,6 @@ class Callbacks(interfaces.plugins.PluginInterface): yield "KeBugCheckCallbackListHead", callback.CallbackRoutine, component def _generator(self): - kernel = self.context.modules[self.config["kernel"]] callback_table_name = self.create_callback_table( @@ -397,7 +394,6 @@ class Callbacks(interfaces.plugins.PluginInterface): kernel.symbol_table_name, callback_table_name, ): - if callback_detail is None: detail = renderers.NotApplicableValue() else: @@ -451,7 +447,6 @@ class Callbacks(interfaces.plugins.PluginInterface): ) def run(self): - return renderers.TreeGrid( [ ("Type", str), diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index 2541629d5..6f39799c1 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -180,7 +180,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): ), ) - except (exceptions.InvalidAddressException): + except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVVV, f"Invalid address identified in drivers and devices: {driver.vol.offset:x}", diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index c1593b836..d73cea652 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -129,12 +129,10 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): nt_major_version == 6 and nt_minor_version >= 1 ) for proc in procs: - proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() for entry in proc.load_order_modules(): - BaseDllName = FullDllName = renderers.UnreadableValue() with contextlib.suppress(exceptions.InvalidAddressException): BaseDllName = entry.BaseDllName.get_string() diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index 4d2c24dea..b5cd33db7 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -71,7 +71,6 @@ class DriverIrp(interfaces.plugins.PluginInterface): for driver in driverscan.DriverScan.scan_drivers( self.context, kernel.layer_name, kernel.symbol_table_name ): - try: driver_name = driver.get_driver_name() except (ValueError, exceptions.InvalidAddressException): @@ -113,7 +112,6 @@ class DriverIrp(interfaces.plugins.PluginInterface): ) def run(self): - return renderers.TreeGrid( [ ("Offset", format_hints.Hex), diff --git a/volatility3/framework/plugins/windows/drivermodule.py b/volatility3/framework/plugins/windows/drivermodule.py index cc735db30..de827602e 100644 --- a/volatility3/framework/plugins/windows/drivermodule.py +++ b/volatility3/framework/plugins/windows/drivermodule.py @@ -73,7 +73,6 @@ class DriverModule(interfaces.plugins.PluginInterface): ) def run(self) -> renderers.TreeGrid: - return renderers.TreeGrid( [ ("Offset", format_hints.Hex), diff --git a/volatility3/framework/plugins/windows/driverscan.py b/volatility3/framework/plugins/windows/driverscan.py index d8df80702..24d81c3d5 100644 --- a/volatility3/framework/plugins/windows/driverscan.py +++ b/volatility3/framework/plugins/windows/driverscan.py @@ -54,7 +54,6 @@ class DriverScan(interfaces.plugins.PluginInterface): for result in poolscanner.PoolScanner.generate_pool_scan( context, layer_name, symbol_table, constraints ): - _constraint, mem_object, _header = result yield mem_object diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index af9568897..38d55d15d 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -229,7 +229,6 @@ class DumpFiles(interfaces.plugins.PluginInterface): ) for proc in procs: - try: object_table = proc.ObjectTable except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index a1dbd7665..66db03c9c 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -221,7 +221,6 @@ class Envars(interfaces.plugins.PluginInterface): ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/filescan.py b/volatility3/framework/plugins/windows/filescan.py index de3331e16..0f68f39d4 100644 --- a/volatility3/framework/plugins/windows/filescan.py +++ b/volatility3/framework/plugins/windows/filescan.py @@ -53,7 +53,6 @@ class FileScan(interfaces.plugins.PluginInterface): for result in poolscanner.PoolScanner.generate_pool_scan( context, layer_name, symbol_table, constraints ): - _constraint, mem_object, _header = result yield mem_object @@ -63,7 +62,6 @@ class FileScan(interfaces.plugins.PluginInterface): for fileobj in self.scan_files( self.context, kernel.layer_name, kernel.symbol_table_name ): - try: file_name = fileobj.FileName.String except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index c4088426f..9b20ed2d0 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -73,7 +73,6 @@ class GetServiceSIDs(interfaces.plugins.PluginInterface): ] def _generator(self): - kernel = self.context.modules[self.config["kernel"]] # Get the system hive for hive in hivelist.HiveList.list_hives( diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index 2334a328d..3e332f85d 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -112,7 +112,6 @@ class GetSIDs(interfaces.plugins.PluginInterface): filter_string="config\\software", hive_offsets=None, ): - try: for subkey in hive.get_key(key).get_subkeys(): sid = str(subkey.get_name()) @@ -165,7 +164,6 @@ class GetSIDs(interfaces.plugins.PluginInterface): return sids def _generator(self, procs): - user_sids = self.lookup_user_sids() # Go all over the process list, get the token @@ -214,7 +212,6 @@ class GetSIDs(interfaces.plugins.PluginInterface): ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 2f25a0597..dd7c90860 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -136,7 +136,6 @@ class Handles(interfaces.plugins.PluginInterface): """ if self._sar_value is None: - if not has_capstone: return None kernel = self.context.modules[self.config["kernel"]] @@ -160,7 +159,7 @@ class Handles(interfaces.plugins.PluginInterface): md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) - for (address, size, mnemonic, op_str) in md.disasm_lite( + for address, size, mnemonic, op_str in md.disasm_lite( data, kvo + func_addr ): # print("{} {} {} {}".format(address, size, mnemonic, op_str)) @@ -300,7 +299,6 @@ class Handles(interfaces.plugins.PluginInterface): masked_offset = offset & layer_object.maximum_address for entry in table: - if level > 0: for x in self._make_handle_array(entry, level - 1, depth): yield x @@ -329,7 +327,6 @@ class Handles(interfaces.plugins.PluginInterface): continue def handles(self, handle_table): - try: TableCode = handle_table.TableCode & ~self._level_mask table_levels = handle_table.TableCode & self._level_mask @@ -395,7 +392,7 @@ class Handles(interfaces.plugins.PluginInterface): except (ValueError, exceptions.InvalidAddressException): obj_name = "" - except (exceptions.InvalidAddressException): + except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, f"Cannot access _OBJECT_HEADER at {entry.vol.offset:#x}", @@ -416,7 +413,6 @@ class Handles(interfaces.plugins.PluginInterface): ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index 72bea2c8b..0c98ab8ca 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -602,7 +602,6 @@ class Hashdump(interfaces.plugins.PluginInterface): kernel.symbol_table_name, hive_offsets=None if offset is None else [offset], ): - if hive.get_name().split("\\")[-1].upper() == "SYSTEM": syshive = hive if hive.get_name().split("\\")[-1].upper() == "SAM": diff --git a/volatility3/framework/plugins/windows/info.py b/volatility3/framework/plugins/windows/info.py index aa7837029..100a677c2 100644 --- a/volatility3/framework/plugins/windows/info.py +++ b/volatility3/framework/plugins/windows/info.py @@ -187,7 +187,6 @@ class Info(plugins.PluginInterface): return nt_header def _generator(self): - kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name @@ -215,7 +214,6 @@ class Info(plugins.PluginInterface): yield (0, (layer.name, f"{i} {layer.__class__.__name__}")) if kdbg.Header.OwnerTag == 0x4742444B: - yield (0, ("KdDebuggerDataBlock", hex(kdbg.vol.offset))) yield (0, ("NTBuildLab", kdbg.get_build_lab())) yield (0, ("CSDVersion", str(kdbg.get_csdversion()))) @@ -285,5 +283,4 @@ class Info(plugins.PluginInterface): ) def run(self): - return TreeGrid([("Variable", str), ("Value", str)], self._generator()) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 354ef31c9..d84c133c0 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -103,7 +103,7 @@ class JobLinks(interfaces.plugins.PluginInterface): ), ) - except (exceptions.InvalidAddressException): + except exceptions.InvalidAddressException: continue def run(self) -> renderers.TreeGrid: diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index a9b229048..9642810a5 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -33,7 +33,6 @@ class LdrModules(interfaces.plugins.PluginInterface): ] def _generator(self, procs): - pe_table_name = intermed.IntermediateSymbolTable.create( self.context, self.config_path, "windows", "pe", class_types=pe.class_types ) diff --git a/volatility3/framework/plugins/windows/lsadump.py b/volatility3/framework/plugins/windows/lsadump.py index 8cb239905..12589b07e 100644 --- a/volatility3/framework/plugins/windows/lsadump.py +++ b/volatility3/framework/plugins/windows/lsadump.py @@ -118,12 +118,10 @@ class Lsadump(interfaces.plugins.PluginInterface): if enc_secret_key: enc_secret_value = next(enc_secret_key.get_values()) if enc_secret_value: - enc_secret = sechive.read( enc_secret_value.Data + 4, enc_secret_value.DataLength ) if enc_secret: - if not is_vista_or_later: secret = cls.decrypt_secret(enc_secret[0xC:], lsakey) else: @@ -160,7 +158,6 @@ class Lsadump(interfaces.plugins.PluginInterface): def _generator( self, syshive: registry.RegistryHive, sechive: registry.RegistryHive ): - kernel = self.context.modules[self.config["kernel"]] vista_or_later = versions.is_vista_or_later( @@ -183,7 +180,6 @@ class Lsadump(interfaces.plugins.PluginInterface): return for key in secrets_key.get_subkeys(): - sec_val_key = hashdump.Hashdump.get_hive_key( sechive, "Policy\\Secrets\\" + key.get_key_path().split("\\")[3] + "\\CurrVal", @@ -208,7 +204,6 @@ class Lsadump(interfaces.plugins.PluginInterface): yield (0, (key.get_name(), secret.decode("latin1"), secret)) def run(self): - offset = self.config.get("offset", None) syshive = sechive = None kernel = self.context.modules[self.config["kernel"]] @@ -220,7 +215,6 @@ class Lsadump(interfaces.plugins.PluginInterface): kernel.symbol_table_name, hive_offsets=None if offset is None else [offset], ): - if hive.get_name().split("\\")[-1].upper() == "SYSTEM": syshive = hive if hive.get_name().split("\\")[-1].upper() == "SECURITY": diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 1e7a009eb..424925955 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -151,7 +151,6 @@ class Malfind(interfaces.plugins.PluginInterface): for vad, data in self.list_injections( self.context, kernel.layer_name, kernel.symbol_table_name, proc ): - # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): architecture = "intel" diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index ccf6eccea..e58ca8c24 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -99,7 +99,6 @@ class MBRScan(interfaces.plugins.PluginInterface): all_zeros = bootcode.count(b"\x00") == len(bootcode) if not all_zeros: - partition_entries = [ partition_table.FirstEntry, partition_table.SecondEntry, @@ -155,7 +154,6 @@ class MBRScan(interfaces.plugins.PluginInterface): for partition_index, partition_entry_object in enumerate( partition_entries, start=1 ): - if not self.config.get("full", True): yield ( 1, diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index bbd9a7b4a..99fadac07 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -70,7 +70,6 @@ class ModScan(interfaces.plugins.PluginInterface): for result in poolscanner.PoolScanner.generate_pool_scan( context, layer_name, symbol_table, constraints ): - _constraint, mem_object, _header = result yield mem_object @@ -175,7 +174,6 @@ class ModScan(interfaces.plugins.PluginInterface): for mod in self.scan_modules( self.context, kernel.layer_name, kernel.symbol_table_name ): - try: BaseDllName = mod.BaseDllName.get_string() except exceptions.InvalidAddressException: @@ -188,7 +186,6 @@ class ModScan(interfaces.plugins.PluginInterface): file_output = "Disabled" if self.config["dump"]: - session_layer_name = self.find_session_layer( self.context, session_layers, mod.DllBase ) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index eba6d1ce7..ff61c215c 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -53,7 +53,6 @@ class Modules(interfaces.plugins.PluginInterface): for mod in self.list_modules( self.context, kernel.layer_name, kernel.symbol_table_name ): - try: BaseDllName = mod.BaseDllName.get_string() except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/windows/mutantscan.py b/volatility3/framework/plugins/windows/mutantscan.py index ad6e024d1..64d3b5470 100644 --- a/volatility3/framework/plugins/windows/mutantscan.py +++ b/volatility3/framework/plugins/windows/mutantscan.py @@ -53,7 +53,6 @@ class MutantScan(interfaces.plugins.PluginInterface): for result in poolscanner.PoolScanner.generate_pool_scan( context, layer_name, symbol_table, constraints ): - _constraint, mem_object, _header = result yield mem_object @@ -63,7 +62,6 @@ class MutantScan(interfaces.plugins.PluginInterface): for mutant in self.scan_mutants( self.context, kernel.layer_name, kernel.symbol_table_name ): - try: name = mutant.get_name() except (ValueError, exceptions.InvalidAddressException): diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 5c866bfeb..d0bbd5cbd 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -375,7 +375,6 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for result in poolscanner.PoolScanner.generate_pool_scan( context, layer_name, nt_symbol_table, constraints ): - _constraint, mem_object, _header = result yield mem_object @@ -394,7 +393,6 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.symbol_table_name, netscan_symbol_table, ): - vollog.debug( f"Found netw obj @ 0x{netw_obj.vol.offset:2x} of assumed type {type(netw_obj)}" ) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 1685f2a21..d3ce3fd2e 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -329,7 +329,6 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): alignment, net_symbol_table, ): - endpoint = context.object( obj_name, layer_name=layer_name, @@ -591,7 +590,6 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): tcpip_module.DllBase, tcpip_symbol_table, ): - # objects passed pool header constraints. check for additional constraints if strict flag is set. if not show_corrupt_results and not netw_obj.is_valid(): continue diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 13c611bf8..e131c5f78 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -144,7 +144,6 @@ class PoolScanner(plugins.PluginInterface): ] def _generator(self): - kernel = self.context.modules[self.config["kernel"]] symbol_table = kernel.symbol_table_name @@ -367,7 +366,6 @@ class PoolScanner(plugins.PluginInterface): for constraint, header in cls.pool_scan( context, scan_layer, symbol_table, constraints, alignment=alignment ): - mem_objects = header.get_object( constraint=constraint, use_top_down=is_windows_8_or_later, diff --git a/volatility3/framework/plugins/windows/privileges.py b/volatility3/framework/plugins/windows/privileges.py index 7a7087c95..0370dfc92 100644 --- a/volatility3/framework/plugins/windows/privileges.py +++ b/volatility3/framework/plugins/windows/privileges.py @@ -66,7 +66,6 @@ class Privs(interfaces.plugins.PluginInterface): ] def _generator(self, procs): - for task in procs: try: process_token = task.Token.dereference().cast("_TOKEN") @@ -107,7 +106,6 @@ class Privs(interfaces.plugins.PluginInterface): ) def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 7a06af36f..88697e71a 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -226,7 +226,6 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.symbol_table_name, filter_func=self.create_pid_filter(self.config.get("pid", None)), ): - if not self.config.get("physical", self.PHYSICAL_DEFAULT): offset = proc.vol.offset else: diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 427814d22..3d9ae5c1e 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -87,7 +87,6 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for result in poolscanner.PoolScanner.generate_pool_scan( context, layer_name, symbol_table, constraints ): - _constraint, mem_object, _header = result if not filter_func(mem_object): yield mem_object @@ -192,7 +191,6 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.symbol_table_name, filter_func=pslist.PsList.create_pid_filter(self.config.get("pid", None)), ): - file_output = "Disabled" if self.config["dump"]: # windows 10 objects (maybe others in the future) are already in virtual memory diff --git a/volatility3/framework/plugins/windows/registry/hivelist.py b/volatility3/framework/plugins/windows/registry/hivelist.py index 4abcd2f15..91798de40 100644 --- a/volatility3/framework/plugins/windows/registry/hivelist.py +++ b/volatility3/framework/plugins/windows/registry/hivelist.py @@ -88,7 +88,6 @@ class HiveList(interfaces.plugins.PluginInterface): symbol_table=kernel.symbol_table_name, filter_string=self.config.get("filter", None), ): - file_output = "Disabled" if self.config["dump"]: # Construct the hive diff --git a/volatility3/framework/plugins/windows/registry/hivescan.py b/volatility3/framework/plugins/windows/registry/hivescan.py index c3a52e303..7b3c0b622 100644 --- a/volatility3/framework/plugins/windows/registry/hivescan.py +++ b/volatility3/framework/plugins/windows/registry/hivescan.py @@ -86,7 +86,6 @@ class HiveScan(interfaces.plugins.PluginInterface): for hive in self.scan_hives( self.context, kernel.layer_name, kernel.symbol_table_name ): - yield (0, (format_hints.Hex(hive.vol.offset),)) def run(self): diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 19527321e..537bfc943 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -241,7 +241,6 @@ class PrintKey(interfaces.plugins.PluginInterface): key: str = None, recurse: bool = False, ): - for hive in hivelist.HiveList.list_hives( self.context, self.config_path, @@ -249,14 +248,13 @@ class PrintKey(interfaces.plugins.PluginInterface): symbol_table=symbol_table, hive_offsets=hive_offsets, ): - try: # Walk it if key is not None: node_path = hive.get_key(key, return_list=True) else: node_path = [hive.get_node(hive.root_cell_offset)] - for (x, y) in self._printkey_iterator(hive, node_path, recurse=recurse): + for x, y in self._printkey_iterator(hive, node_path, recurse=recurse): yield (x - len(node_path), y) except ( exceptions.InvalidAddressException, diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index f64a130fa..f90724f66 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -248,7 +248,6 @@ class UserAssist(interfaces.plugins.PluginInterface): # output any values under Count for value in countkey.get_values(): - value_name = value.get_name() with contextlib.suppress(UnicodeDecodeError): value_name = codecs.encode(value_name, "rot_13") @@ -281,7 +280,6 @@ class UserAssist(interfaces.plugins.PluginInterface): yield result def _generator(self): - hive_offsets = None if self.config.get("offset", None) is not None: hive_offsets = [self.config.get("offset", None)] diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py index 3e15878bd..d766b40ea 100644 --- a/volatility3/framework/plugins/windows/sessions.py +++ b/volatility3/framework/plugins/windows/sessions.py @@ -51,7 +51,6 @@ class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) kernel.symbol_table_name, filter_func=filter_func, ): - session_id = proc.get_session_id() # Detect RDP, Console or set default value @@ -112,7 +111,6 @@ class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) yield (description, timeliner.TimeLinerType.CREATED, row_data[5]) def run(self): - return renderers.TreeGrid( [ ("Session ID", int), diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index e7a1820e4..b697774cb 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -187,7 +187,6 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): proc_layer_name: str, cryptdll_base: int, ) -> Tuple[interfaces.objects.ObjectInterface, int, int, int]: - """ Finds the CSystems array through use of PDB symbols @@ -574,7 +573,6 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): scanners.BytesScanner(b"\x17\x00\x00\x00\x01\x00\x00\x00"), sections=[(cryptdll_base, cryptdll_size)], ): - # this occurs across page boundaries if not proc_layer.is_valid(address, ecrypt_size): continue diff --git a/volatility3/framework/plugins/windows/ssdt.py b/volatility3/framework/plugins/windows/ssdt.py index 184d8388c..6a47c36e9 100644 --- a/volatility3/framework/plugins/windows/ssdt.py +++ b/volatility3/framework/plugins/windows/ssdt.py @@ -56,7 +56,6 @@ class SSDT(plugins.PluginInterface): context_modules = [] for mod in mods: - try: module_name_with_ext = mod.BaseDllName.get_string() except exceptions.InvalidAddressException: @@ -83,7 +82,6 @@ class SSDT(plugins.PluginInterface): return contexts.ModuleCollection(context_modules) def _generator(self) -> Iterator[Tuple[int, Tuple[int, int, Any, Any]]]: - kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name @@ -132,7 +130,6 @@ class SSDT(plugins.PluginInterface): ) for idx, function_obj in enumerate(functions): - function = find_address(function_obj) module_symbols = collection.get_module_symbols_by_absolute_location( function diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index e6c1829e9..60562915e 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -180,7 +180,6 @@ class SvcScan(interfaces.plugins.PluginInterface): symbol_table=kernel.symbol_table_name, filter_func=filter_func, ): - proc_id = "Unknown" try: proc_id = task.UniqueProcessId @@ -200,7 +199,6 @@ class SvcScan(interfaces.plugins.PluginInterface): scanner=scanners.BytesScanner(needle=service_tag), sections=vadyarascan.VadYaraScan.get_vad_maps(task), ): - if not is_vista_or_later: service_record = self.context.object( service_table_name + constants.BANG + "_SERVICE_RECORD", diff --git a/volatility3/framework/plugins/windows/symlinkscan.py b/volatility3/framework/plugins/windows/symlinkscan.py index 78c2c6931..89fdf142e 100644 --- a/volatility3/framework/plugins/windows/symlinkscan.py +++ b/volatility3/framework/plugins/windows/symlinkscan.py @@ -52,7 +52,6 @@ class SymlinkScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfa for result in poolscanner.PoolScanner.generate_pool_scan( context, layer_name, symbol_table, constraints ): - _constraint, mem_object, _header = result yield mem_object @@ -62,7 +61,6 @@ class SymlinkScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfa for link in self.scan_symlinks( self.context, kernel.layer_name, kernel.symbol_table_name ): - try: from_name = link.get_link_name() except (ValueError, exceptions.InvalidAddressException): diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 3214c7134..812affe86 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -214,7 +214,6 @@ class VadInfo(interfaces.plugins.PluginInterface): process_name = utility.array_to_string(proc.ImageFileName) for vad in self.list_vads(proc, filter_func=filter_func): - file_output = "Disabled" if self.config["dump"]: file_handle = self.vad_dump( diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index fea4a0f80..1c6615804 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -222,7 +222,6 @@ class VerInfo(interfaces.plugins.PluginInterface): continue for entry in proc.load_order_modules(): - try: BaseDllName = entry.BaseDllName.get_string() except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/windows/virtmap.py b/volatility3/framework/plugins/windows/virtmap.py index 6fbf13932..5190bec8d 100644 --- a/volatility3/framework/plugins/windows/virtmap.py +++ b/volatility3/framework/plugins/windows/virtmap.py @@ -31,7 +31,7 @@ class VirtMap(interfaces.plugins.PluginInterface): def _generator(self, map): for entry in sorted(map): - for (start, end) in map[entry]: + for start, end in map[entry]: yield (0, (entry, format_hints.Hex(start), format_hints.Hex(end))) @classmethod diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index ee87b3b85..534686022 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -181,7 +181,7 @@ class TreeGrid(interfaces.renderers.TreeGrid): converted_columns: List[interfaces.renderers.Column] = [] if len(columns) < 1: raise ValueError("Columns must be a list containing at least one column") - for (name, column_type) in columns: + for name, column_type in columns: is_simple_type = issubclass(column_type, self.base_types) if not is_simple_type: raise TypeError( @@ -238,7 +238,7 @@ class TreeGrid(interfaces.renderers.TreeGrid): if not self.populated: try: prev_nodes: List[interfaces.renderers.TreeNode] = [] - for (level, item) in self._generator: + for level, item in self._generator: parent_index = min(len(prev_nodes), level) parent = prev_nodes[parent_index - 1] if parent_index > 0 else None treenode = self._append(parent, item) diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index 239acbde3..6ec9ebab9 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -36,7 +36,6 @@ class MultiTypeData(bytes): split_nulls: bool = False, show_hex: bool = False, ) -> "MultiTypeData": - if isinstance(original, int): data = str(original).encode(encoding) else: diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index d1af56a26..10cf39cf1 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -192,7 +192,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): replacements.add((traverser, child)) elif child.children: template_traverse_list.append(child) - for (parent, child) in replacements: + for parent, child in replacements: parent.replace_child(child, self._resolved[child.vol.type_name]) def get_type(self, type_name: str) -> interfaces.objects.Template: diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0d7cbb7e4..ce07167e5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -62,7 +62,6 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): # based on __d_path from the Linux kernel @classmethod def _do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> str: - ret_path: List[str] = [] while dentry != rdentry or vfsmnt != rmnt: @@ -204,7 +203,6 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): symbol_table: str, task: interfaces.objects.ObjectInterface, ): - # task.files can be null if not task.files: return @@ -225,7 +223,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): fd_table, count=max_fds, subtype=file_type, context=context ) - for (fd_num, filp) in enumerate(fds): + for fd_num, filp in enumerate(fds): if filp != 0: full_path = LinuxUtilities.path_for_file(context, task, filp) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 55f139730..5ab8f1aa0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -595,7 +595,6 @@ class list_head(objects.StructType, collections.abc.Iterable): seen = {self.vol.offset} while link.vol.offset not in seen: - obj = self._context.object( symbol_type, layer, offset=link.vol.offset - relative_offset ) @@ -630,7 +629,6 @@ class files_struct(objects.StructType): class mount(objects.StructType): - MNT_NOSUID = 0x01 MNT_NODEV = 0x02 MNT_NOEXEC = 0x04 @@ -755,7 +753,6 @@ class mount(objects.StructType): and current_mnt.has_parent() and current_mnt.vol.offset not in mnt_seen ): - current_dentry = current_mnt.mnt_mountpoint mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.mnt_parent diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index df6b23df8..416a7e4d2 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -22,7 +22,6 @@ class elf(objects.StructType): size: int, members: Dict[str, Tuple[int, interfaces.objects.Template]], ) -> None: - super().__init__( context=context, type_name=type_name, diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index 3909817ea..56ac96633 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -70,7 +70,6 @@ class MacUtilities(interfaces.configuration.VersionableInterface): kernel, # ikelos - how to type this?? mods_list: Iterator[Any], ): - try: start_addr = kernel.object_from_symbol("vm_kernel_stext") except exceptions.SymbolError: @@ -231,7 +230,6 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( queue, "tqh_first", "tqe_next", next_member, max_elements ): @@ -244,7 +242,6 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( queue, "lh_first", "le_next", next_member, max_elements ): @@ -257,7 +254,6 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( queue, "slh_first", "sle_next", next_member, max_elements ): diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index b678304b8..c89b527e6 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -206,7 +206,7 @@ class vm_map_entry(objects.StructType): permask = "rwx" perms = "" - for (ctr, i) in enumerate([1, 3, 5]): + for ctr, i in enumerate([1, 3, 5]): if (self.protection & i) == i: perms = perms + permask[ctr] else: @@ -593,7 +593,7 @@ class sysctl_oid(objects.StructType): checks = [0x80000000, 0x40000000, 0x00800000] perms = ["R", "W", "L"] - for (i, c) in enumerate(checks): + for i, c in enumerate(checks): if c & self.oid_kind: ret = ret + perms[i] else: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d34d6a22f..ba00a4053 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -202,7 +202,6 @@ class MMVAD_SHORT(objects.StructType): # this is for windows 8 and 10 elif self.has_member("VadNode"): - if self.VadNode.has_member("u1"): return self.VadNode.u1.Parent & ~0x3 @@ -211,7 +210,6 @@ class MMVAD_SHORT(objects.StructType): # also for windows 8 and 10 elif self.has_member("Core"): - if self.Core.VadNode.has_member("u1"): return self.Core.VadNode.u1.Parent & ~0x3 @@ -224,14 +222,12 @@ class MMVAD_SHORT(objects.StructType): """Get the VAD's starting virtual address. This is the first accessible byte in the range.""" if self.has_member("StartingVpn"): - if self.has_member("StartingVpnHigh"): return (self.StartingVpn << 12) | (self.StartingVpnHigh << 44) else: return self.StartingVpn << 12 elif self.has_member("Core"): - if self.Core.has_member("StartingVpnHigh"): return (self.Core.StartingVpn << 12) | (self.Core.StartingVpnHigh << 44) else: @@ -243,7 +239,6 @@ class MMVAD_SHORT(objects.StructType): """Get the VAD's ending virtual address. This is the last accessible byte in the range.""" if self.has_member("EndingVpn"): - if self.has_member("EndingVpnHigh"): return (((self.EndingVpn + 1) << 12) | (self.EndingVpnHigh << 44)) - 1 else: @@ -376,7 +371,6 @@ class EX_FAST_REF(objects.StructType): """ def dereference(self) -> interfaces.objects.ObjectInterface: - if constants.BANG not in self.vol.type_name: raise ValueError( f"Invalid symbol table name syntax (no {constants.BANG} found)" @@ -771,7 +765,6 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return False def get_vad_root(self): - # windows 8 and 2012 (_MM_AVL_TABLE) if self.VadRoot.has_member("BalancedRoot"): return self.VadRoot.BalancedRoot @@ -1346,7 +1339,6 @@ class SHARED_CACHE_MAP(objects.StructType): limit_depth = level_depth if section_size > self.VACB_SIZE_OF_FIRST_LEVEL: - # Create an array of 128 entries for the VACB index array. vacb_array = self._context.object( object_type=symbol_table_name + constants.BANG + "array", diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index c0f2bd61a..9b7573c2e 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -64,7 +64,6 @@ class _TCP_LISTENER(objects.StructType): size: int, members: Dict[str, Tuple[int, interfaces.objects.Template]], ) -> None: - super().__init__( context=context, type_name=type_name, @@ -167,7 +166,6 @@ class _TCP_LISTENER(objects.StructType): yield "v6", inaddr6_any, inaddr6_any def is_valid(self): - try: if not self.get_address_family() in (AF_INET, AF_INET6): vollog.debug( @@ -189,7 +187,6 @@ class _TCP_ENDPOINT(_TCP_LISTENER): """Class for objects found in TcpE pools""" def _ipv4_or_ipv6(self, inaddr): - if self.get_address_family() == AF_INET: return inet_ntop(socket.AF_INET, inaddr.addr4) else: @@ -214,7 +211,6 @@ class _TCP_ENDPOINT(_TCP_LISTENER): return None def is_valid(self): - if self.State not in self.State.choices.values(): vollog.debug( f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid tcp state {self.State}" diff --git a/volatility3/framework/symbols/windows/extensions/pe.py b/volatility3/framework/symbols/windows/extensions/pe.py index adee956f7..3f34fc3dd 100644 --- a/volatility3/framework/symbols/windows/extensions/pe.py +++ b/volatility3/framework/symbols/windows/extensions/pe.py @@ -151,7 +151,6 @@ class IMAGE_DOS_HEADER(objects.StructType): counter = 0 for sect in nt_header.get_sections(): - if sect.VirtualAddress > size_of_image: raise ValueError( f"Section VirtualAddress is too large: {sect.VirtualAddress}" diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 74fd0e4e8..a43933ccf 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -249,7 +249,6 @@ class PDBUtility(interfaces.configuration.VersionableInterface): # Check for writability filter_string = os.path.join(pdb_name, guid + "-" + str(age)) for path in symbols.__path__: - # Store any temporary files created by downloading PDB files tmp_files = [] potential_output_filename = os.path.join( @@ -353,7 +352,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): if end is None: end = ctx.layers[layer_name].maximum_address - for (GUID, age, pdb_name, signature_offset) in ctx.layers[layer_name].scan( + for GUID, age, pdb_name, signature_offset in ctx.layers[layer_name].scan( ctx, PdbSignatureScanner(pdb_names), progress_callback=progress_callback, @@ -426,7 +425,6 @@ class PDBUtility(interfaces.configuration.VersionableInterface): module_size: int = None, create_module: bool = False, ) -> Tuple[Optional[str], Optional[str]]: - if module_offset is None: module_offset = context.layers[layer_name].minimum_address if module_size is None: diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 3212cb465..5ef840f32 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -77,7 +77,6 @@ class Certificates(interfaces.plugins.PluginInterface): layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, ): - for top_key in [ "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", From aac4c735280537c55c8f6eb738f08aaa8304b8e2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 3 Feb 2023 09:22:30 +0000 Subject: [PATCH 287/526] Actions: Bump black checkout to Node16/wqv3 --- .github/workflows/black.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index dba5b5b80..5f4523072 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -6,7 +6,7 @@ jobs: lint: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: psf/black@stable with: options: "--check --diff --verbose" From 4bb6d93e122693b89a5a3947c12271400e25be3e Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 3 Feb 2023 10:18:06 +0000 Subject: [PATCH 288/526] Update linux.psscan --- volatility3/framework/plugins/linux/psscan.py | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 volatility3/framework/plugins/linux/psscan.py diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py new file mode 100644 index 000000000..f87b78eb1 --- /dev/null +++ b/volatility3/framework/plugins/linux/psscan.py @@ -0,0 +1,158 @@ +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import Iterable, List, Tuple +import struct +from enum import Enum + +from volatility3.framework import renderers, interfaces, symbols, constants +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints + +vollog = logging.getLogger(__name__) + + +class DescExitStateEnum(Enum): + """Enum for linux task exit_state as defined in include/linux/sched.h""" + + TASK_RUNNING = 0x00000000 + EXIT_DEAD = 0x00000010 + EXIT_ZOMBIE = 0x00000020 + EXIT_TRACE = EXIT_ZOMBIE | EXIT_DEAD + + +class PsScan(interfaces.plugins.PluginInterface): + """Scans for processes present in a particular linux image.""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + ] + + def _get_task_fields( + self, task: interfaces.objects.ObjectInterface + ) -> Tuple[int, int, int, str, str]: + """Extract the fields needed for the final output + + Args: + task: A task object from where to get the fields. + Returns: + A tuple with the fields to show in the plugin output. + """ + pid = task.tgid + tid = task.pid + ppid = task.parent.tgid if task.parent else 0 + name = utility.array_to_string(task.comm) + exit_state = DescExitStateEnum(task.exit_state).name + + task_fields = ( + format_hints.Hex(task.vol.offset), + pid, + tid, + ppid, + name, + exit_state, + ) + return task_fields + + def _generator(self): + """Generates the tasks found from scanning.""" + + for task in self.scan_tasks( + self.context, self.config["kernel"], self.config["kernel.layer_name"] + ): + row = self._get_task_fields(task) + yield (0, row) + + @classmethod + def scan_tasks( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + kernel_layer_name: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Scans for tasks in the memory layer. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + kernel_layer_name: The name for the kernel layer + Yields: + Task objects + """ + vmlinux = context.modules[vmlinux_module_name] + + # check if this image is 32bit or 64bit + is_32bit = not symbols.symbol_table_is_64bit(context, vmlinux.symbol_table_name) + if is_32bit: + pack_format = "I" + else: + pack_format = "Q" + + # get task_struct to find the offset to the sched_class pointer + sched_class_offset = vmlinux.get_type("task_struct").members["sched_class"][0] + kernel_layer = context.layers[kernel_layer_name] + + needles = [] + for symbol in vmlinux.symbols: + + # find all sched_class names by searching by if they include '_sched_class', e.g. 'fair_sched_class' + if "_sched_class" in symbol: + + # use canonicalize to set the appropriate sign extension for the addr + addr = kernel_layer.canonicalize(vmlinux.get_symbol(symbol).address) + + # append to needles list the packed hex for searching + needles.append(struct.pack(pack_format, addr)) + + # scan the memory_layer for these needles + memory_layer = context.layers["memory_layer"] + for address, _ in memory_layer.scan( + context, scanners.MultiStringScanner(needles) + ): + # create task in the memory_layer + ptask = context.object( + vmlinux.symbol_table_name + constants.BANG + "task_struct", + offset=address - sched_class_offset, + layer_name="memory_layer", + ) + + # sanity check exit_state + try: + # attempt tp parse the exist_state using the enum + DescExitStateEnum(ptask.exit_state) + except ValueError: + vollog.debug( + f"Skipping task_struct at {hex(ptask.vol.offset)} as exit_state {ptask.exit_state} is likely not valid" + ) + continue + + # sanity check pid + if not (0 < ptask.pid < 65535): + vollog.debug( + f"Skipping task_struct at {hex(ptask.vol.offset)} as pid {ptask.pid} is likely not valid" + ) + continue + + yield ptask + + def run(self): + columns = [ + ("OFFSET (P)", format_hints.Hex), + ("PID", int), + ("TID", int), + ("PPID", int), + ("COMM", str), + ("EXIT_STATE", str), + ] + return renderers.TreeGrid(columns, self._generator()) From e81935869ab51f5b792aaa0c390418ab64e90755 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 3 Feb 2023 10:37:29 +0000 Subject: [PATCH 289/526] Update linux.psscan to find kernel layer name correctly --- volatility3/framework/plugins/linux/psscan.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index f87b78eb1..25e7206ec 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -68,8 +68,11 @@ class PsScan(interfaces.plugins.PluginInterface): def _generator(self): """Generates the tasks found from scanning.""" + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] + for task in self.scan_tasks( - self.context, self.config["kernel"], self.config["kernel.layer_name"] + self.context, vmlinux_module_name, vmlinux.layer_name ): row = self._get_task_fields(task) yield (0, row) From 9e5a98ca40e5614df28ba5b09ffc64a656f19fb9 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 3 Feb 2023 10:51:18 +0000 Subject: [PATCH 290/526] Update linux.psscan with black linting and version --- volatility3/framework/plugins/linux/psscan.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 25e7206ec..ba233e53d 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -28,6 +28,7 @@ class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -101,23 +102,19 @@ class PsScan(interfaces.plugins.PluginInterface): pack_format = "I" else: pack_format = "Q" - # get task_struct to find the offset to the sched_class pointer sched_class_offset = vmlinux.get_type("task_struct").members["sched_class"][0] kernel_layer = context.layers[kernel_layer_name] needles = [] for symbol in vmlinux.symbols: - # find all sched_class names by searching by if they include '_sched_class', e.g. 'fair_sched_class' if "_sched_class" in symbol: - # use canonicalize to set the appropriate sign extension for the addr addr = kernel_layer.canonicalize(vmlinux.get_symbol(symbol).address) # append to needles list the packed hex for searching needles.append(struct.pack(pack_format, addr)) - # scan the memory_layer for these needles memory_layer = context.layers["memory_layer"] for address, _ in memory_layer.scan( @@ -139,14 +136,12 @@ class PsScan(interfaces.plugins.PluginInterface): f"Skipping task_struct at {hex(ptask.vol.offset)} as exit_state {ptask.exit_state} is likely not valid" ) continue - # sanity check pid if not (0 < ptask.pid < 65535): vollog.debug( f"Skipping task_struct at {hex(ptask.vol.offset)} as pid {ptask.pid} is likely not valid" ) continue - yield ptask def run(self): From d6ebad8235060a257e9af9a5e9831bb64a607606 Mon Sep 17 00:00:00 2001 From: Ashley Date: Thu, 9 Feb 2023 21:19:11 -0700 Subject: [PATCH 291/526] Update simple-plugin.rst Very minor typo fix. --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index c4908caf3..39670a62d 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -259,7 +259,7 @@ The plugin then takes the process's ``BaseDllName`` value, and calls :py:meth:`~ as defined by the symbols, are directly accessible and use the case-style of the symbol library it came from (in Windows, attributes are CamelCase), such as ``entry.BaseDllName`` in this instance. Any attributes not defined by the symbol but added by Volatility extensions cannot be properties (in case they overlap with the attributes defined in the symbol libraries) -and are therefore always methods and pretended with ``get_``, in this example ``BaseDllName.get_string()``. +and are therefore always methods and prepended with ``get_``, in this example ``BaseDllName.get_string()``. Finally, ``FullDllName`` is populated. These operations read from memory, and as such, the memory image may be unable to read the data at a particular offset. This will cause an exception to be thrown. In Volatility 3, exceptions are thrown From 4734a3d1f83295af45997758f1b31c07ba4e79fe Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 18 Feb 2023 21:14:02 +0000 Subject: [PATCH 292/526] Automagic: Fix cache issue with missing files --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 63c6fc7fa..1ca5ba210 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -332,7 +332,7 @@ class SqliteCache(CacheManagerInterface): if inner_url.scheme == "file": pathname = inner_url.path.split("!")[0] - if pathname: + if pathname and os.path.exists(pathname): timestamp = datetime.datetime.fromtimestamp( os.stat(pathname).st_mtime ) From 471b19b037deab511bc7e9144bc5b25b47cfc81d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 18 Feb 2023 21:05:05 +0000 Subject: [PATCH 293/526] Layers: Use ctypes for snappy support --- requirements-dev.txt | 4 ---- requirements.txt | 4 ---- volatility3/framework/layers/avml.py | 35 ++++++++++++++++++++++------ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7c372da2a..9db14d441 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,7 +20,3 @@ jsonschema>=2.3.0 # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 - -# This is required for analyzing Linux samples compressed using AVMLs native -# compression format. It is not required for AVML's standard LiME compression. -python-snappy==0.6.0 diff --git a/requirements.txt b/requirements.txt index 1793012f1..99e0786cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,3 @@ pycryptodome # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 - -# This is required for analyzing Linux samples compressed using AVMLs native -# compression format. It is not required for AVML's standard LiME compression. -python-snappy==0.6.0 diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index 66f3f0e4f..3ce25ca6f 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -6,6 +6,7 @@ The user of the file doesn't have to worry about the compression, but random access is not allowed.""" +import ctypes import logging import struct from typing import Tuple, List, Optional @@ -16,13 +17,35 @@ from volatility3.framework.layers import segmented vollog = logging.getLogger(__name__) try: - import snappy + from ctypes import cdll + + # TODO: Find library for windows if needed + lib_snappy = cdll.LoadLibrary("libsnappy.so.1") + __snappy_uncompress = lib_snappy.snappy_uncompress + __snappy_uncompressed_length = lib_snappy.snappy_uncompressed_length HAS_SNAPPY = True -except ImportError: +except OSError: HAS_SNAPPY = False +class SnappyException(Exception): + pass + + +def uncompress(s): + """Uncompress a snappy compressed string.""" + ulen = ctypes.c_int(0) + cresult = __snappy_uncompressed_length(s, len(s), ctypes.byref(ulen)) + if cresult != 0: + raise SnappyException(f"Error in snappy_uncompressed_length: {cresult}") + ubuf = ctypes.create_string_buffer(ulen.value) + __snappy_uncompress(s, len(s), ubuf, ctypes.byref(ulen)) + if cresult != 0: + raise SnappyException(f"Error in snappy_uncompress: {cresult}") + return ubuf.raw + + class AVMLLayer(segmented.NonLinearlySegmentedLayer): """A Lime format TranslationLayer. @@ -44,9 +67,7 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer): if magic not in [0x4C4D5641] or version != 2: raise exceptions.LayerException("File not completely in AVML format") if not HAS_SNAPPY: - vollog.warning( - "AVML file detected, but snappy python library not installed" - ) + vollog.warning("AVML file detected, but snappy library could not be found") raise exceptions.LayerException( "AVML format dependencies not satisfied (snappy)" ) @@ -131,7 +152,7 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer): ] if frame_type == 0x00: # Compressed data - frame_data = snappy.decompress(frame_data) + frame_data = uncompress(frame_data) # TODO: Verify CRC segments.append( ( @@ -156,7 +177,7 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer): ) -> bytes: start_offset, _, _, _ = self._find_segment(offset) if self._compressed[mapped_offset]: - decoded_data = snappy.decompress(data) + decoded_data = uncompress(data) else: decoded_data = data decoded_data = decoded_data[offset - start_offset :] From c7252e9707ac0fb96c5fd65036cf8a8ff4b96672 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 19 Feb 2023 10:03:23 +0000 Subject: [PATCH 294/526] Automagic: Handle snappy for windows. --- volatility3/framework/layers/avml.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index 3ce25ca6f..83c10186f 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -20,7 +20,23 @@ try: from ctypes import cdll # TODO: Find library for windows if needed - lib_snappy = cdll.LoadLibrary("libsnappy.so.1") + try: + # Linux/Mac + lib_snappy = cdll.LoadLibrary("libsnappy.so.1") + except OSError: + lib_snappy = None + + try: + if not lib_snappy: + # Windows 64 + lib_snappy = cdll.LoadLibrary("snappy64") + except OSError: + lib_snappy = None + + if lib_snappy: + # Windows 32 + lib_snappy = cdll.LoadLibrary("snappy32") + __snappy_uncompress = lib_snappy.snappy_uncompress __snappy_uncompressed_length = lib_snappy.snappy_uncompressed_length @@ -29,7 +45,7 @@ except OSError: HAS_SNAPPY = False -class SnappyException(Exception): +class SnappyException(exceptions.VolatilityException): pass @@ -65,9 +81,12 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer): layer.read(layer.minimum_address, struct.calcsize(header_structure)), ) if magic not in [0x4C4D5641] or version != 2: - raise exceptions.LayerException("File not completely in AVML format") + raise exceptions.LayerException("File not in AVML format") if not HAS_SNAPPY: - vollog.warning("AVML file detected, but snappy library could not be found") + vollog.warning( + "AVML file detected, but snappy library could not be found\n" + "Please install the snappy from your distribution or https://google.github.io/snappy/." + ) raise exceptions.LayerException( "AVML format dependencies not satisfied (snappy)" ) From ad3773d89884650cba6573280588f29e14e6ba0e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 19 Feb 2023 10:23:04 +0000 Subject: [PATCH 295/526] Automagic: Improve identified AVML CodeQL issues --- volatility3/framework/layers/avml.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index 83c10186f..1ba564c61 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -17,25 +17,23 @@ from volatility3.framework.layers import segmented vollog = logging.getLogger(__name__) try: - from ctypes import cdll - # TODO: Find library for windows if needed try: # Linux/Mac - lib_snappy = cdll.LoadLibrary("libsnappy.so.1") + lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.so.1") except OSError: lib_snappy = None try: if not lib_snappy: # Windows 64 - lib_snappy = cdll.LoadLibrary("snappy64") + lib_snappy = ctypes.cdll.LoadLibrary("snappy64") except OSError: lib_snappy = None if lib_snappy: # Windows 32 - lib_snappy = cdll.LoadLibrary("snappy32") + lib_snappy = ctypes.cdll.LoadLibrary("snappy32") __snappy_uncompress = lib_snappy.snappy_uncompress __snappy_uncompressed_length = lib_snappy.snappy_uncompressed_length @@ -56,7 +54,7 @@ def uncompress(s): if cresult != 0: raise SnappyException(f"Error in snappy_uncompressed_length: {cresult}") ubuf = ctypes.create_string_buffer(ulen.value) - __snappy_uncompress(s, len(s), ubuf, ctypes.byref(ulen)) + cresult = __snappy_uncompress(s, len(s), ubuf, ctypes.byref(ulen)) if cresult != 0: raise SnappyException(f"Error in snappy_uncompress: {cresult}") return ubuf.raw From 1770edf6fa7a87f5c714aaeaedb4e02ce91e040f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 22 Feb 2023 17:24:12 +0000 Subject: [PATCH 296/526] Automagic: Fix typo in cache stats --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 1ca5ba210..a58bf0091 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -371,7 +371,7 @@ class SqliteCache(CacheManagerInterface): # Get stats stats_base_types = len(json_obj.get("base_types", {})) - stats_types = len(json_obj.get("types", {})) + stats_types = len(json_obj.get("user_types", {})) stats_enums = len(json_obj.get("enums", {})) stats_symbols = len(json_obj.get("symbols", {})) From 588b0962541887dbfddedc779e5d93c5ceda87d2 Mon Sep 17 00:00:00 2001 From: Maxime THIEBAUT <46688461+0xThiebaut@users.noreply.github.com> Date: Sat, 25 Feb 2023 18:23:42 +0100 Subject: [PATCH 297/526] Add PID filtering to `windows.pstree` --- .../framework/plugins/windows/pstree.py | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/pstree.py b/volatility3/framework/plugins/windows/pstree.py index 88a3697da..5c78d1682 100644 --- a/volatility3/framework/plugins/windows/pstree.py +++ b/volatility3/framework/plugins/windows/pstree.py @@ -3,7 +3,7 @@ # import datetime import logging -from typing import Dict, Set, Tuple +from typing import Callable, Dict, Set, Tuple from volatility3.framework import objects, interfaces, renderers from volatility3.framework.configuration import requirements @@ -24,6 +24,7 @@ class PsTree(interfaces.plugins.PluginInterface): self._processes: Dict[int, Tuple[interfaces.objects.ObjectInterface, int]] = {} self._levels: Dict[int, int] = {} self._children: Dict[int, Set[int]] = {} + self._ancestors: Set[int] = set([]) @classmethod def get_requirements(cls): @@ -45,18 +46,26 @@ class PsTree(interfaces.plugins.PluginInterface): requirements.ListRequirement( name="pid", element_type=int, - description="Process ID to include (all other processes are excluded)", + description="Process ID to include (with ancestors and descendants, all other processes are excluded)", optional=True, ), ] - def find_level(self, pid: objects.Pointer) -> None: + def find_level( + self, + pid: objects.Pointer, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: False, + ) -> None: """Finds how deep the pid is in the processes list.""" - seen = set([]) - seen.add(pid) + seen = {pid} level = 0 proc, _ = self._processes.get(pid, None) + filtered = not filter_func(proc) while proc is not None and proc.InheritedFromUniqueProcessId not in seen: + if filtered: + self._ancestors.add(proc.UniqueProcessId) child_list = self._children.get(proc.InheritedFromUniqueProcessId, set([])) child_list.add(proc.UniqueProcessId) self._children[proc.InheritedFromUniqueProcessId] = child_list @@ -67,7 +76,12 @@ class PsTree(interfaces.plugins.PluginInterface): level += 1 self._levels[pid] = level - def _generator(self): + def _generator( + self, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: False, + ): """Generates the Tree of processes.""" kernel = self.context.modules[self.config["kernel"]] @@ -87,15 +101,21 @@ class PsTree(interfaces.plugins.PluginInterface): # Build the child/level maps for pid in self._processes: - self.find_level(pid) + self.find_level(pid, filter_func) process_pids = set([]) - def yield_processes(pid): + def yield_processes(pid, descendant: bool = False): if pid in process_pids: vollog.debug(f"Pid cycle: already processed pid {pid}") return + process_pids.add(pid) + + if pid not in self._ancestors and not descendant: + vollog.debug(f"Pid cycle: pid {pid} not in filtered tree") + return + proc, offset = self._processes[pid] row = ( proc.UniqueProcessId, @@ -114,7 +134,9 @@ class PsTree(interfaces.plugins.PluginInterface): yield (self._levels[pid] - 1, row) for child_pid in self._children.get(pid, []): - yield from yield_processes(child_pid) + yield from yield_processes( + child_pid, descendant or not filter_func(proc) + ) for pid in self._levels: if self._levels[pid] == 1: @@ -140,5 +162,9 @@ class PsTree(interfaces.plugins.PluginInterface): ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime), ], - self._generator(), + self._generator( + filter_func=pslist.PsList.create_pid_filter( + self.config.get("pid", None) + ), + ), ) From 3791b21695083f522fe6dd6009ffef1e5b05fc2f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 6 Mar 2023 00:13:47 +0000 Subject: [PATCH 298/526] Layers: Fix new snappy implementation error --- volatility3/framework/layers/avml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index 1ba564c61..c9c682ac4 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -39,7 +39,7 @@ try: __snappy_uncompressed_length = lib_snappy.snappy_uncompressed_length HAS_SNAPPY = True -except OSError: +except (AttributeError, OSError): HAS_SNAPPY = False From f3dea3619cc2521c9616a530222a47bff74f738b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 5 Mar 2023 23:14:09 +0000 Subject: [PATCH 299/526] Windows: Memoize part of the pool handling code --- volatility3/framework/contexts/__init__.py | 7 ++++- .../symbols/windows/extensions/pool.py | 29 +++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 226a303dd..ecce5041c 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -381,6 +381,7 @@ class ModuleCollection(interfaces.context.ModuleContainer): def __init__( self, modules: Optional[List[interfaces.context.ModuleInterface]] = None ) -> None: + self._prefix_count = {} super().__init__(modules) def deduplicate(self) -> "ModuleCollection": @@ -400,9 +401,13 @@ class ModuleCollection(interfaces.context.ModuleContainer): def free_module_name(self, prefix: str = "module") -> str: """Returns an unused module name""" - count = 1 + if prefix not in self._prefix_count: + self._prefix_count[prefix] = 1 + return prefix + count = self._prefix_count[prefix] while prefix + str(count) in self: count += 1 + self._prefix_count[prefix] = count return prefix + str(count) @property diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index ac7f36a99..052ae4dd6 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -396,13 +396,9 @@ class OBJECT_HEADER(objects.StructType): ) symbol_table_name = self.vol.type_name.split(constants.BANG)[0] - - try: - header_offset = self.NameInfoOffset - except AttributeError: - # http://codemachine.com/article_objectheader.html (Windows 7 and later) - name_info_bit = 0x2 - + if symbol_table_name in self._context.modules: + ntkrnlmp = self._context.modules[symbol_table_name] + else: layer = self._context.layers[self.vol.native_layer_name] kvo = layer.config.get("kernel_virtual_offset", None) @@ -411,16 +407,25 @@ class OBJECT_HEADER(objects.StructType): f"Could not find kernel_virtual_offset for layer: {self.vol.layer_name}" ) + # We know this symbol table name can't exist because we checked for it earlier ntkrnlmp = self._context.module( symbol_table_name, layer_name=self.vol.layer_name, offset=kvo ) + self._context.add_module(ntkrnlmp) + + try: + header_offset = self.NameInfoOffset + except AttributeError: + # http://codemachine.com/article_objectheader.html (Windows 7 and later) + name_info_bit = 0x2 + address = ntkrnlmp.get_symbol("ObpInfoMaskToOffset").address calculated_index = self.InfoMask & (name_info_bit | (name_info_bit - 1)) - header_offset = self._context.object( - symbol_table_name + constants.BANG + "unsigned char", + header_offset = ntkrnlmp.object( + "unsigned char", layer_name=self.vol.native_layer_name, - offset=kvo + address + calculated_index, + offset=address + calculated_index, ) if header_offset == 0: @@ -430,8 +435,8 @@ class OBJECT_HEADER(objects.StructType): ) ) - header = self._context.object( - symbol_table_name + constants.BANG + "_OBJECT_HEADER_NAME_INFO", + header = ntkrnlmp.object( + "_OBJECT_HEADER_NAME_INFO", layer_name=self.vol.layer_name, offset=self.vol.offset - header_offset, native_layer_name=self.vol.native_layer_name, From 21e34ec605d001a7f272907a13fe30210eeab278 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 5 Mar 2023 23:25:42 +0000 Subject: [PATCH 300/526] Windows: Fix up double adding the symbol from context.module --- volatility3/framework/symbols/windows/extensions/pool.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index 052ae4dd6..b761ddad8 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -396,6 +396,7 @@ class OBJECT_HEADER(objects.StructType): ) symbol_table_name = self.vol.type_name.split(constants.BANG)[0] + if symbol_table_name in self._context.modules: ntkrnlmp = self._context.modules[symbol_table_name] else: @@ -411,7 +412,6 @@ class OBJECT_HEADER(objects.StructType): ntkrnlmp = self._context.module( symbol_table_name, layer_name=self.vol.layer_name, offset=kvo ) - self._context.add_module(ntkrnlmp) try: header_offset = self.NameInfoOffset @@ -440,5 +440,6 @@ class OBJECT_HEADER(objects.StructType): layer_name=self.vol.layer_name, offset=self.vol.offset - header_offset, native_layer_name=self.vol.native_layer_name, + absolute=True, ) return header From a34fb8497633394062e10366553a2c69ba10c85a Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 8 Mar 2023 13:31:37 +0000 Subject: [PATCH 301/526] Fix linux.psscan to use kernel offset when finding symbol location --- volatility3/framework/plugins/linux/psscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index ba233e53d..60b96ba40 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -111,7 +111,7 @@ class PsScan(interfaces.plugins.PluginInterface): # find all sched_class names by searching by if they include '_sched_class', e.g. 'fair_sched_class' if "_sched_class" in symbol: # use canonicalize to set the appropriate sign extension for the addr - addr = kernel_layer.canonicalize(vmlinux.get_symbol(symbol).address) + addr = kernel_layer.canonicalize(vmlinux.get_symbol(symbol).address + vmlinux.offset) # append to needles list the packed hex for searching needles.append(struct.pack(pack_format, addr)) From bdf57f071697aa7688595efed438c8b80aa336d6 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 8 Mar 2023 13:44:08 +0000 Subject: [PATCH 302/526] Add extra debug messages to linux.psscan when finding symbol locations --- volatility3/framework/plugins/linux/psscan.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 60b96ba40..7cf0aa631 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -112,9 +112,16 @@ class PsScan(interfaces.plugins.PluginInterface): if "_sched_class" in symbol: # use canonicalize to set the appropriate sign extension for the addr addr = kernel_layer.canonicalize(vmlinux.get_symbol(symbol).address + vmlinux.offset) + packed_addr = struct.pack(pack_format, addr) + + # debug message to show needles being searched for and symbol names + vollog.debug( + f"Found a sched_class named {symbol} at offset {hex(addr)}. Will scan for these bytes: {packed_addr.hex()}" + ) # append to needles list the packed hex for searching - needles.append(struct.pack(pack_format, addr)) + needles.append(packed_addr) + # scan the memory_layer for these needles memory_layer = context.layers["memory_layer"] for address, _ in memory_layer.scan( From abfe104eb96e5ddd4a07e7a7a4dd5073fc88e5d0 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 8 Mar 2023 14:02:13 +0000 Subject: [PATCH 303/526] Fix linux.pscan to find memory layer to scan using kernel layers dependencies rather than hard coded value. --- volatility3/framework/plugins/linux/psscan.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 7cf0aa631..e8cc17a40 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -6,7 +6,7 @@ from typing import Iterable, List, Tuple import struct from enum import Enum -from volatility3.framework import renderers, interfaces, symbols, constants +from volatility3.framework import renderers, interfaces, symbols, constants, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.layers import scanners @@ -122,8 +122,20 @@ class PsScan(interfaces.plugins.PluginInterface): # append to needles list the packed hex for searching needles.append(packed_addr) + # find the memory layer to scan + if len(kernel_layer.dependencies) > 1: + vollog.warning( + f"Kernel layer depends on multiple layers however only {kernel_layer.dependencies[0]} will be scanned by this plugin." + ) + elif len(kernel_layer.dependencies) == 0: + vollog.error( + f"Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." + ) + raise exceptions.LayerException(kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies") + + memory_layer = context.layers[kernel_layer.dependencies[0]] + # scan the memory_layer for these needles - memory_layer = context.layers["memory_layer"] for address, _ in memory_layer.scan( context, scanners.MultiStringScanner(needles) ): From 32db4c0e5f3804b33f4cc1a4f66fae90494c0df8 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 8 Mar 2023 14:08:44 +0000 Subject: [PATCH 304/526] Fix black linting for linux.psscan. --- volatility3/framework/plugins/linux/psscan.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index e8cc17a40..f4bfd347e 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -111,7 +111,9 @@ class PsScan(interfaces.plugins.PluginInterface): # find all sched_class names by searching by if they include '_sched_class', e.g. 'fair_sched_class' if "_sched_class" in symbol: # use canonicalize to set the appropriate sign extension for the addr - addr = kernel_layer.canonicalize(vmlinux.get_symbol(symbol).address + vmlinux.offset) + addr = kernel_layer.canonicalize( + vmlinux.get_symbol(symbol).address + vmlinux.offset + ) packed_addr = struct.pack(pack_format, addr) # debug message to show needles being searched for and symbol names @@ -121,20 +123,20 @@ class PsScan(interfaces.plugins.PluginInterface): # append to needles list the packed hex for searching needles.append(packed_addr) - # find the memory layer to scan if len(kernel_layer.dependencies) > 1: vollog.warning( - f"Kernel layer depends on multiple layers however only {kernel_layer.dependencies[0]} will be scanned by this plugin." - ) + f"Kernel layer depends on multiple layers however only {kernel_layer.dependencies[0]} will be scanned by this plugin." + ) elif len(kernel_layer.dependencies) == 0: vollog.error( - f"Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." - ) - raise exceptions.LayerException(kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies") - + f"Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." + ) + raise exceptions.LayerException( + kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" + ) memory_layer = context.layers[kernel_layer.dependencies[0]] - + # scan the memory_layer for these needles for address, _ in memory_layer.scan( context, scanners.MultiStringScanner(needles) From a35afd4f343c10d7f8d1df2cb5eec8364c3dbd5a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 8 Mar 2023 20:40:04 +0000 Subject: [PATCH 305/526] Core: Bump framwork version after release branch --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 8f1163fe1..3a6b24ea8 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 4 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From b8755ac574e8226321ed169a1d6cac3a39505617 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 8 Mar 2023 20:42:24 +0000 Subject: [PATCH 306/526] Linux: fix black lint issues --- volatility3/framework/plugins/linux/envars.py | 1 - volatility3/framework/plugins/linux/iomem.py | 1 - 2 files changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 028eb2a57..5cbf0f502 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -70,7 +70,6 @@ class Envars(plugins.PluginInterface): # if mm exists attempt to get envars if mm: - # get process layer to read envars from proc_layer_name = task.add_process_layer() if proc_layer_name is None: diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index fddea4668..8efbf3b57 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -128,7 +128,6 @@ class IOMem(interfaces.plugins.PluginInterface): # only continue if iomem_root address was located if iomem_root_offset is not None: - # recursively parse the resources starting from the root resource at 'iomem_resource' for depth, (name, start, end) in self.parse_resource( self.context, vmlinux_module_name, iomem_root_offset From 85e86d45c547654afe7c2dca86f6c6d200cb05df Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 8 Mar 2023 20:42:24 +0000 Subject: [PATCH 307/526] Linux: fix black lint issues --- volatility3/framework/plugins/linux/envars.py | 1 - volatility3/framework/plugins/linux/iomem.py | 1 - 2 files changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 028eb2a57..5cbf0f502 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -70,7 +70,6 @@ class Envars(plugins.PluginInterface): # if mm exists attempt to get envars if mm: - # get process layer to read envars from proc_layer_name = task.add_process_layer() if proc_layer_name is None: diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index fddea4668..8efbf3b57 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -128,7 +128,6 @@ class IOMem(interfaces.plugins.PluginInterface): # only continue if iomem_root address was located if iomem_root_offset is not None: - # recursively parse the resources starting from the root resource at 'iomem_resource' for depth, (name, start, end) in self.parse_resource( self.context, vmlinux_module_name, iomem_root_offset From 99417cdfcc87d93d82b7288d8ee06ec4e06ada07 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 14:17:37 +0000 Subject: [PATCH 308/526] Linux: Psscan check parent pointer is valid --- volatility3/framework/plugins/linux/psscan.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index f4bfd347e..ca9d30586 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -52,7 +52,10 @@ class PsScan(interfaces.plugins.PluginInterface): """ pid = task.tgid tid = task.pid - ppid = task.parent.tgid if task.parent else 0 + ppid = 0 + + if task.parent.is_readable(): + ppid = task.parent.tgid name = utility.array_to_string(task.comm) exit_state = DescExitStateEnum(task.exit_state).name From 36ec5164703d2b5eaf0b3ff0d5f3a5f59572a5ff Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 15:17:24 +0000 Subject: [PATCH 309/526] Linux: Fix psscan task native_layer --- volatility3/framework/plugins/linux/psscan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index ca9d30586..462577e58 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -138,6 +138,7 @@ class PsScan(interfaces.plugins.PluginInterface): raise exceptions.LayerException( kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" ) + memory_layer_name = kernel_layer.dependencies[0] memory_layer = context.layers[kernel_layer.dependencies[0]] # scan the memory_layer for these needles @@ -148,7 +149,8 @@ class PsScan(interfaces.plugins.PluginInterface): ptask = context.object( vmlinux.symbol_table_name + constants.BANG + "task_struct", offset=address - sched_class_offset, - layer_name="memory_layer", + layer_name=memory_layer_name, + native_layer_name=kernel_layer_name, ) # sanity check exit_state From 46f56770af1785f5f0bcfb887849bb96e164f23c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 15:18:58 +0000 Subject: [PATCH 310/526] Core: Pointer is_readable should check the native layer --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index a04eedd87..3b1745718 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -442,7 +442,7 @@ class Pointer(Integer): def is_readable(self, layer_name: Optional[str] = None) -> bool: """Determines whether the address of this pointer can be read from memory.""" - layer_name = layer_name or self.vol.layer_name + layer_name = layer_name or self.vol.native_layer_name return self._context.layers[layer_name].is_valid(self, self.vol.subtype.size) def __getattr__(self, attr: str) -> Any: From cfec8e4b1214329726855ffc89541f9eec29f3ec Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 15:18:58 +0000 Subject: [PATCH 311/526] Core: Pointer is_readable should check the native layer --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index a04eedd87..3b1745718 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -442,7 +442,7 @@ class Pointer(Integer): def is_readable(self, layer_name: Optional[str] = None) -> bool: """Determines whether the address of this pointer can be read from memory.""" - layer_name = layer_name or self.vol.layer_name + layer_name = layer_name or self.vol.native_layer_name return self._context.layers[layer_name].is_valid(self, self.vol.subtype.size) def __getattr__(self, attr: str) -> Any: From b5ef7248ab1ef5ab5cab9fc5b6d05a88ef26c7bf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 15:17:24 +0000 Subject: [PATCH 312/526] Linux: Fix psscan task native_layer --- volatility3/framework/plugins/linux/psscan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index f4bfd347e..7b03d8c87 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -135,6 +135,7 @@ class PsScan(interfaces.plugins.PluginInterface): raise exceptions.LayerException( kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" ) + memory_layer_name = kernel_layer.dependencies[0] memory_layer = context.layers[kernel_layer.dependencies[0]] # scan the memory_layer for these needles @@ -145,7 +146,8 @@ class PsScan(interfaces.plugins.PluginInterface): ptask = context.object( vmlinux.symbol_table_name + constants.BANG + "task_struct", offset=address - sched_class_offset, - layer_name="memory_layer", + layer_name=memory_layer_name, + native_layer_name=kernel_layer_name, ) # sanity check exit_state From 9216bab61b3187fc248760ffdda5b86c8a694c9a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 17:32:46 +0000 Subject: [PATCH 313/526] Core: Improve import exception reporting --- volatility3/framework/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index ec0edc2fe..479925fb7 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -26,6 +26,7 @@ import importlib import inspect import logging import os +import traceback from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces @@ -183,7 +184,11 @@ def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str try: importlib.import_module(module) except ImportError as e: - vollog.debug(str(e)) + vollog.debug( + "".join( + traceback.TracebackException.from_exception(e).format(chain=True) + ) + ) vollog.debug( "Failed to import module {} based on file: {}".format(module, path) ) From d9a365d96fcd990c7faba32ab7aa63523203e9f8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 17:33:20 +0000 Subject: [PATCH 314/526] Linux: Rename linux.envars to linux.envvars --- volatility3/framework/plugins/linux/envars.py | 121 +----------------- .../framework/plugins/linux/envvars.py | 121 ++++++++++++++++++ 2 files changed, 127 insertions(+), 115 deletions(-) create mode 100644 volatility3/framework/plugins/linux/envvars.py diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 5cbf0f502..c4e3ed3c9 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -1,121 +1,12 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - +from volatility3.plugins import envvars import logging -from volatility3.framework import exceptions, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins -from volatility3.framework.objects import utility -from volatility3.plugins.linux import pslist - vollog = logging.getLogger(__name__) -class Envars(plugins.PluginInterface): - """Lists processes with their environment variables""" - - _required_framework_version = (2, 0, 0) - - @classmethod - def get_requirements(cls): - # Since we're calling the plugin, make sure we have the plugin's requirements - return [ - requirements.ModuleRequirement( - name="kernel", - description="Linux kernel", - architectures=["Intel32", "Intel64"], - ), - requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) - ), - requirements.ListRequirement( - name="pid", - description="Filter on specific process IDs", - element_type=int, - optional=True, - ), - ] - - def _generator(self, tasks): - """Generates a listing of processes along with environment variables""" - - # walk the process list and return the envars - for task in tasks: - pid = task.pid - - # get process name as string - name = utility.array_to_string(task.comm) - - # try and get task parent - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." - ) - ppid = 0 - - # kernel threads never have an mm as they do not have userland mappings - try: - mm = task.mm - except exceptions.InvalidAddressException: - # no mm so cannot get envars - vollog.debug( - f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." - ) - mm = None - continue - - # if mm exists attempt to get envars - if mm: - # get process layer to read envars from - proc_layer_name = task.add_process_layer() - if proc_layer_name is None: - vollog.debug( - f"Unable to construct process layer for task {pid} {name}, will not extract any envars." - ) - continue - proc_layer = self.context.layers[proc_layer_name] - - # get the size of the envars with sanity checking - envars_size = task.mm.env_end - task.mm.env_start - if not (0 < envars_size <= 8192): - vollog.debug( - f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." - ) - continue - - # attempt to read all envars data - try: - envar_data = proc_layer.read(task.mm.env_start, envars_size) - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." - ) - continue - - # parse envar data, envars are null terminated, keys and values are separated by '=' - envar_data = envar_data.rstrip(b"\x00") - for envar_pair in envar_data.split(b"\x00"): - try: - key, value = envar_pair.decode().split("=", 1) - except ValueError: - vollog.debug( - f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" - ) - continue - yield (0, (pid, ppid, name, key, value)) - - def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - - return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], - self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ), +class Envars(envvars.Envvars): + def run(self, *args, **kwargs): + vollog.warning( + "The linux.envars plugin has been renamed to linux.envvars and will only be accessible through the new name in a future release" ) + return super().run(*args, **kwargs) diff --git a/volatility3/framework/plugins/linux/envvars.py b/volatility3/framework/plugins/linux/envvars.py new file mode 100644 index 000000000..1d6c8b784 --- /dev/null +++ b/volatility3/framework/plugins/linux/envvars.py @@ -0,0 +1,121 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from volatility3.framework import exceptions, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class Envvars(plugins.PluginInterface): + """Lists processes with their environment variables""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + ] + + def _generator(self, tasks): + """Generates a listing of processes along with environment variables""" + + # walk the process list and return the envars + for task in tasks: + pid = task.pid + + # get process name as string + name = utility.array_to_string(task.comm) + + # try and get task parent + try: + ppid = task.parent.pid + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." + ) + ppid = 0 + + # kernel threads never have an mm as they do not have userland mappings + try: + mm = task.mm + except exceptions.InvalidAddressException: + # no mm so cannot get envars + vollog.debug( + f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." + ) + mm = None + continue + + # if mm exists attempt to get envars + if mm: + # get process layer to read envars from + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + vollog.debug( + f"Unable to construct process layer for task {pid} {name}, will not extract any envars." + ) + continue + proc_layer = self.context.layers[proc_layer_name] + + # get the size of the envars with sanity checking + envars_size = task.mm.env_end - task.mm.env_start + if not (0 < envars_size <= 8192): + vollog.debug( + f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." + ) + continue + + # attempt to read all envars data + try: + envar_data = proc_layer.read(task.mm.env_start, envars_size) + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." + ) + continue + + # parse envar data, envars are null terminated, keys and values are separated by '=' + envar_data = envar_data.rstrip(b"\x00") + for envar_pair in envar_data.split(b"\x00"): + try: + key, value = envar_pair.decode().split("=", 1) + except ValueError: + vollog.debug( + f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" + ) + continue + yield (0, (pid, ppid, name, key, value)) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + return renderers.TreeGrid( + [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), + ) From 3386a4fdc0e0c75d3ba9fb03b50a8d99b0cbc57d Mon Sep 17 00:00:00 2001 From: cstation Date: Tue, 14 Mar 2023 22:22:51 +0100 Subject: [PATCH 315/526] Remove broad try-except clause --- volatility3/framework/plugins/linux/elfs.py | 73 ++++++++++----------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index f56438f4f..23bdf1c3a 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -75,55 +75,52 @@ class Elfs(plugins.PluginInterface): proc_layer = context.layers[layer_name] file_handle = None - try: - elf_object = context.object( - elf_table_name + constants.BANG + "Elf", - offset=vma.vm_start, - layer_name=layer_name, - ) + elf_object = context.object( + elf_table_name + constants.BANG + "Elf", + offset=vma.vm_start, + layer_name=layer_name, + ) - if not elf_object.is_valid(): - return None + if not elf_object.is_valid(): + return None - sections = {} - # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? - for phdr in elf_object.get_program_headers(): - if phdr.p_type != 1: # PT_LOAD = 1 - continue + sections = {} + # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? + for phdr in elf_object.get_program_headers(): + if phdr.p_type != 1: # PT_LOAD = 1 + continue - start = phdr.p_vaddr - size = phdr.p_memsz - end = start + size + start = phdr.p_vaddr + size = phdr.p_memsz + end = start + size - # Use complete memory pages for dumping - # If start isn't a multiple of 4096, stick to the highest multiple < start - # If end isn't a multiple of 4096, stick to the lowest multiple > end - if start % 4096: - start = start & ~0xFFF + # Use complete memory pages for dumping + # If start isn't a multiple of 4096, stick to the highest multiple < start + # If end isn't a multiple of 4096, stick to the lowest multiple > end + if start % 4096: + start = start & ~0xFFF - if end % 4096: - end = (end & ~0xFFF) + 4096 + if end % 4096: + end = (end & ~0xFFF) + 4096 - real_size = end - start + real_size = end - start - if real_size < 0 or real_size > 100000000: - continue + if real_size < 0 or real_size > 100000000: + continue - sections[start] = real_size + sections[start] = real_size - elf_data = b"" - for section_start in sorted(sections.keys()): - read_size = sections[section_start] + elf_data = b"" + for section_start in sorted(sections.keys()): + read_size = sections[section_start] - buf = proc_layer.read(vma.vm_start + section_start, read_size, pad=True) - elf_data = elf_data + buf + buf = proc_layer.read(vma.vm_start + section_start, read_size, pad=True) + elf_data = elf_data + buf - file_handle = open_method( - f"pid.{task.pid}.{utility.array_to_string(task.comm)}.{vma.vm_start:#x}.dmp" - ) - file_handle.write(elf_data) - except Exception as e: - vollog.debug(f"Unable to dump ELF with pid {task.pid}: {e}") + file_handle = open_method( + f"pid.{task.pid}.{utility.array_to_string(task.comm)}.{vma.vm_start:#x}.dmp" + ) + file_handle.write(elf_data) return file_handle From 3d51b4a41a7110670826557ea82f3bd0ecf55e5c Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 24 Mar 2023 15:08:59 +0000 Subject: [PATCH 316/526] Add basic support for linux maple tree struct --- .../framework/symbols/linux/__init__.py | 2 + .../symbols/linux/extensions/__init__.py | 142 ++++++++++++++++++ 2 files changed, 144 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index ce07167e5..7a22241a5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -50,6 +50,8 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.optional_set_type_class("bt_sock", extensions.bt_sock) self.optional_set_type_class("xdp_sock", extensions.xdp_sock) + # Only found in 6.1+ kernels + self.optional_set_type_class("maple_tree", extensions.maple_tree) class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 5ab8f1aa0..72173f471 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -294,11 +294,128 @@ class fs_struct(objects.StructType): raise AttributeError("Unable to find the root mount") +class maple_tree(objects.StructType): + # include/linux/maple_tree.h + # Mask for Maple Tree Flags + MT_FLAGS_HEIGHT_MASK = 0x7C + MT_FLAGS_HEIGHT_OFFSET = 0x02 + + # Shift and mask to extract information from maple tree node pointers + MAPLE_NODE_TYPE_SHIFT = 0x03 + MAPLE_NODE_TYPE_MASK = 0x0F + MAPLE_NODE_POINTER_MASK = 0xFF + + # types of Maple Tree Nodes + MAPLE_DENSE = 0 + MAPLE_LEAF_64 = 1 + MAPLE_RANGE_64 = 2 + MAPLE_ARANGE_64 = 3 + + def get_slot_iter(self): + """Parse the Maple Tree and return every non zero slot.""" + maple_tree_offset, _, _ = self._parse_maple_tree_entry(self.vol.offset) + maple_tree_depth = ( + self.ma_flags & self.MT_FLAGS_HEIGHT_MASK + ) >> self.MT_FLAGS_HEIGHT_OFFSET + yield from self._parse_maple_tree_node( + self.ma_root, maple_tree_offset, maple_tree_depth + ) + + def _parse_maple_tree_node( + self, maple_tree_entry, parent, maple_tree_depth, seen=set(), depth=1 + ): + """Recursively parse Maple Tree Nodes and yield all non empty slots""" + + # protect against unlikely loop + if maple_tree_entry in seen: + vollog.warning( + f"The mte {hex(maple_tree_entry)} has all ready been seen, no further results will be produced for this node." + ) + return + else: + seen.add(maple_tree_entry) + if maple_tree_depth < depth: + vollog.warning( + f"The depth for the maple tree at {hex(self.vol.offset)} is {maple_tree_depth}, however when parsing the nodes " + f"a depth of {depth} was reached. This is unexpected and may lead to incorrect results." + ) + # parse the mte to extract the pointer value, node type, and leaf status + pointer, node_type, is_leaf = self._parse_maple_tree_entry(maple_tree_entry) + + # create a pointer object for the node parent mte (note this will include flags in the low bits) + symbol_table_name = self.get_symbol_table_name() + node_parent_mte = self._context.object( + symbol_table_name + constants.BANG + "pointer", + layer_name=self.vol.layer_name, + offset=pointer, + ) + + # extract the actual pointer to the parent of this node + node_parent_pointer, _, _ = self._parse_maple_tree_entry(node_parent_mte) + + # verify that the node_parent_pointer correctly points to the parent + assert node_parent_pointer == parent + + # create a node object + node = self._context.object( + symbol_table_name + constants.BANG + "maple_node", + layer_name=self.vol.layer_name, + offset=pointer, + ) + + # parse the slots based on the node type + if node_type == self.MAPLE_DENSE: + assert is_leaf == True + for slot in node.alloc.slot: + if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: + yield slot + elif node_type == self.MAPLE_LEAF_64: + assert is_leaf == True + for slot in node.mr64.slot: + if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: + yield slot + elif node_type == self.MAPLE_RANGE_64: + assert is_leaf == False + for slot in node.mr64.slot: + if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: + yield from self._parse_maple_tree_node( + slot, pointer, maple_tree_depth, seen, depth + 1 + ) + elif node_type == self.MAPLE_ARANGE_64: + assert is_leaf == False + for slot in node.ma64.slot: + if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: + yield from self._parse_maple_tree_node( + slot, pointer, maple_tree_depth, seen, depth + 1 + ) + else: + # unkown maple node type + raise AttributeError( + f"Unkown Maple Tree node type {node_type} at offset {hex(pointer)}." + ) + + def _parse_maple_tree_entry(self, maple_tree_entry): + """Parse a Maple Tree Entry and return the pointer, node type, if the node is a leaf, if the node is the root""" + # Extract the node type + node_type = ( + maple_tree_entry >> self.MAPLE_NODE_TYPE_SHIFT + ) & self.MAPLE_NODE_TYPE_MASK + + # Determine if it's a leaf node or not + is_leaf = node_type < self.MAPLE_RANGE_64 + + # Clear the lower bits to get the true pointer value + pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) + + return pointer, node_type, is_leaf class mm_struct(objects.StructType): def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct.""" + if not self.has_member('mmap'): + raise AttributeError("get_mmap_iter called on mm_struct where no mmap member exists.") + if not self.mmap: return @@ -312,7 +429,32 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next + def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns an iterator for the mm_mt member of an mm_struct.""" + + if not self.has_member('mm_mt'): + raise AttributeError("get_maple_tree_iter called on mm_struct where no mm_mt member exists.") + symbol_table_name = self.get_symbol_table_name() + for vma_pointer in self.mm_mt.get_slot_iter(): + # convert pointer to vm_area_struct and yield + vma = self._context.object( + symbol_table_name + constants.BANG + "vm_area_struct", + layer_name=self.vol.layer_name, + offset=vma_pointer + ) + yield vma + + def get_vma_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" + + if self.has_member('mmap'): + yield from self.get_mmap_iter() + elif self.has_member('mm_mt'): + yield from self.get_maple_tree_iter() + else: + raise AttributeError("Unable to find mmap or mm_mt in mm_struct") + class super_block(objects.StructType): # include/linux/kdev_t.h MINORBITS = 20 From 5207cfcb93cf7f8955b5a499ac13a46ba962da78 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 24 Mar 2023 15:10:34 +0000 Subject: [PATCH 317/526] Modify linux plugins to support both mmap and mm_mt --- volatility3/framework/plugins/linux/elfs.py | 2 +- volatility3/framework/plugins/linux/malfind.py | 2 +- volatility3/framework/plugins/linux/proc.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 822a69dd6..2ff5eb591 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -48,7 +48,7 @@ class Elfs(plugins.PluginInterface): name = utility.array_to_string(task.comm) - for vma in task.mm.get_mmap_iter(): + for vma in task.mm.get_vma_iter(): hdr = proc_layer.read(vma.vm_start, 4, pad=True) if not ( hdr[0] == 0x7F diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 18237b80c..1fd005de8 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -46,7 +46,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] - for vma in task.mm.get_mmap_iter(): + for vma in task.mm.get_vma_iter(): if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]": data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 9d8af482e..8979f6f63 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -44,7 +44,7 @@ class Maps(plugins.PluginInterface): name = utility.array_to_string(task.comm) - for vma in task.mm.get_mmap_iter(): + for vma in task.mm.get_vma_iter(): flags = vma.get_protection() page_offset = vma.get_page_offset() major = 0 From 89e477a068092cad50acfaa0fd965097ae5da8c7 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 24 Mar 2023 20:07:32 +0000 Subject: [PATCH 318/526] Remove _parse_maple_tree_entry function and make parsing easier to read --- .../symbols/linux/extensions/__init__.py | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 72173f471..9375a288b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -313,7 +313,7 @@ class maple_tree(objects.StructType): def get_slot_iter(self): """Parse the Maple Tree and return every non zero slot.""" - maple_tree_offset, _, _ = self._parse_maple_tree_entry(self.vol.offset) + maple_tree_offset = self.vol.offset & ~(self.MAPLE_NODE_POINTER_MASK) maple_tree_depth = ( self.ma_flags & self.MT_FLAGS_HEIGHT_MASK ) >> self.MT_FLAGS_HEIGHT_OFFSET @@ -340,7 +340,11 @@ class maple_tree(objects.StructType): f"a depth of {depth} was reached. This is unexpected and may lead to incorrect results." ) # parse the mte to extract the pointer value, node type, and leaf status - pointer, node_type, is_leaf = self._parse_maple_tree_entry(maple_tree_entry) + pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) + node_type = ( + maple_tree_entry >> self.MAPLE_NODE_TYPE_SHIFT + ) & self.MAPLE_NODE_TYPE_MASK + is_leaf = node_type < self.MAPLE_RANGE_64 # create a pointer object for the node parent mte (note this will include flags in the low bits) symbol_table_name = self.get_symbol_table_name() @@ -351,7 +355,7 @@ class maple_tree(objects.StructType): ) # extract the actual pointer to the parent of this node - node_parent_pointer, _, _ = self._parse_maple_tree_entry(node_parent_mte) + node_parent_pointer = node_parent_mte & ~(self.MAPLE_NODE_POINTER_MASK) # verify that the node_parent_pointer correctly points to the parent assert node_parent_pointer == parent @@ -394,21 +398,6 @@ class maple_tree(objects.StructType): f"Unkown Maple Tree node type {node_type} at offset {hex(pointer)}." ) - def _parse_maple_tree_entry(self, maple_tree_entry): - """Parse a Maple Tree Entry and return the pointer, node type, if the node is a leaf, if the node is the root""" - # Extract the node type - node_type = ( - maple_tree_entry >> self.MAPLE_NODE_TYPE_SHIFT - ) & self.MAPLE_NODE_TYPE_MASK - - # Determine if it's a leaf node or not - is_leaf = node_type < self.MAPLE_RANGE_64 - - # Clear the lower bits to get the true pointer value - pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) - - return pointer, node_type, is_leaf - class mm_struct(objects.StructType): def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct.""" From 089234671c70b746c91c25ac2980a6a4f35607b0 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 27 Mar 2023 06:29:30 +0100 Subject: [PATCH 319/526] Remove checks for leaf node in linux maple tree parsing, node type is enough. --- volatility3/framework/symbols/linux/extensions/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9375a288b..a8bf1b6aa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -344,7 +344,6 @@ class maple_tree(objects.StructType): node_type = ( maple_tree_entry >> self.MAPLE_NODE_TYPE_SHIFT ) & self.MAPLE_NODE_TYPE_MASK - is_leaf = node_type < self.MAPLE_RANGE_64 # create a pointer object for the node parent mte (note this will include flags in the low bits) symbol_table_name = self.get_symbol_table_name() @@ -369,24 +368,20 @@ class maple_tree(objects.StructType): # parse the slots based on the node type if node_type == self.MAPLE_DENSE: - assert is_leaf == True for slot in node.alloc.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield slot elif node_type == self.MAPLE_LEAF_64: - assert is_leaf == True for slot in node.mr64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield slot elif node_type == self.MAPLE_RANGE_64: - assert is_leaf == False for slot in node.mr64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( slot, pointer, maple_tree_depth, seen, depth + 1 ) elif node_type == self.MAPLE_ARANGE_64: - assert is_leaf == False for slot in node.ma64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( From 18a9f898325bf84ad48500a043a4c3b49d4afcdd Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 29 Mar 2023 10:00:55 +0100 Subject: [PATCH 320/526] update logic for checking if a vma should be saved to disk in linux.proc plugin --- volatility3/framework/plugins/linux/proc.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 99c8761c1..e885978b1 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -105,7 +105,6 @@ class Maps(plugins.PluginInterface): Args: context: The context to retrieve required elements (layers, symbol tables) from task: an task_struct instance - vma: The suspected VMA to extract (ObjectInterface) vm_start: The start virtual address from the vma to dump vm_end: The end virtual address from the vma to dump open_method: class to provide context manager for opening the file @@ -127,7 +126,16 @@ class Maps(plugins.PluginInterface): return None vm_size = vm_end - vm_start - if 0 < maxsize < vm_size: + + # check if vm_size is negative, this should never happen. + if vm_size < 0: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is negative." + ) + return None + + # check if vm_size is larger than the maxsize limit, and therefore is not saved out. + if maxsize <= vm_size: vollog.warning( f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is larger than maxsize limit of {maxsize}" ) From 61a2f78baaaa6bc7a957ab5c811494e2923ebf0d Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 29 Mar 2023 10:05:37 +0100 Subject: [PATCH 321/526] fix black linting in linux.proc plugin --- volatility3/framework/plugins/linux/proc.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index e885978b1..2d7348fff 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -124,7 +124,6 @@ class Maps(plugins.PluginInterface): ) ) return None - vm_size = vm_end - vm_start # check if vm_size is negative, this should never happen. @@ -133,14 +132,12 @@ class Maps(plugins.PluginInterface): f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is negative." ) return None - # check if vm_size is larger than the maxsize limit, and therefore is not saved out. if maxsize <= vm_size: vollog.warning( f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is larger than maxsize limit of {maxsize}" ) return None - proc_layer = context.layers[proc_layer_name] file_name = f"pid.{pid}.vma.{vm_start:#x}-{vm_end:#x}.dmp" try: @@ -152,11 +149,9 @@ class Maps(plugins.PluginInterface): data = proc_layer.read(offset, to_read, pad=True) file_handle.write(data) offset += to_read - except Exception as excp: vollog.debug(f"Unable to dump virtual memory {file_name}: {excp}") return None - return file_handle def _generator(self, tasks): @@ -179,11 +174,9 @@ class Maps(plugins.PluginInterface): return False vma_filter_func = vma_filter_function - for task in tasks: if not task.mm: continue - name = utility.array_to_string(task.comm) for vma in self.list_vmas(task, filter_func=vma_filter_func): @@ -200,7 +193,6 @@ class Maps(plugins.PluginInterface): major = inode_object.i_sb.major minor = inode_object.i_sb.minor inode = inode_object.i_ino - path = vma.get_name(self.context, task) file_output = "Disabled" @@ -215,7 +207,6 @@ class Maps(plugins.PluginInterface): ) vm_start = None vm_end = None - if vm_start and vm_end: # only attempt to dump the memory if we have vm_start and vm_end file_handle = self.vma_dump( @@ -230,7 +221,6 @@ class Maps(plugins.PluginInterface): if file_handle: file_handle.close() file_output = file_handle.preferred_filename - yield ( 0, ( From 60292c2da1efa886d0f46bd720af537a6069a623 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Mar 2023 20:46:06 +0100 Subject: [PATCH 322/526] Linux: Fix slight issue in envvars renaming --- volatility3/framework/plugins/linux/envars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index c4e3ed3c9..758943312 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -1,4 +1,4 @@ -from volatility3.plugins import envvars +from volatility3.plugins.linux import envvars import logging vollog = logging.getLogger(__name__) From 90b9157dcf2ababcd1c52128d72aa9732319701a Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 30 Mar 2023 08:54:56 +0100 Subject: [PATCH 323/526] Linux.proc: Fix broken variable in debug msg. --- volatility3/framework/plugins/linux/proc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 2d7348fff..c1a834bbe 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,6 +21,7 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod @@ -203,7 +204,7 @@ class Maps(plugins.PluginInterface): vm_end = vma.vm_end except AttributeError: vollog.debug( - f"Unable to find the vm_start and vm_end for vma at {vma.vol.offset:#x} for pid {pid}" + f"Unable to find the vm_start and vm_end for vma at {vma.vol.offset:#x} for pid {task.pid}" ) vm_start = None vm_end = None From 8d47f89eddbdf958406290f2da5affeb934dfc7a Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 30 Mar 2023 08:57:43 +0100 Subject: [PATCH 324/526] Linux.proc: Add debug msg when task has no mm member. --- volatility3/framework/plugins/linux/proc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index c1a834bbe..3ce9216fa 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -90,6 +90,10 @@ class Maps(plugins.PluginInterface): vollog.debug( f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.pid} due to filter_func" ) + else: + vollog.debug( + f"Excluded pid {task.pid} as there is no mm member. It is likely a kernel thread." + ) @classmethod def vma_dump( From 41edab23099b1e011d1b0acc4ef538f9ad406ec1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 4 Apr 2023 23:23:27 +0100 Subject: [PATCH 325/526] Plugins: Support yara-4.3.0 and above --- volatility3/framework/plugins/yarascan.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 1c548e5a6..1c8467689 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -34,13 +34,27 @@ class YaraScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules + self.st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( + 4, + 3, + ) def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: for match in self._rules.match(data=data): - for offset, name, value in match.strings: - yield (offset + data_offset, match.rule, name, value) + if self.st_object: + for match_string in match.strings: + for instance in match_string.instances: + yield ( + instance.offset + data_offset, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield (offset + data_offset, match.rule, name, value) class YaraScan(plugins.PluginInterface): From f33ea67e860837f579f7af2d60861baa68f350d5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 4 Apr 2023 23:57:35 +0100 Subject: [PATCH 326/526] Automagic: Update clear-cache and do removals first --- volatility3/framework/__init__.py | 6 +----- volatility3/framework/automagic/symbol_cache.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 479925fb7..c7b23a9c3 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -224,8 +224,4 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: def clear_cache(complete=False): - glob_pattern = "*.cache" - if not complete: - glob_pattern = "data_" + glob_pattern - for cache_filename in glob.glob(os.path.join(constants.CACHE_PATH, glob_pattern)): - os.unlink(cache_filename) + os.unlink(os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index a58bf0091..29f2cfd08 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -310,6 +310,14 @@ class SqliteCache(CacheManagerInterface): new_locations = on_disk_locations.difference(cached_locations) missing_locations = cached_locations.difference(on_disk_locations) + # Missing entries + if missing_locations: + self._database.cursor().execute( + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", + [x for x in missing_locations], + ) + self._database.commit() + cache_update = set() files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: @@ -437,15 +445,6 @@ class SqliteCache(CacheManagerInterface): progress_callback(100, "Reading remote ISF list") self._database.commit() - # Missing entries - - if missing_locations: - self._database.cursor().execute( - f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", - [x for x in missing_locations], - ) - self._database.commit() - def get_identifier_dictionary( self, operating_system: Optional[str] = None, local_only: bool = False ) -> Dict[bytes, str]: From 2e0ecdd770b84aab991aba85ae2d33143212ba43 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 7 Apr 2023 18:04:56 +0900 Subject: [PATCH 327/526] Fix: typo for linux iomem plugin --- volatility3/framework/plugins/linux/iomem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index 8efbf3b57..2056851aa 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -42,7 +42,7 @@ class IOMem(interfaces.plugins.PluginInterface): Args: context: The context to retrieve required elements (layers, symbol tables) from vmlinux_module_name: The name of the kernel module on which to operate - resource_offset: The offset to the resouce to be parsed + resource_offset: The offset to the resource to be parsed seen: The set of resource offsets that have already been parsed depth: How deep into the resource structure we are @@ -57,7 +57,7 @@ class IOMem(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.warning( f"Unable to create resource object at {resource_offset:#x}. This resource, " - "its sibling, and any of it's childern and will be missing from the output." + "its sibling, and any of it's children and will be missing from the output." ) return None From ebb74e2f9292b7cdd211c4886b06fb407e751c80 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Apr 2023 19:57:15 +0100 Subject: [PATCH 328/526] Core: Bump the copyright year of the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 502e26f10..471735af8 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 12 Apr 2023 19:57:15 +0100 Subject: [PATCH 329/526] Core: Bump the copyright year of the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 502e26f10..471735af8 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ The latest generated copy of the documentation can be found at: Date: Fri, 14 Apr 2023 08:08:27 +0100 Subject: [PATCH 330/526] Volshell: Mark the script as executable in the repo --- volshell.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 volshell.py diff --git a/volshell.py b/volshell.py old mode 100644 new mode 100755 From 58af80df3c9adfbe8df382b2242087d385c319a7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Apr 2023 13:29:41 +0100 Subject: [PATCH 331/526] Requirements: Shift location_from_file to the requirement from the CLI class --- volatility3/cli/__init__.py | 21 +++++---------- volatility3/framework/automagic/windows.py | 3 +++ .../framework/configuration/requirements.py | 26 +++++++++++++++++++ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 336902d50..99052d82e 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -353,7 +353,9 @@ class CommandLine: ### if args.file: try: - single_location = self.location_from_file(args.file) + single_location = requirements.URLRequirement.location_from_file( + args.file + ) ctx.config["automagic.LayerStacker.single_location"] = single_location except ValueError as excp: parser.error(str(excp)) @@ -456,19 +458,10 @@ class CommandLine: Returns: The URL for the location of the file """ - # We want to work in URLs, but we need to accept absolute and relative files (including on windows) - single_location = parse.urlparse(filename, "") - if single_location.scheme == "" or len(single_location.scheme) == 1: - single_location = parse.urlparse( - parse.urljoin("file:", request.pathname2url(os.path.abspath(filename))) - ) - if single_location.scheme == "file": - if not os.path.exists(request.url2pathname(single_location.path)): - filename = request.url2pathname(single_location.path) - if not filename: - raise ValueError("File URL looks incorrect (potentially missing /)") - raise ValueError(f"File does not exist: {filename}") - return parse.urlunparse(single_location) + vollog.debug( + f"{__name__}.location_from_file has been deprecated and moved to requirements.URIRequirement.location_from_file" + ) + return requirements.URIRequirement.location_from_file(filename) def process_exceptions(self, excp): """Provide useful feedback if an exception occurs during a run of a plugin.""" diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 986eeae22..ccc8de2eb 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -367,6 +367,7 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): progress_callback: constants.ProgressCallback = None, ) -> None: """Finds translation layers that can have swap layers added.""" + path_join = interfaces.configuration.path_join self._translation_requirement = self.find_requirements( context, @@ -382,11 +383,13 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): swap_sub_config, swap_req = self.find_swap_requirement( trans_sub_config, trans_req ) + counter = 0 swap_config = interfaces.configuration.parent_path(swap_sub_config) if swap_req and swap_req.unsatisfied(context, swap_config): # See if any of them need constructing + for swap_location in self.config.get("single_swap_locations", []): # Setup config locations/paths current_layer_name = swap_req.name + str(counter) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 6b64b1cb9..abdffdbe4 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -10,7 +10,9 @@ expect to be in the context (such as particular layers or symboltables). """ import abc import logging +import os from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type +from urllib import parse, request from volatility3.framework import constants, interfaces @@ -55,6 +57,30 @@ class URIRequirement(StringRequirement): # TODO: Maybe a a check that to unsatisfied that the path really is a URL? + @classmethod + def location_from_file(cls, filename: str) -> str: + """Returns the URL location from a file parameter (which may be a URL) + + Args: + filename: The path to the file (either an absolute, relative, or URL path) + + Returns: + The URL for the location of the file + """ + # We want to work in URLs, but we need to accept absolute and relative files (including on windows) + single_location = parse.urlparse(filename, "") + if single_location.scheme == "" or len(single_location.scheme) == 1: + single_location = parse.urlparse( + parse.urljoin("file:", request.pathname2url(os.path.abspath(filename))) + ) + if single_location.scheme == "file": + if not os.path.exists(request.url2pathname(single_location.path)): + filename = request.url2pathname(single_location.path) + if not filename: + raise ValueError("File URL looks incorrect (potentially missing /)") + raise ValueError(f"File does not exist: {filename}") + return parse.urlunparse(single_location) + class BytesRequirement(interfaces.configuration.SimpleTypeRequirement): """A requirement type that contains a byte string.""" From a5e6c550e3fbaa0110d6398b57a570c16914f8aa Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Apr 2023 13:30:55 +0100 Subject: [PATCH 332/526] Automagic: Handle file swap locations and throw a warning if they don't exist --- volatility3/framework/automagic/windows.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index ccc8de2eb..a8530829b 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -401,7 +401,17 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): # Fill in the config if swap_location: context.config[current_layer_path] = current_layer_name - context.config[layer_loc_path] = swap_location + try: + context.config[ + layer_loc_path + ] = requirements.URIRequirement.location_from_file( + swap_location + ) + except ValueError: + vollog.warning( + f"Volatility swap_location {swap_location} could not be validated - swap layer disabled" + ) + continue context.config[ layer_class_path ] = "volatility3.framework.layers.physical.FileLayer" From 33f54f8cf7be2039c3e976917f66ced49ce8365b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 26 Apr 2023 00:33:35 +1000 Subject: [PATCH 333/526] Replace LinuxUtilities._do_get_path() with the new mountinfo _do_get_path() avoiding duplicate code. It also fixes issue #930 --- .../framework/plugins/linux/mountinfo.py | 40 +---- .../framework/symbols/linux/__init__.py | 149 ++++++++++-------- 2 files changed, 84 insertions(+), 105 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index ebd6e55a0..1de776412 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -9,6 +9,7 @@ from typing import Tuple, List, Iterable, Union from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins +from volatility3.framework.symbols import linux from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) @@ -71,40 +72,9 @@ class MountInfo(plugins.PluginInterface): ), ] - @classmethod - def _do_get_path(cls, mnt, fs_root) -> Union[None, str]: - """It mimics the Linux kernel prepend_path function.""" - vfsmnt = mnt.mnt - dentry = vfsmnt.get_mnt_root() - - path_reversed = [] - while dentry != fs_root.dentry or vfsmnt.vol.offset != fs_root.mnt: - if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): - parent = mnt.get_mnt_parent().dereference() - # Escaped? - if dentry != vfsmnt.get_mnt_root(): - return None - - # Global root? - if mnt.vol.offset != parent.vol.offset: - dentry = mnt.get_mnt_mountpoint() - mnt = parent - vfsmnt = mnt.mnt - continue - - return None - - parent = dentry.d_parent - dname = dentry.d_name.name_as_str() - path_reversed.append(dname.strip("/")) - dentry = parent - - path = "/" + "/".join(reversed(path_reversed)) - return path - @classmethod def get_mountinfo( - cls, mnt, task + cls, mnt, task, context ) -> Union[ None, Tuple[int, int, str, str, str, List[str], List[str], str, str, List[str]] ]: @@ -115,8 +85,8 @@ class MountInfo(plugins.PluginInterface): if not mnt_root: return None - path_root = cls._do_get_path(mnt, task.fs.root) - if path_root is None: + path_root = linux.LinuxUtilities._get_path_root(context, mnt, task.fs.root) + if not path_root: return None mnt_root_path = mnt_root.path() @@ -207,7 +177,7 @@ class MountInfo(plugins.PluginInterface): if mnt_ns_ids and mnt_ns_id not in mnt_ns_ids: continue - mnt_info = self.get_mountinfo(mnt, task) + mnt_info = MountInfo.get_mountinfo(mnt, task, self.context) if mnt_info is None: continue diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index ce07167e5..80f5e9990 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,11 +1,11 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Iterator, List, Tuple, Optional +from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects -from volatility3.framework.objects import utility +from volatility3.framework.objects import utility, Pointer from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -59,83 +59,92 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) - # based on __d_path from the Linux kernel @classmethod - def _do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> str: - ret_path: List[str] = [] - - while dentry != rdentry or vfsmnt != rmnt: - dname = dentry.path() - if dname == "": - break - - ret_path.insert(0, dname.strip("/")) - if dentry == vfsmnt.get_mnt_root() or dentry == dentry.d_parent: - if vfsmnt.get_mnt_parent() == vfsmnt: - break - - dentry = vfsmnt.get_mnt_mountpoint() - vfsmnt = vfsmnt.get_mnt_parent() - - continue - - parent = dentry.d_parent - dentry = parent - - # if we did not gather any valid dentrys in the path, then the entire file is - # either 1) smeared out of memory or 2) de-allocated and corresponding structures overwritten - # we return an empty string in this case to avoid confusion with something like a handle to the root - # directory (e.g., "/") - if not ret_path: - return "" - - ret_val = "/".join([str(p) for p in ret_path if p != ""]) - - if ret_val.startswith(("socket:", "pipe:")): - if ret_val.find("]") == -1: - try: - inode = dentry.d_inode - ino = inode.i_ino - except exceptions.InvalidAddressException: - ino = 0 - - ret_val = ret_val[:-1] + f":[{ino}]" - else: - ret_val = ret_val.replace("/", "") - - elif ret_val != "inotify": - ret_val = "/" + ret_val - - return ret_val - - # method used by 'older' kernels - # TODO: lookup when dentry_operations->d_name was merged into the mainline kernel for exact version - @classmethod - def _get_path_file(cls, task, filp) -> str: + def _get_path_file(cls, context, task, filp) -> str: rdentry = task.fs.get_root_dentry() rmnt = task.fs.get_root_mnt() - dentry = filp.get_dentry() vfsmnt = filp.get_vfsmnt() + dentry = filp.get_dentry() - return LinuxUtilities._do_get_path(rdentry, rmnt, dentry, vfsmnt) + return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt, context) + + @classmethod + def _get_path_root(cls, context, mnt, fs_root) -> str: + rdentry = fs_root.dentry + rmnt = fs_root.mnt + vfsmnt = mnt.mnt + dentry = vfsmnt.mnt_root + + return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt, context) + + @classmethod + def _get_vmlinux_from_volobj(cls, volobj, context): + symbol_table_arr = volobj.vol.type_name.split("!", 1) + symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None + + module_names = context.modules.get_modules_by_symbol_tables(symbol_table) + module_names = list(module_names) + + if not module_names: + raise ValueError(f"No module using the symbol table '{symbol_table}'") + + kernel_module_name = module_names[0] + kernel = context.modules[kernel_module_name] + + return kernel + + @classmethod + def _get_mnt_from_vfsmnt(cls, vfsmnt, dentry, context): + vmlinux = cls._get_vmlinux_from_volobj(dentry, context) + + # When it's called from _get_path_file(), 'vfsmnt' is a Pointer + # struct file->f_path->mnt is "struct vfsmount *". + # However, when called from _get_path_root() + # struct mount -> mnt is "struct vfsmount" + vfsmnt_ptr = vfsmnt if type(vfsmnt) == Pointer else vfsmnt.vol.offset + + mnt = cls.container_of(vfsmnt_ptr, "mount", "mnt", vmlinux) + + return mnt + + @classmethod + def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt, context) -> Union[None, str]: + """It mimics the Linux kernel prepend_path function.""" + + mnt = cls._get_mnt_from_vfsmnt(vfsmnt, dentry, context) + + path_reversed = [] + while dentry != rdentry or vfsmnt.vol.offset != rmnt: + if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): + parent = mnt.get_mnt_parent().dereference() + # Escaped? + if dentry != vfsmnt.get_mnt_root(): + break + + # Global root? + if mnt.vol.offset != parent.vol.offset: + dentry = mnt.get_mnt_mountpoint() + mnt = parent + vfsmnt = mnt.mnt + continue + + break + + parent = dentry.d_parent + dname = dentry.d_name.name_as_str() + path_reversed.append(dname.strip("/")) + dentry = parent + + path = "/" + "/".join(reversed(path_reversed)) + return path @classmethod def _get_new_sock_pipe_path(cls, context, task, filp) -> str: dentry = filp.get_dentry() + kernel_module = cls._get_vmlinux_from_volobj(dentry, context) + sym_addr = dentry.d_op.d_dname - - symbol_table_arr = sym_addr.vol.type_name.split("!") - symbol_table = None - if len(symbol_table_arr) == 2: - symbol_table = symbol_table_arr[0] - - for module_name in context.modules.get_modules_by_symbol_tables(symbol_table): - kernel_module = context.modules[module_name] - break - else: - raise ValueError(f"No module using the symbol table {symbol_table}") - symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) if len(symbs) == 1: @@ -151,7 +160,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): pre_name = "pipe" elif sym == "simple_dname": - pre_name = cls._get_path_file(task, filp) + pre_name = cls._get_path_file(context, task, filp) else: pre_name = f"" @@ -192,7 +201,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): if dname_is_valid: ret = LinuxUtilities._get_new_sock_pipe_path(context, task, filp) else: - ret = LinuxUtilities._get_path_file(task, filp) + ret = LinuxUtilities._get_path_file(context, task, filp) return ret From b2d33c2cb0cf8535647f77feac35c44bc124ace5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 26 Apr 2023 00:36:58 +1000 Subject: [PATCH 334/526] Check if 'mnt_namespace' has the 'ns' member before using it --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 5ab8f1aa0..da5bf5dad 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -825,7 +825,7 @@ class mnt_namespace(objects.StructType): def get_inode(self): if self.has_member("proc_inum"): return self.proc_inum - elif self.ns.has_member("inum"): + elif self.has_member("ns") and self.ns.has_member("inum"): return self.ns.inum else: raise AttributeError("Unable to find mnt_namespace inode") From 317f565685c4d63b26d315cc77f74afcb65a525b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 30 Apr 2023 08:28:22 +0100 Subject: [PATCH 335/526] Core: Renable the import protection for volatility3.framework.plugins In commit 21d916b (about 10 months ago) the logic for the warning about volatility3.framework importing was disabled. This re-enables it. Also updates to support pyinstaller 5.10 function stack depth. Closes #944 --- volatility3/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/__init__.py b/volatility3/__init__.py index 28df7da5a..94a6721e1 100644 --- a/volatility3/__init__.py +++ b/volatility3/__init__.py @@ -38,10 +38,13 @@ class WarningFindSpec(abc.MetaPathFinder): """Mock find_spec method that just checks the name, this must go first.""" if fullname.startswith("volatility3.framework.plugins."): - warning = "Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins" + warning = f"Import {fullname}: Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins" # Pyinstaller uses walk_packages/_collect_submodules to import, but needs to read the modules to figure out dependencies # As such, we only print the warning when directly imported rather than from within walk_packages/_collect_submodules - if inspect.stack()[-2].function in ["walk_packages", "_collect_submodules"]: + if inspect.stack()[-2].function not in [ + "walk_packages", + "_collect_submodules", + ] and inspect.stack()[-3].function not in ["_collect_submodules"]: raise Warning(warning) From 77081d3f6832d187eeff5fc0c2d7b64c2655a4b8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 May 2023 05:51:24 +0900 Subject: [PATCH 336/526] Add: code comment for plugins --- volatility3/framework/plugins/windows/crashinfo.py | 2 ++ volatility3/framework/plugins/windows/ldrmodules.py | 2 ++ volatility3/plugins/windows/statistics.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/volatility3/framework/plugins/windows/crashinfo.py b/volatility3/framework/plugins/windows/crashinfo.py index a9f32f63d..4ecd85087 100644 --- a/volatility3/framework/plugins/windows/crashinfo.py +++ b/volatility3/framework/plugins/windows/crashinfo.py @@ -14,6 +14,8 @@ vollog = logging.getLogger(__name__) class Crashinfo(interfaces.plugins.PluginInterface): + """Lists the information from a Windows crash dump.""" + _required_framework_version = (2, 0, 0) @classmethod diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 9642810a5..3c2b8a42d 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -7,6 +7,8 @@ from volatility3.plugins.windows import pslist, vadinfo class LdrModules(interfaces.plugins.PluginInterface): + """Lists the loaded modules in a particular windows memory image.""" + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index e921b3565..4cc05440c 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -13,6 +13,8 @@ vollog = logging.getLogger(__name__) class Statistics(plugins.PluginInterface): + """Lists statistics about the memory space.""""" + _required_framework_version = (2, 0, 0) @classmethod From 79cd3fe2feddf4f1041f2941eb34149c5083ab58 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 May 2023 05:57:56 +0900 Subject: [PATCH 337/526] Lint: black issue for windows.statistics --- volatility3/plugins/windows/statistics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 4cc05440c..9915312e3 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -13,7 +13,7 @@ vollog = logging.getLogger(__name__) class Statistics(plugins.PluginInterface): - """Lists statistics about the memory space.""""" + """Lists statistics about the memory space.""" _required_framework_version = (2, 0, 0) From c06836e45dfe56dfc5c5dc9977cdc638aed4785e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 May 2023 05:58:57 +0900 Subject: [PATCH 338/526] Lint: black issue for windows.ldrmodules --- volatility3/framework/plugins/windows/ldrmodules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 3c2b8a42d..4c8456fa9 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -8,7 +8,7 @@ from volatility3.plugins.windows import pslist, vadinfo class LdrModules(interfaces.plugins.PluginInterface): """Lists the loaded modules in a particular windows memory image.""" - + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) From 159e5a2fbd7393934f73a50b6512df9334357e27 Mon Sep 17 00:00:00 2001 From: cpuu Date: Thu, 4 May 2023 11:59:30 +0900 Subject: [PATCH 339/526] Fix requirements.URLRequirement to requirements.URIRequirement [Refactor] Fix requirements.URLRequirement to requirements.URIRequirement in single_location assignment --- volatility3/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 99052d82e..9bfd14c6c 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -353,7 +353,7 @@ class CommandLine: ### if args.file: try: - single_location = requirements.URLRequirement.location_from_file( + single_location = requirements.URIRequirement.location_from_file( args.file ) ctx.config["automagic.LayerStacker.single_location"] = single_location From 14f449367d68c38211c5cba095dee41d38c60f0a Mon Sep 17 00:00:00 2001 From: cpuu Date: Thu, 4 May 2023 12:52:00 +0900 Subject: [PATCH 340/526] Delete getting-started-macos-tutorial.rst --- doc/source/getting-started-macos-tutorial.rst | 154 ------------------ 1 file changed, 154 deletions(-) delete mode 100644 doc/source/getting-started-macos-tutorial.rst diff --git a/doc/source/getting-started-macos-tutorial.rst b/doc/source/getting-started-macos-tutorial.rst deleted file mode 100644 index bc0cb1b92..000000000 --- a/doc/source/getting-started-macos-tutorial.rst +++ /dev/null @@ -1,154 +0,0 @@ -macOS Tutorial -============== - -This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite. - -Acquiring memory ----------------- - -Volatility3 does not provide the ability to acquire memory. The example below is an open source tool. Other commercial tools are also available. - -* `osxpmem `_ - - - -Procedure to create symbol tables for macOS --------------------------------------------- - -To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. - -.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , - which is built and maintained by `volatilityfoundation `_. - After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/mac``. - If necessary create a mac directory under the symbols directory (this will become unnecessary in future versions). - - -Listing plugins ---------------- - -The following is a sample of the macOS plugins available for volatility3, it is not complete and more more plugins may -be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. -For plugin requests, please create an issue with a description of the requested plugin. - -.. code-block:: shell-session - - $ python3 vol.py --help | grep -i mac. | head -n 5 - mac.bash.Bash Recovers bash command history from memory. - mac.check_syscall.Check_syscall - mac.check_sysctl.Check_sysctl - mac.check_trap_table.Check_trap_table - -.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of macOS plugins. - - -Using plugins -------------- - -The following is the syntax to run the volatility CLI. - -.. code-block:: shell-session - - $ python3 vol.py -f - - -Example -------- - -banners -~~~~~~~ - -In this example we will be using a memory dump from the Securinets CTF Quals 2019 Challenge called Contact_me. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge. -Thanks go to `stuxnet `_ for providing this memory dump and `writeup `_. - - -.. code-block:: shell-session - - $ python3 vol.py -f contact_me banners.Banners - - Volatility 3 Framework 2.1.0 - - Progress: 100.00 PDB scanning finished - Offset Banner - - 0x4d2c7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - 0xb42b180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - 0xcda9100 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - 0x1275e7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - 0x1284fba4 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - 0x34ad0180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - - -The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols/mac`` directory. - -mac.pslist -~~~~~~~~~~~~ - -.. code-block:: shell-session - - $ python3 vol.py -f contact_me mac.pslist - - Volatility 3 Framework 2.1.0 Stacking attempts finished - - PID PPID COMM - - 0 0 kernel_task - 1 0 launchd - 35 1 UserEventAgent - 38 1 kextd - 39 1 fseventsd - 37 1 uninstalld - 45 1 configd - 46 1 powerd - 52 1 logd - 58 1 warmd - ..... - -``mac.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. - -mac.pstree -~~~~~~~~~~~~ - -.. code-block:: shell-session - - $ python3 vol.py -f contact_me mac.pstree - Volatility 3 Framework 2.1.0 - Progress: 100.00 Stacking attempts finished - PID PPID COMM - - 35 1 UserEventAgent - 38 1 kextd - 39 1 fseventsd - 37 1 uninstalld - 204 1 softwareupdated - * 449 204 SoftwareUpdateCo - 337 1 system_installd - * 455 337 update_dyld_shar - -``mac.pstree`` helps us to display the parent child relationships between processes. - -mac.ifconfig -~~~~~~~~~~ - -we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. - - -.. code-block:: shell-session - - $ python3 vol.py -f contact_me mac.ifconfig - - Volatility 3 Framework 2.1.0 - Progress: 100.00 Stacking attempts finished - Interface IP Address Mac Address Promiscuous - - lo0 False - lo0 127.0.0.1 False - lo0 ::1 False - lo0 fe80:1::1 False - gif0 False - stf0 False - en0 00:0C:29:89:8B:F0 00:0C:29:89:8B:F0 False - en0 fe80:4::10fb:c89d:217f:52ae 00:0C:29:89:8B:F0 False - en0 192.168.140.128 00:0C:29:89:8B:F0 False - utun0 False - utun0 fe80:5::2a95:bb15:87e3:977c False From c9e0b6695f2fe88eb2021df504c3ed5eece559e7 Mon Sep 17 00:00:00 2001 From: cpuu Date: Thu, 4 May 2023 12:52:16 +0900 Subject: [PATCH 341/526] Update index.rst --- doc/source/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index e096731c7..9b1d05858 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -26,7 +26,6 @@ There is also some information to get you started quickly: getting-started-linux-tutorial getting-started-windows-tutorial - getting-started-macos-tutorial .. toctree:: From 1ad8c950dab7005e7f603bbf1c34c962c27eb943 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 4 May 2023 13:47:57 +0300 Subject: [PATCH 342/526] Added modules name flag --- volatility3/framework/plugins/windows/modules.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index eba6d1ce7..e25d0425e 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -42,6 +42,7 @@ class Modules(interfaces.plugins.PluginInterface): default=False, optional=True, ), + requirements.StringRequirement(name="name", description="module name/sub string", optional=True, default=""), ] def _generator(self): @@ -64,6 +65,9 @@ class Modules(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: FullDllName = "" + if self.config['name'] and self.config['name'] not in BaseDllName: + continue + file_output = "Disabled" if self.config["dump"]: file_handle = dlllist.DllList.dump_pe( From 9249758dde5c7a1325f25b16ca49b0577d00d714 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 4 May 2023 15:15:23 +0300 Subject: [PATCH 343/526] formatting --- volatility3/framework/plugins/windows/modules.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index e25d0425e..a1d480bb4 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -42,7 +42,12 @@ class Modules(interfaces.plugins.PluginInterface): default=False, optional=True, ), - requirements.StringRequirement(name="name", description="module name/sub string", optional=True, default=""), + requirements.StringRequirement( + name="name", + description="module name/sub string", + optional=True, + default="", + ), ] def _generator(self): @@ -66,7 +71,7 @@ class Modules(interfaces.plugins.PluginInterface): FullDllName = "" if self.config['name'] and self.config['name'] not in BaseDllName: - continue + continue file_output = "Disabled" if self.config["dump"]: From 8952c6c985c8cafd038ae4ed8db0ea4db6b04517 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 4 May 2023 15:18:37 +0300 Subject: [PATCH 344/526] formatting --- volatility3/framework/plugins/windows/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index a1d480bb4..879da4a5f 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -70,7 +70,7 @@ class Modules(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: FullDllName = "" - if self.config['name'] and self.config['name'] not in BaseDllName: + if self.config["name"] and self.config["name"] not in BaseDllName: continue file_output = "Disabled" From 093fc9ab797e2953ecdfb8cd17c096b3bf573a29 Mon Sep 17 00:00:00 2001 From: cpuu Date: Fri, 5 May 2023 23:39:33 +0900 Subject: [PATCH 345/526] Add tutorial for macOS Analysis Add tutorial for macOS Analysis --- doc/source/getting-started-mac-tutorial.rst | 155 ++++++++++++++++++++ doc/source/index.rst | 1 + 2 files changed, 156 insertions(+) create mode 100644 doc/source/getting-started-mac-tutorial.rst diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst new file mode 100644 index 000000000..3e650fede --- /dev/null +++ b/doc/source/getting-started-mac-tutorial.rst @@ -0,0 +1,155 @@ +macOS Tutorial +============== + +This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite. + +Acquiring memory +---------------- + +Volatility3 does not provide the ability to acquire memory. The example below is an open source tool. Other commercial tools are also available. + +* `osxpmem `_ + + + +Procedure to create symbol tables for macOS +-------------------------------------------- + +To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. + +.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , + which is built and maintained by `volatilityfoundation `_. + After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/mac``. + If necessary create a mac directory under the symbols directory (this will become unnecessary in future versions). + + +Listing plugins +--------------- + +The following is a sample of the macOS plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. + +.. code-block:: shell-session + + $ python3 vol.py --help | grep -i mac. | head -n 4 + mac.bash.Bash Recovers bash command history from memory. + mac.check_syscall.Check_syscall + mac.check_sysctl.Check_sysctl + mac.check_trap_table.Check_trap_table + +.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of macOS plugins. + + +Using plugins +------------- + +The following is the syntax to run the volatility CLI. + +.. code-block:: shell-session + + $ python3 vol.py -f + + +Example +------- + +banners +~~~~~~~ + +In this example we will be using a memory dump from the Securinets CTF Quals 2019 Challenge called Contact_me. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge. +Thanks go to `stuxnet `_ for providing this memory dump and `writeup `_. + + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me banners.Banners + + Volatility 3 Framework 2.4.2 + + Progress: 100.00 PDB scanning finished + Offset Banner + + 0x4d2c7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0xb42b180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0xcda9100 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x1275e7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x1284fba4 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x34ad0180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + + +The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols/mac`` directory. + +mac.pslist +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.pslist.PsList + + Volatility 3 Framework 2.4.2 + Progress: 100.00 Stacking attempts finished + + PID PPID COMM + + 0 0 kernel_task + 1 0 launchd + 35 1 UserEventAgent + 38 1 kextd + 39 1 fseventsd + 37 1 uninstalld + 45 1 configd + 46 1 powerd + 52 1 logd + 58 1 warmd + ..... + +``mac.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. + +mac.pstree +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.pstree.PsTree + Volatility 3 Framework 2.4.2 + Progress: 100.00 Stacking attempts finished + PID PPID COMM + + 35 1 UserEventAgent + 38 1 kextd + 39 1 fseventsd + 37 1 uninstalld + 204 1 softwareupdated + * 449 204 SoftwareUpdateCo + 337 1 system_installd + * 455 337 update_dyld_shar + +``mac.pstree`` helps us to display the parent child relationships between processes. + +mac.ifconfig +~~~~~~~~~~ + +we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. + + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.ifconfig.Ifconfig + + Volatility 3 Framework 2.4.2 + Progress: 100.00 Stacking attempts finished + Interface IP Address Mac Address Promiscuous + + lo0 False + lo0 127.0.0.1 False + lo0 ::1 False + lo0 fe80:1::1 False + gif0 False + stf0 False + en0 00:0C:29:89:8B:F0 00:0C:29:89:8B:F0 False + en0 fe80:4::10fb:c89d:217f:52ae 00:0C:29:89:8B:F0 False + en0 192.168.140.128 00:0C:29:89:8B:F0 False + utun0 False + utun0 fe80:5::2a95:bb15:87e3:977c False \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 9b1d05858..7f35e9bcb 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -25,6 +25,7 @@ There is also some information to get you started quickly: :caption: Getting Started getting-started-linux-tutorial + getting-started-mac-tutorial getting-started-windows-tutorial From 7063e6094481e940026b9654f4a52580aa6805cb Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 6 May 2023 14:12:09 +0200 Subject: [PATCH 346/526] mountinfo improvements: + Fixes issue with repeated path suffixes. + Adds support for older kernels (<3.3.8) and + Adds a function which simplify the way the framework internally can gets a context and a vmlinux. This is obtaining the context and symbol space from the same vol object instead of drag a context everywhere. This changes also affects other plugins such as elfs, malfind and proc.maps. It also adds doc strings to some of the existent functions. --- volatility3/framework/plugins/linux/elfs.py | 2 +- .../framework/plugins/linux/malfind.py | 2 +- .../framework/plugins/linux/mountinfo.py | 4 +- volatility3/framework/plugins/linux/proc.py | 2 +- .../framework/symbols/linux/__init__.py | 161 ++++++++----- .../symbols/linux/extensions/__init__.py | 216 ++++++++++++++++-- 6 files changed, 308 insertions(+), 79 deletions(-) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 822a69dd6..fa14dcd49 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -58,7 +58,7 @@ class Elfs(plugins.PluginInterface): ): continue - path = vma.get_name(self.context, task) + path = vma.get_name(task) yield ( 0, diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 18237b80c..552fb8f53 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -47,7 +47,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.mm.get_mmap_iter(): - if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]": + if vma.is_suspicious() and vma.get_name(task) != "[vdso]": data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 1de776412..dfb2e23b4 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -74,7 +74,7 @@ class MountInfo(plugins.PluginInterface): @classmethod def get_mountinfo( - cls, mnt, task, context + cls, mnt, task ) -> Union[ None, Tuple[int, int, str, str, str, List[str], List[str], str, str, List[str]] ]: @@ -85,7 +85,7 @@ class MountInfo(plugins.PluginInterface): if not mnt_root: return None - path_root = linux.LinuxUtilities._get_path_root(context, mnt, task.fs.root) + path_root = linux.LinuxUtilities._get_path_mnt(task, mnt) if not path_root: return None diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 9d8af482e..fa7bc1629 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -59,7 +59,7 @@ class Maps(plugins.PluginInterface): minor = inode_object.i_sb.minor inode = inode_object.i_ino - path = vma.get_name(self.context, task) + path = vma.get_name(task) yield ( 0, diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 80f5e9990..486314dd5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -5,7 +5,7 @@ from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects -from volatility3.framework.objects import utility, Pointer +from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -60,75 +60,74 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) @classmethod - def _get_path_file(cls, context, task, filp) -> str: + def _get_path_file(cls, task, filp) -> str: + """Returns the file pathname relative to the task's root directory. + + Args: + task (task_struct): A reference task + filp (file *): A pointer to an open file + + Returns: + str: File pathname relative to the task's root directory. + """ rdentry = task.fs.get_root_dentry() rmnt = task.fs.get_root_mnt() vfsmnt = filp.get_vfsmnt() dentry = filp.get_dentry() - return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt, context) + return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt) @classmethod - def _get_path_root(cls, context, mnt, fs_root) -> str: - rdentry = fs_root.dentry - rmnt = fs_root.mnt - vfsmnt = mnt.mnt - dentry = vfsmnt.mnt_root + def _get_path_mnt(cls, task, mnt) -> str: + """Returns the mount point pathname relative to the task's root directory. - return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt, context) + Args: + task (task_struct): A reference task + mnt (vfsmount or mount): A mounted filesystem or a mount point. + - kernels < 3.3.8 type is 'vfsmount' + - kernels >= 3.3.8 type is 'mount' + + Returns: + str: Pathname of the mount point relative to the task's root directory. + """ + rdentry = task.fs.get_root_dentry() + rmnt = task.fs.get_root_mnt() + + vfsmnt = mnt.get_vfsmnt_current() + dentry = mnt.get_dentry_current() + + return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt) @classmethod - def _get_vmlinux_from_volobj(cls, volobj, context): - symbol_table_arr = volobj.vol.type_name.split("!", 1) - symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None + def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> Union[None, str]: + """Returns a pathname of the mount point or file + It mimics the Linux kernel prepend_path function. - module_names = context.modules.get_modules_by_symbol_tables(symbol_table) - module_names = list(module_names) + Args: + rdentry (dentry *): A pointer to the root dentry + rmnt (vfsmount *): A pointer to the root vfsmount + dentry (dentry *): A pointer to the dentry + vfsmnt (vfsmount *): A pointer to the vfsmount - if not module_names: - raise ValueError(f"No module using the symbol table '{symbol_table}'") - - kernel_module_name = module_names[0] - kernel = context.modules[kernel_module_name] - - return kernel - - @classmethod - def _get_mnt_from_vfsmnt(cls, vfsmnt, dentry, context): - vmlinux = cls._get_vmlinux_from_volobj(dentry, context) - - # When it's called from _get_path_file(), 'vfsmnt' is a Pointer - # struct file->f_path->mnt is "struct vfsmount *". - # However, when called from _get_path_root() - # struct mount -> mnt is "struct vfsmount" - vfsmnt_ptr = vfsmnt if type(vfsmnt) == Pointer else vfsmnt.vol.offset - - mnt = cls.container_of(vfsmnt_ptr, "mount", "mnt", vmlinux) - - return mnt - - @classmethod - def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt, context) -> Union[None, str]: - """It mimics the Linux kernel prepend_path function.""" - - mnt = cls._get_mnt_from_vfsmnt(vfsmnt, dentry, context) + Returns: + str: Pathname of the mount point or file + """ path_reversed = [] - while dentry != rdentry or vfsmnt.vol.offset != rmnt: + while dentry != rdentry or not vfsmnt.is_equal(rmnt): if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): - parent = mnt.get_mnt_parent().dereference() # Escaped? if dentry != vfsmnt.get_mnt_root(): break # Global root? - if mnt.vol.offset != parent.vol.offset: - dentry = mnt.get_mnt_mountpoint() - mnt = parent - vfsmnt = mnt.mnt - continue + if not vfsmnt.has_parent(): + break - break + dentry = vfsmnt.get_dentry_parent() + vfsmnt = vfsmnt.get_vfsmnt_parent() + + continue parent = dentry.d_parent dname = dentry.d_name.name_as_str() @@ -139,10 +138,19 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return path @classmethod - def _get_new_sock_pipe_path(cls, context, task, filp) -> str: + def _get_new_sock_pipe_path(cls, task, filp) -> str: + """Returns the sock pipe pathname relative to the task's root directory. + + Args: + task (task_struct): A reference task + filp (file *): A pointer to a sock pipe open file + + Returns: + str: Sock pipe pathname relative to the task's root directory. + """ dentry = filp.get_dentry() - kernel_module = cls._get_vmlinux_from_volobj(dentry, context) + kernel_module = cls.get_vmlinux_from_volobj(dentry) sym_addr = dentry.d_op.d_dname symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) @@ -160,7 +168,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): pre_name = "pipe" elif sym == "simple_dname": - pre_name = cls._get_path_file(context, task, filp) + pre_name = cls._get_path_file(task, filp) else: pre_name = f"" @@ -172,10 +180,20 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return ret - # a 'file' structure doesn't have enough information to properly restore its full path - # we need the root mount information from task_struct to determine this @classmethod - def path_for_file(cls, context, task, filp) -> str: + def path_for_file(cls, task, filp) -> str: + """Returns a file (or sock pipe) pathname relative to the task's root directory. + + A 'file' structure doesn't have enough information to properly restore its + full path we need the root mount information from task_struct to determine this + + Args: + task (task_struct): A reference task + filp (file *): A pointer to an open file + + Returns: + str: A file (or sock pipe) pathname relative to the task's root directory. + """ try: dentry = filp.get_dentry() except exceptions.InvalidAddressException: @@ -199,9 +217,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): dname_is_valid = False if dname_is_valid: - ret = LinuxUtilities._get_new_sock_pipe_path(context, task, filp) + ret = LinuxUtilities._get_new_sock_pipe_path(task, filp) else: - ret = LinuxUtilities._get_path_file(context, task, filp) + ret = LinuxUtilities._get_path_file(task, filp) return ret @@ -234,7 +252,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): for fd_num, filp in enumerate(fds): if filp != 0: - full_path = LinuxUtilities.path_for_file(context, task, filp) + full_path = LinuxUtilities.path_for_file(task, filp) yield fd_num, filp, full_path @@ -357,3 +375,30 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return vmlinux.object( object_type=type_name, offset=container_addr, absolute=True ) + + @classmethod + def get_vmlinux_from_volobj(cls, volobj): + """Get the vmlinux from a vol obj + + Args: + volobj (vol object): A vol object + + Raises: + ValueError: If it cannot obtain any module from the symbol table + + Returns: + volatility3.framework.contexts.Module: A kernel object (vmlinux) + """ + symbol_table_arr = volobj.vol.type_name.split("!", 1) + symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None + + module_names = volobj._context.modules.get_modules_by_symbol_tables(symbol_table) + module_names = list(module_names) + + if not module_names: + raise ValueError(f"No module using the symbol table '{symbol_table}'") + + kernel_module_name = module_names[0] + kernel = volobj._context.modules[kernel_module_name] + + return kernel diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index da5bf5dad..b2225f764 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -435,9 +435,9 @@ class vm_area_struct(objects.StructType): return self.vm_pgoff << constants.linux.PAGE_SHIFT - def get_name(self, context, task): + def get_name(self, task): if self.vm_file != 0: - fname = linux.LinuxUtilities.path_for_file(context, task, self.vm_file) + fname = linux.LinuxUtilities.path_for_file(task, self.vm_file) elif self.vm_start <= task.mm.start_brk and self.vm_end >= task.mm.brk: fname = "[heap]" elif self.vm_start <= task.mm.start_stack <= self.vm_end: @@ -544,6 +544,7 @@ class struct_file(objects.StructType): raise AttributeError("Unable to find file -> dentry") def get_vfsmnt(self) -> interfaces.objects.ObjectInterface: + """Returns the fs (vfsmount) where this file is mounted""" if self.has_member("f_vfsmnt"): return self.f_vfsmnt elif self.has_member("f_path"): @@ -675,11 +676,70 @@ class mount(objects.StructType): raise AttributeError("Unable to find mount -> mount flags") def get_mnt_parent(self): + """Gets the fs where we are mounted on + + Returns: + A 'mount *' + """ return self.mnt_parent def get_mnt_mountpoint(self): + """Gets the dentry of the mountpoint + + Returns: + A 'dentry *' + """ + return self.mnt_mountpoint + def get_parent_mount(self): + return self.mnt.get_parent_mount() + + def has_parent(self) -> bool: + """Checks if this mount has a parent + + Returns: + bool: 'True' if this mount has a parent + """ + return self.mnt_parent != self.vol.offset + + def get_vfsmnt_current(self): + """Returns the fs where we are mounted on + + Returns: + A 'vfsmount' + """ + return self.mnt + + def get_vfsmnt_parent(self): + """Gets the parent fs (vfsmount) to where it's mounted on + + Returns: + A 'vfsmount' + """ + + return self.get_mnt_parent().get_vfsmnt_current() + + def get_dentry_current(self): + """Returns the root of the mounted tree + + Returns: + A 'dentry *' + """ + vfsmnt = self.get_vfsmnt_current() + dentry = vfsmnt.mnt_root + + return dentry + + def get_dentry_parent(self): + """Returns the parent root of the mounted tree + + Returns: + A 'dentry *' + """ + + return self.get_mnt_parent().get_dentry_current() + def get_flags_access(self) -> str: return "ro" if self.get_mnt_flags() & self.MNT_READONLY else "rw" @@ -703,9 +763,6 @@ class mount(objects.StructType): def get_devname(self) -> str: return utility.pointer_to_string(self.mnt_devname, count=255) - def has_parent(self) -> bool: - return self.vol.offset != self.mnt_parent - def get_dominating_id(self, root) -> int: """Get ID of closest dominating peer group having a representative under the given root.""" mnt_seen = set() @@ -783,24 +840,117 @@ class vfsmount(objects.StructType): and self.get_mnt_parent() != 0 ) - def _get_real_mnt(self): - table_name = self.vol.type_name.split(constants.BANG)[0] - mount_struct = f"{table_name}{constants.BANG}mount" - offset = self._context.symbol_space.get_type( - mount_struct - ).relative_child_offset("mnt") + def _is_kernel_prior_to_struct_mount(self) -> bool: + """Helper to distinguish between kernels prior to version 3.3.8 that + lacked the 'mount' structure and later versions that have it. - return self._context.object( - mount_struct, self.vol.layer_name, offset=self.vol.offset - offset - ) + The 'mnt_parent' member was moved from struct 'vfsmount' to struct + 'mount' when the latter was introduced. + + Alternatively, vmlinux.has_type('mount') can be used here but it is faster. + + Returns: + bool: 'True' if the kernel + """ + + return self.has_member("mnt_parent") + + def is_equal(self, vfsmount_ptr) -> bool: + """Helper to make sure it is comparing two pointers to 'vfsmount'. + + Depending on the kernel version, the calling object (self) could be + a 'vfsmount *' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust + in the framework "auto" dereferencing ability to assure that when we + reach this point 'self' will be a 'vfsmount' already and self.vol.offset + a 'vfsmount *' and not a 'vfsmount **'. The argument must be a 'vfsmount *'. + Typically, it's called from do_get_path(). + + Args: + vfsmount_ptr (vfsmount *): A pointer to a 'vfsmount' + + Raises: + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount *' + + Returns: + bool: 'True' if the given argument points to the the same 'vfsmount' + as 'self'. + """ + if type(vfsmount_ptr) == objects.Pointer: + return self.vol.offset == vfsmount_ptr + else: + raise exceptions.VolatilityException("Unexpected argument type. It has to be a 'vfsmount *'") + + def _get_real_mnt(self): + """Gets the struct 'mount' containing this 'vfsmount'. + + It should be only called from kernels >= 3.3.8 when 'struct mount' was introduced. + + Returns: + mount: the struct 'mount' containing this 'vfsmount'. + """ + vmlinux = linux.LinuxUtilities.get_vmlinux_from_volobj(self) + return linux.LinuxUtilities.container_of(self.vol.offset, "mount", "mnt", vmlinux) + + def get_vfsmnt_current(self): + """Returns the current fs where we are mounted on + + Returns: + A 'vfsmount *' + """ + return self.get_mnt_parent() + + def get_vfsmnt_parent(self): + """Gets the parent fs (vfsmount) to where it's mounted on + + Returns: + For kernels < 3.3.8: A 'vfsmount *' + For kernels >= 3.3.8: A 'vfsmount' + """ + if self._is_kernel_prior_to_struct_mount(): + return self.get_mnt_parent() + else: + return self._get_real_mnt().get_vfsmnt_parent() + + def get_dentry_current(self): + """Returns the root of the mounted tree + + Returns: + A 'dentry *' + """ + if self._is_kernel_prior_to_struct_mount(): + return self.get_mnt_mountpoint() + else: + return self._get_real_mnt().get_dentry_current() + + def get_dentry_parent(self): + """Returns the parent root of the mounted tree + + Returns: + A 'dentry *' + """ + if self._is_kernel_prior_to_struct_mount(): + return self.get_mnt_mountpoint() + else: + return self._get_real_mnt().get_mnt_mountpoint() def get_mnt_parent(self): - if self.has_member("mnt_parent"): + """Gets the mnt_parent member. + + Returns: + For kernels < 3.3.8: A 'vfsmount *' + For kernels >= 3.3.8: A 'mount *' + """ + if self._is_kernel_prior_to_struct_mount(): return self.mnt_parent else: - return self._get_real_mnt().mnt_parent + return self._get_real_mnt().get_mnt_parent() def get_mnt_mountpoint(self): + """Gets the dentry of the mountpoint + + Returns: + A 'dentry *' + """ if self.has_member("mnt_mountpoint"): return self.mnt_mountpoint else: @@ -809,6 +959,40 @@ class vfsmount(objects.StructType): def get_mnt_root(self): return self.mnt_root + def has_parent(self) -> bool: + if self._is_kernel_prior_to_struct_mount(): + return self.mnt_parent != self.vol.offset + else: + return self._get_real_mnt().has_parent() + + def get_mnt_sb(self): + return self.mnt_sb + + def get_flags_access(self) -> str: + return "ro" if self.mnt_flags & mount.MNT_READONLY else "rw" + + def get_flags_opts(self) -> Iterable[str]: + flags = [ + mntflagtxt + for mntflag, mntflagtxt in mount.MNT_FLAGS.items() + if mntflag & self.mnt_flags != 0 + ] + return flags + + def get_mnt_flags(self): + return self.mnt_flags + + def is_shared(self) -> bool: + return self.get_mnt_flags() & mount.MNT_SHARED + + def is_unbindable(self) -> bool: + return self.get_mnt_flags() & mount.MNT_UNBINDABLE + + def is_slave(self) -> bool: + return self.mnt_master and self.mnt_master.vol.offset != 0 + + def get_devname(self) -> str: + return utility.pointer_to_string(self.mnt_devname, count=255) class kobject(objects.StructType): def reference_count(self): From 2d922a8cd645c318e4e10c5d286bc86589f64f8b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 6 May 2023 14:18:33 +0200 Subject: [PATCH 347/526] Improves the way the mount points and the namespaces are filtered. Previously, it was filtering by mount namespaces id. Even that approach was working as expected, it's not viable for older kernels were there was not a mount namespace ID. With this changes we filtered the mount points individually by the mount ID which is unique system wide, no mather the namespace to which it belongs to. --- .../framework/plugins/linux/mountinfo.py | 70 ++++++++++++------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index dfb2e23b4..e03659aec 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -12,6 +12,7 @@ from volatility3.framework.interfaces import plugins from volatility3.framework.symbols import linux from volatility3.plugins.linux import pslist + vollog = logging.getLogger(__name__) MountInfoData = namedtuple( @@ -140,30 +141,38 @@ class MountInfo(plugins.PluginInterface): ) def _get_tasks_mountpoints( - self, tasks: Iterable[interfaces.objects.ObjectInterface], per_namespace: bool + self, tasks: Iterable[interfaces.objects.ObjectInterface], filtered_by_pids: bool ): - seen_namespaces = set() + seen_mountpoints = set() for task in tasks: if not ( - task - and task.fs - and task.fs.root - and task.nsproxy - and task.nsproxy.mnt_ns + task and + task.fs and + task.fs.root and + task.nsproxy and + task.nsproxy.mnt_ns ): - # This task doesn't have all the information required + # This task doesn't have all the information required. + # It should be a kernel < 2.6.30 continue mnt_namespace = task.nsproxy.mnt_ns - mnt_ns_id = mnt_namespace.get_inode() - - if per_namespace: - if mnt_ns_id in seen_namespaces: - continue - else: - seen_namespaces.add(mnt_ns_id) + try: + mnt_ns_id = str(mnt_namespace.get_inode()) + except AttributeError: + mnt_ns_id = renderers.NotAvailableValue() for mount in mnt_namespace.get_mount_points(): + # When PIDs are filtered, it makes sense that the user want to + # see each of those processes mount points. So we don't filter + # by mount id in this case. + if not filtered_by_pids: + mnt_id = int(mount.mnt_id) + if mnt_id in seen_mountpoints: + continue + else: + seen_mountpoints.add(mnt_id) + yield task, mount, mnt_ns_id def _generator( @@ -171,13 +180,26 @@ class MountInfo(plugins.PluginInterface): tasks: Iterable[interfaces.objects.ObjectInterface], mnt_ns_ids: List[int], mount_format: bool, - per_namespace: bool, + filtered_by_pids: bool, ) -> Iterable[Tuple[int, Tuple]]: - for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, per_namespace): - if mnt_ns_ids and mnt_ns_id not in mnt_ns_ids: + warning_shown = False + for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, filtered_by_pids): + if ( + not warning_shown and + mnt_ns_ids and + isinstance(mnt_ns_id, renderers.NotAvailableValue) + ): + vollog.warning("Cannot filter by namespace id, it is not available in this kernel.") + warning_shown = True + + if ( + not isinstance(mnt_ns_id, renderers.NotAvailableValue) and + mnt_ns_ids and + mnt_ns_id not in mnt_ns_ids + ): continue - mnt_info = MountInfo.get_mountinfo(mnt, task, self.context) + mnt_info = self.get_mountinfo(mnt, task) if mnt_info is None: continue @@ -212,7 +234,7 @@ class MountInfo(plugins.PluginInterface): ] fields_values = [mnt_ns_id] - if not per_namespace: + if filtered_by_pids: fields_values.append(task.pid) fields_values.extend(extra_fields_values) @@ -228,14 +250,14 @@ class MountInfo(plugins.PluginInterface): self.context, self.config["kernel"], filter_func=pid_filter ) - columns = [("MNT_NS_ID", int)] + columns = [("MNT_NS_ID", str)] # The PID column does not make sense when a PID filter is not specified. In that case, the default behavior is # to displays the mountpoints per namespace. if pids: columns.append(("PID", int)) - per_namespace = False + filtered_by_pids = True else: - per_namespace = True + filtered_by_pids = False if self.config.get("mount-format"): extra_columns = [ @@ -262,5 +284,5 @@ class MountInfo(plugins.PluginInterface): columns.extend(extra_columns) return renderers.TreeGrid( - columns, self._generator(tasks, mount_ns_ids, mount_format, per_namespace) + columns, self._generator(tasks, mount_ns_ids, mount_format, filtered_by_pids) ) From a09f77897b24b4b0da06a8b45a353ee730cadd08 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 6 May 2023 14:21:58 +0200 Subject: [PATCH 348/526] sockstat improvements: + Fixes issues with netlink sockets for older kernels, supporting now kernels < 3.7.10 + Fixes issue with network namespace id for older kernels. --- .../framework/plugins/linux/sockstat.py | 26 +++++++++++++++---- .../symbols/linux/extensions/__init__.py | 24 ++++++++++++++++- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index f03a2ad8e..e37b8a1bb 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -28,7 +28,11 @@ class SockHandlers(interfaces.configuration.VersionableInterface): self._vmlinux = vmlinux self._task = task - netns_id = task.nsproxy.net_ns.get_inode() + try: + netns_id = task.nsproxy.net_ns.get_inode() + except AttributeError: + netns_id = NotAvailableValue() + self._netdevices = self._build_network_devices_map(netns_id) self._sock_family_handlers = { @@ -61,7 +65,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): self._vmlinux.symbol_table_name + constants.BANG + "net_device" ) for net_dev in net.dev_base_head.to_list(net_device_symname, "dev_list"): - if net.get_inode() != netns_id: + if isinstance(netns_id, NotAvailableValue) or net.get_inode() != netns_id: continue dev_name = utility.array_to_string(net_dev.name) netdevices_map[net_dev.ifindex] = dev_name @@ -227,14 +231,22 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if netlink_sock.groups: groups_bitmap = netlink_sock.groups.dereference() src_addr = f"groups:0x{groups_bitmap:08x}" - src_port = netlink_sock.portid + + try: + # Kernel >= 3.7.10 + src_port = netlink_sock.get_portid() + except AttributeError: + src_port = NotAvailableValue() dst_addr = f"group:0x{netlink_sock.dst_group:08x}" module = netlink_sock.module if module and module.name: module_name_str = utility.array_to_string(module.name) dst_addr = f"{dst_addr},lkm:{module_name_str}" - dst_port = netlink_sock.dst_portid + try: + dst_port = netlink_sock.get_dst_portid() + except AttributeError: + dst_port = NotAvailableValue() state = netlink_sock.get_state() @@ -518,7 +530,11 @@ class Sockstat(plugins.PluginInterface): protocol = child_sock.get_protocol() net = task.nsproxy.net_ns - netns_id = net.get_inode() + try: + netns_id = net.get_inode() + except AttributeError: + netns_id = NotAvailableValue() + yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields def _format_fields(self, sock_stat, protocol): diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b2225f764..d97916b74 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1028,10 +1028,13 @@ class mnt_namespace(objects.StructType): class net(objects.StructType): def get_inode(self): if self.has_member("proc_inum"): + # 3.8.13 <= kernel < 3.19.8 return self.proc_inum - elif self.ns.has_member("inum"): + elif self.has_member("ns") and self.ns.has_member("inum"): + # kernel >= 3.19.8 return self.ns.inum else: + # kernel < 3.8.13 raise AttributeError("Unable to find net_namespace inode") @@ -1239,6 +1242,25 @@ class netlink_sock(objects.StructType): # Return the generic socket state return self.sk.sk_socket.get_state() + def get_portid(self): + if self.has_member("pid"): + # kernel < 3.7.10 + return self.pid + if self.has_member("portid"): + # kernel >= 3.7.10 + return self.portid + else: + raise AttributeError("Unable to find a source port id") + + def get_dst_portid(self): + if self.has_member("dst_pid"): + # kernel < 3.7.10 + return self.dst_pid + if self.has_member("dst_portid"): + # kernel >= 3.7.10 + return self.dst_portid + else: + raise AttributeError("Unable to find a destination port id") class vsock_sock(objects.StructType): def get_protocol(self): From c40b3a043f2c91506514c9fa7cdc7d3bcce81116 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 8 May 2023 05:07:19 +0300 Subject: [PATCH 349/526] CR fixes --- volatility3/framework/plugins/windows/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 879da4a5f..c1be1b0a6 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -46,7 +46,7 @@ class Modules(interfaces.plugins.PluginInterface): name="name", description="module name/sub string", optional=True, - default="", + default=None, ), ] From ea09c4732843f27830d2afe859203d825fa15b76 Mon Sep 17 00:00:00 2001 From: cpuu Date: Mon, 8 May 2023 17:32:14 +0900 Subject: [PATCH 350/526] Update getting-started-mac-tutorial.rst --- doc/source/getting-started-mac-tutorial.rst | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index 3e650fede..cb0dda845 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -17,16 +17,11 @@ Procedure to create symbol tables for macOS To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. -.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , - which is built and maintained by `volatilityfoundation `_. - After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/mac``. - If necessary create a mac directory under the symbols directory (this will become unnecessary in future versions). - Listing plugins --------------- -The following is a sample of the macOS plugins available for volatility3, it is not complete and more more plugins may +The following is a sample of the macOS plugins available for volatility3, it is not complete and more plugins may be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. For plugin requests, please create an issue with a description of the requested plugin. @@ -79,7 +74,7 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols/mac`` directory. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. mac.pslist ~~~~~~~~~~~~ @@ -131,9 +126,6 @@ mac.pstree mac.ifconfig ~~~~~~~~~~ -we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. - - .. code-block:: shell-session $ python3 vol.py -f contact_me mac.ifconfig.Ifconfig @@ -152,4 +144,6 @@ we can use the ``mac.ifconfig`` plugin to get information about the configuratio en0 fe80:4::10fb:c89d:217f:52ae 00:0C:29:89:8B:F0 False en0 192.168.140.128 00:0C:29:89:8B:F0 False utun0 False - utun0 fe80:5::2a95:bb15:87e3:977c False \ No newline at end of file + utun0 fe80:5::2a95:bb15:87e3:977c False + + we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. From e86ff963e2325d33bf2fff612e9f494afbea836b Mon Sep 17 00:00:00 2001 From: cpuu Date: Mon, 8 May 2023 17:43:24 +0900 Subject: [PATCH 351/526] Update getting-started-mac-tutorial.rst --- doc/source/getting-started-mac-tutorial.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index cb0dda845..d70d260ca 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -17,6 +17,10 @@ Procedure to create symbol tables for macOS To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. +.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , + which is built and maintained by `volatilityfoundation `_. + After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/``. + Listing plugins --------------- From 6b2ae6bd653b384d7e5001272878450370da74d0 Mon Sep 17 00:00:00 2001 From: cpuu Date: Mon, 8 May 2023 17:44:17 +0900 Subject: [PATCH 352/526] Update getting-started-mac-tutorial.rst --- doc/source/getting-started-mac-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index d70d260ca..cfb0afa9a 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -150,4 +150,4 @@ mac.ifconfig utun0 False utun0 fe80:5::2a95:bb15:87e3:977c False - we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. +we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. From ced6ff346cd8694ed02f835da8bec62f60ff9b56 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 8 May 2023 11:54:48 +0200 Subject: [PATCH 353/526] Apply 'Black' suggestions --- .../framework/plugins/linux/mountinfo.py | 37 +++++++++++-------- .../framework/plugins/linux/sockstat.py | 5 ++- .../framework/symbols/linux/__init__.py | 4 +- .../symbols/linux/extensions/__init__.py | 10 ++++- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index e03659aec..0606884ff 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -141,16 +141,18 @@ class MountInfo(plugins.PluginInterface): ) def _get_tasks_mountpoints( - self, tasks: Iterable[interfaces.objects.ObjectInterface], filtered_by_pids: bool + self, + tasks: Iterable[interfaces.objects.ObjectInterface], + filtered_by_pids: bool, ): seen_mountpoints = set() for task in tasks: if not ( - task and - task.fs and - task.fs.root and - task.nsproxy and - task.nsproxy.mnt_ns + task + and task.fs + and task.fs.root + and task.nsproxy + and task.nsproxy.mnt_ns ): # This task doesn't have all the information required. # It should be a kernel < 2.6.30 @@ -183,19 +185,23 @@ class MountInfo(plugins.PluginInterface): filtered_by_pids: bool, ) -> Iterable[Tuple[int, Tuple]]: warning_shown = False - for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, filtered_by_pids): + for task, mnt, mnt_ns_id in self._get_tasks_mountpoints( + tasks, filtered_by_pids + ): if ( - not warning_shown and - mnt_ns_ids and - isinstance(mnt_ns_id, renderers.NotAvailableValue) + not warning_shown + and mnt_ns_ids + and isinstance(mnt_ns_id, renderers.NotAvailableValue) ): - vollog.warning("Cannot filter by namespace id, it is not available in this kernel.") + vollog.warning( + "Cannot filter by namespace id, it is not available in this kernel." + ) warning_shown = True if ( - not isinstance(mnt_ns_id, renderers.NotAvailableValue) and - mnt_ns_ids and - mnt_ns_id not in mnt_ns_ids + not isinstance(mnt_ns_id, renderers.NotAvailableValue) + and mnt_ns_ids + and mnt_ns_id not in mnt_ns_ids ): continue @@ -284,5 +290,6 @@ class MountInfo(plugins.PluginInterface): columns.extend(extra_columns) return renderers.TreeGrid( - columns, self._generator(tasks, mount_ns_ids, mount_format, filtered_by_pids) + columns, + self._generator(tasks, mount_ns_ids, mount_format, filtered_by_pids), ) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index e37b8a1bb..72a1e453e 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -65,7 +65,10 @@ class SockHandlers(interfaces.configuration.VersionableInterface): self._vmlinux.symbol_table_name + constants.BANG + "net_device" ) for net_dev in net.dev_base_head.to_list(net_device_symname, "dev_list"): - if isinstance(netns_id, NotAvailableValue) or net.get_inode() != netns_id: + if ( + isinstance(netns_id, NotAvailableValue) + or net.get_inode() != netns_id + ): continue dev_name = utility.array_to_string(net_dev.name) netdevices_map[net_dev.ifindex] = dev_name diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 486314dd5..858845125 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -392,7 +392,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): symbol_table_arr = volobj.vol.type_name.split("!", 1) symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None - module_names = volobj._context.modules.get_modules_by_symbol_tables(symbol_table) + module_names = volobj._context.modules.get_modules_by_symbol_tables( + symbol_table + ) module_names = list(module_names) if not module_names: diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d97916b74..3dbf560c6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -878,7 +878,9 @@ class vfsmount(objects.StructType): if type(vfsmount_ptr) == objects.Pointer: return self.vol.offset == vfsmount_ptr else: - raise exceptions.VolatilityException("Unexpected argument type. It has to be a 'vfsmount *'") + raise exceptions.VolatilityException( + "Unexpected argument type. It has to be a 'vfsmount *'" + ) def _get_real_mnt(self): """Gets the struct 'mount' containing this 'vfsmount'. @@ -889,7 +891,9 @@ class vfsmount(objects.StructType): mount: the struct 'mount' containing this 'vfsmount'. """ vmlinux = linux.LinuxUtilities.get_vmlinux_from_volobj(self) - return linux.LinuxUtilities.container_of(self.vol.offset, "mount", "mnt", vmlinux) + return linux.LinuxUtilities.container_of( + self.vol.offset, "mount", "mnt", vmlinux + ) def get_vfsmnt_current(self): """Returns the current fs where we are mounted on @@ -994,6 +998,7 @@ class vfsmount(objects.StructType): def get_devname(self) -> str: return utility.pointer_to_string(self.mnt_devname, count=255) + class kobject(objects.StructType): def reference_count(self): refcnt = self.kref.refcount @@ -1262,6 +1267,7 @@ class netlink_sock(objects.StructType): else: raise AttributeError("Unable to find a destination port id") + class vsock_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for vsocks From 200099f8b3f92e2ad228e0a481a835f1fbc9d8b6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 8 May 2023 14:58:51 +0200 Subject: [PATCH 354/526] Improve support for socket filters in kernels < 4.1.52 --- .../framework/plugins/linux/sockstat.py | 23 +++++++++++++++---- .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 17 ++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 72a1e453e..f06b3ad8e 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -150,19 +150,32 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return bpfprog = sock_filter.prog - if bpfprog.type == 0: - # BPF_PROG_TYPE_UNSPEC = 0 + + BPF_PROG_TYPE_UNSPEC = 0 # cBPF filter + try: + bpfprog_type = bpfprog.get_type() + if bpfprog_type == BPF_PROG_TYPE_UNSPEC: + return # cBPF filter + except AttributeError: + # kernel < 3.18.140, it's a cBPF filter + return + + BPF_PROG_TYPE_SOCKET_FILTER = 1 # eBPF filter + if bpfprog_type != BPF_PROG_TYPE_SOCKET_FILTER: + socket_filter["bpf_filter_type"] = f"UNK({bpfprog_type})" + vollog.warning(f"Unexpected BPF type {bpfprog_type} for a socket") return socket_filter["bpf_filter_type"] = "eBPF" if not bpfprog.has_member("aux") or not bpfprog.aux: - return + return # kernel < 3.18.140 bpfprog_aux = bpfprog.aux + if bpfprog_aux.has_member("id"): - # `id` member was added to `bpf_prog_aux` in kernels 4.13 + # `id` member was added to `bpf_prog_aux` in kernels 4.13.16 socket_filter["bpf_filter_id"] = str(bpfprog_aux.id) if bpfprog_aux.has_member("name"): - # `name` was added to `bpf_prog_aux` in kernels 4.15 + # `name` was added to `bpf_prog_aux` in kernels 4.15.18 bpfprog_name = utility.array_to_string(bpfprog_aux.name) if bpfprog_name: socket_filter["bpf_filter_name"] = bpfprog_name diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 858845125..3780a86f0 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -30,6 +30,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("kobject", extensions.kobject) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) + self.optional_set_type_class("bpf_prog", extensions.bpf_prog) # Mount self.set_type_class("vfsmount", extensions.vfsmount) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3dbf560c6..87bd76554 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1313,3 +1313,20 @@ class xdp_sock(objects.StructType): def get_state(self): # xdp_sock.state is an enum return self.state.lookup() + + +class bpf_prog(objects.StructType): + def get_type(self): + # The program type was in `bpf_prog_aux::prog_type` from 3.18.140 to + # 4.1.52 before it was moved to `bpf_prog::type` + if self.has_member("type"): + # kernel >= 4.1.52 + return self.type + + if self.has_member("aux") and self.aux: + if self.aux.has_member("prog_type"): + # 3.18.140 <= kernel < 4.1.52 + return self.aux.prog_type + + # kernel < 3.18.140 + raise AttributeError("Unable to find the BPF type") From 7d65c20cc0b971c862889065d8a90d98e24819c9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 8 May 2023 14:59:58 +0200 Subject: [PATCH 355/526] Fix f-string --- volatility3/framework/plugins/linux/iomem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index 2056851aa..785405ef3 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -66,7 +66,7 @@ class IOMem(interfaces.plugins.PluginInterface): name = utility.pointer_to_string(resource.name, 128) except exceptions.InvalidAddressException: vollog.warning( - "Unable to follow pointer to name for resource object at {resource_offset:#x}, " + f"Unable to follow pointer to name for resource object at {resource_offset:#x}, " "replaced with UnreadableValue" ) name = renderers.UnreadableValue() From de28b5ab7077c04ac776018264c22484d1766c37 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 8 May 2023 19:59:53 +0200 Subject: [PATCH 356/526] Rollback explicit context --- volatility3/framework/plugins/linux/elfs.py | 2 +- .../framework/plugins/linux/malfind.py | 2 +- volatility3/framework/plugins/linux/proc.py | 2 +- .../framework/symbols/linux/__init__.py | 27 +++++++++++-------- .../symbols/linux/extensions/__init__.py | 6 ++--- 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index fa14dcd49..822a69dd6 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -58,7 +58,7 @@ class Elfs(plugins.PluginInterface): ): continue - path = vma.get_name(task) + path = vma.get_name(self.context, task) yield ( 0, diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 552fb8f53..18237b80c 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -47,7 +47,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.mm.get_mmap_iter(): - if vma.is_suspicious() and vma.get_name(task) != "[vdso]": + if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]": data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index fa7bc1629..9d8af482e 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -59,7 +59,7 @@ class Maps(plugins.PluginInterface): minor = inode_object.i_sb.minor inode = inode_object.i_ino - path = vma.get_name(task) + path = vma.get_name(self.context, task) yield ( 0, diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3780a86f0..3f51d3c0c 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -139,10 +139,11 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return path @classmethod - def _get_new_sock_pipe_path(cls, task, filp) -> str: + def _get_new_sock_pipe_path(cls, context, task, filp) -> str: """Returns the sock pipe pathname relative to the task's root directory. Args: + context: The context to retrieve required elements (layers, symbol tables) from task (task_struct): A reference task filp (file *): A pointer to a sock pipe open file @@ -151,7 +152,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): """ dentry = filp.get_dentry() - kernel_module = cls.get_vmlinux_from_volobj(dentry) + kernel_module = cls.get_vmlinux_from_volobj(context, dentry) sym_addr = dentry.d_op.d_dname symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) @@ -182,13 +183,14 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return ret @classmethod - def path_for_file(cls, task, filp) -> str: + def path_for_file(cls, context, task, filp) -> str: """Returns a file (or sock pipe) pathname relative to the task's root directory. A 'file' structure doesn't have enough information to properly restore its full path we need the root mount information from task_struct to determine this Args: + context: The context to retrieve required elements (layers, symbol tables) from task (task_struct): A reference task filp (file *): A pointer to an open file @@ -218,7 +220,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): dname_is_valid = False if dname_is_valid: - ret = LinuxUtilities._get_new_sock_pipe_path(task, filp) + ret = LinuxUtilities._get_new_sock_pipe_path(context, task, filp) else: ret = LinuxUtilities._get_path_file(task, filp) @@ -253,7 +255,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): for fd_num, filp in enumerate(fds): if filp != 0: - full_path = LinuxUtilities.path_for_file(task, filp) + full_path = LinuxUtilities.path_for_file(context, task, filp) yield fd_num, filp, full_path @@ -378,30 +380,33 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ) @classmethod - def get_vmlinux_from_volobj(cls, volobj): + def get_vmlinux_from_volobj( + cls, + context: interfaces.context.ContextInterface, + volobj: interfaces.objects.ObjectInterface, + ) -> interfaces.context.ModuleInterface: """Get the vmlinux from a vol obj Args: + context: The context to retrieve required elements (layers, symbol tables) from volobj (vol object): A vol object Raises: ValueError: If it cannot obtain any module from the symbol table Returns: - volatility3.framework.contexts.Module: A kernel object (vmlinux) + A kernel object (vmlinux) """ symbol_table_arr = volobj.vol.type_name.split("!", 1) symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None - module_names = volobj._context.modules.get_modules_by_symbol_tables( - symbol_table - ) + module_names = context.modules.get_modules_by_symbol_tables(symbol_table) module_names = list(module_names) if not module_names: raise ValueError(f"No module using the symbol table '{symbol_table}'") kernel_module_name = module_names[0] - kernel = volobj._context.modules[kernel_module_name] + kernel = context.modules[kernel_module_name] return kernel diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 87bd76554..1caab3ce5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -435,9 +435,9 @@ class vm_area_struct(objects.StructType): return self.vm_pgoff << constants.linux.PAGE_SHIFT - def get_name(self, task): + def get_name(self, context, task): if self.vm_file != 0: - fname = linux.LinuxUtilities.path_for_file(task, self.vm_file) + fname = linux.LinuxUtilities.path_for_file(context, task, self.vm_file) elif self.vm_start <= task.mm.start_brk and self.vm_end >= task.mm.brk: fname = "[heap]" elif self.vm_start <= task.mm.start_stack <= self.vm_end: @@ -890,7 +890,7 @@ class vfsmount(objects.StructType): Returns: mount: the struct 'mount' containing this 'vfsmount'. """ - vmlinux = linux.LinuxUtilities.get_vmlinux_from_volobj(self) + vmlinux = linux.LinuxUtilities.get_vmlinux_from_volobj(self._context, self) return linux.LinuxUtilities.container_of( self.vol.offset, "mount", "mnt", vmlinux ) From 859482445907fd9c2ae579d37e0a898b10857690 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 9 May 2023 06:47:07 +0100 Subject: [PATCH 357/526] Linux: Ensure that objects made when parsing maple tree use the native_layer_name --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index a8bf1b6aa..931003b60 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -349,7 +349,7 @@ class maple_tree(objects.StructType): symbol_table_name = self.get_symbol_table_name() node_parent_mte = self._context.object( symbol_table_name + constants.BANG + "pointer", - layer_name=self.vol.layer_name, + layer_name=self.vol.native_layer_name, offset=pointer, ) @@ -424,7 +424,7 @@ class mm_struct(objects.StructType): # convert pointer to vm_area_struct and yield vma = self._context.object( symbol_table_name + constants.BANG + "vm_area_struct", - layer_name=self.vol.layer_name, + layer_name=self.vol.native_layer_name, offset=vma_pointer ) yield vma From a2906ad270b3ec09dd38db3bc1d24b58de0c7054 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 9 May 2023 07:07:39 +0100 Subject: [PATCH 358/526] Linux: Update comments and var names for maple tree depth warnings --- .../symbols/linux/extensions/__init__.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 931003b60..67a4cd2d0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -314,15 +314,15 @@ class maple_tree(objects.StructType): def get_slot_iter(self): """Parse the Maple Tree and return every non zero slot.""" maple_tree_offset = self.vol.offset & ~(self.MAPLE_NODE_POINTER_MASK) - maple_tree_depth = ( + expected_maple_tree_depth = ( self.ma_flags & self.MT_FLAGS_HEIGHT_MASK ) >> self.MT_FLAGS_HEIGHT_OFFSET yield from self._parse_maple_tree_node( - self.ma_root, maple_tree_offset, maple_tree_depth + self.ma_root, maple_tree_offset, expected_maple_tree_depth ) def _parse_maple_tree_node( - self, maple_tree_entry, parent, maple_tree_depth, seen=set(), depth=1 + self, maple_tree_entry, parent, expected_maple_tree_depth, seen=set(), current_depth=1 ): """Recursively parse Maple Tree Nodes and yield all non empty slots""" @@ -334,11 +334,16 @@ class maple_tree(objects.StructType): return else: seen.add(maple_tree_entry) - if maple_tree_depth < depth: + + # check if we have exceeded the expected depth of this maple tree. + # e.g. when current_depth is larger than expected_maple_tree_depth there may be an issue. + # it is normal that expected_maple_tree_depth is equal to current_depth. + if expected_maple_tree_depth < current_depth: vollog.warning( - f"The depth for the maple tree at {hex(self.vol.offset)} is {maple_tree_depth}, however when parsing the nodes " - f"a depth of {depth} was reached. This is unexpected and may lead to incorrect results." + f"The depth for the maple tree at {hex(self.vol.offset)} is {expected_maple_tree_depth}, however when parsing the nodes " + f"a depth of {current_depth} was reached. This is unexpected and may lead to incorrect results." ) + # parse the mte to extract the pointer value, node type, and leaf status pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) node_type = ( @@ -379,13 +384,13 @@ class maple_tree(objects.StructType): for slot in node.mr64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( - slot, pointer, maple_tree_depth, seen, depth + 1 + slot, pointer, expected_maple_tree_depth, seen, current_depth + 1 ) elif node_type == self.MAPLE_ARANGE_64: for slot in node.ma64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( - slot, pointer, maple_tree_depth, seen, depth + 1 + slot, pointer, expected_maple_tree_depth, seen, current_depth + 1 ) else: # unkown maple node type From d367b973a5cc305aad8f3a5f0b92e4b525122d11 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 9 May 2023 09:05:37 +0200 Subject: [PATCH 359/526] Renamed get_vmlinux_from_volobj() to get_module_from_volobj_type() --- volatility3/framework/symbols/linux/__init__.py | 4 ++-- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3f51d3c0c..dbdf1b777 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -152,7 +152,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): """ dentry = filp.get_dentry() - kernel_module = cls.get_vmlinux_from_volobj(context, dentry) + kernel_module = cls.get_module_from_volobj_type(context, dentry) sym_addr = dentry.d_op.d_dname symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) @@ -380,7 +380,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ) @classmethod - def get_vmlinux_from_volobj( + def get_module_from_volobj_type( cls, context: interfaces.context.ContextInterface, volobj: interfaces.objects.ObjectInterface, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1caab3ce5..060da4d7d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -890,7 +890,7 @@ class vfsmount(objects.StructType): Returns: mount: the struct 'mount' containing this 'vfsmount'. """ - vmlinux = linux.LinuxUtilities.get_vmlinux_from_volobj(self._context, self) + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) return linux.LinuxUtilities.container_of( self.vol.offset, "mount", "mnt", vmlinux ) From 6c5db21ae76519ffff538eba822dccfb2c63f4c2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 9 May 2023 09:22:02 +0200 Subject: [PATCH 360/526] Undo mnt_ns_id cast to str --- volatility3/framework/plugins/linux/mountinfo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 0606884ff..87ba4f9c1 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -160,7 +160,7 @@ class MountInfo(plugins.PluginInterface): mnt_namespace = task.nsproxy.mnt_ns try: - mnt_ns_id = str(mnt_namespace.get_inode()) + mnt_ns_id = mnt_namespace.get_inode() except AttributeError: mnt_ns_id = renderers.NotAvailableValue() @@ -256,7 +256,7 @@ class MountInfo(plugins.PluginInterface): self.context, self.config["kernel"], filter_func=pid_filter ) - columns = [("MNT_NS_ID", str)] + columns = [("MNT_NS_ID", int)] # The PID column does not make sense when a PID filter is not specified. In that case, the default behavior is # to displays the mountpoints per namespace. if pids: From 89ed65cc56bdce8b11c3480f5dcc93d0e1021961 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 9 May 2023 09:59:21 +0200 Subject: [PATCH 361/526] Improve filter warning implementation --- .../framework/plugins/linux/mountinfo.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 87ba4f9c1..e4081dc83 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -184,19 +184,12 @@ class MountInfo(plugins.PluginInterface): mount_format: bool, filtered_by_pids: bool, ) -> Iterable[Tuple[int, Tuple]]: - warning_shown = False + show_filter_warning = False for task, mnt, mnt_ns_id in self._get_tasks_mountpoints( tasks, filtered_by_pids ): - if ( - not warning_shown - and mnt_ns_ids - and isinstance(mnt_ns_id, renderers.NotAvailableValue) - ): - vollog.warning( - "Cannot filter by namespace id, it is not available in this kernel." - ) - warning_shown = True + if mnt_ns_ids and isinstance(mnt_ns_id, renderers.NotAvailableValue): + show_filter_warning = True if ( not isinstance(mnt_ns_id, renderers.NotAvailableValue) @@ -246,6 +239,11 @@ class MountInfo(plugins.PluginInterface): yield (0, fields_values) + if show_filter_warning: + vollog.warning( + "Could not filter by mount namespace id. This field is not available in this kernel." + ) + def run(self): pids = self.config.get("pids") mount_ns_ids = self.config.get("mntns") From 186b1ed1c2087a5e28edbf972f603c8c03ca26bf Mon Sep 17 00:00:00 2001 From: cpuu Date: Wed, 10 May 2023 09:49:11 +0900 Subject: [PATCH 362/526] Add more process information to mac.pslist plugin Enhance the PsList plugin by including additional process information such as offset, UID, GID, and start time in the output. Update the TreeGrid columns to display the new information. --- volatility3/framework/plugins/mac/pslist.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 1d97216bf..f88715c9c 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime import logging from typing import Callable, Iterable, List, Dict @@ -9,6 +10,7 @@ from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.symbols import mac +from volatility3.framework.renderers import format_hints vollog = logging.getLogger(__name__) @@ -105,10 +107,19 @@ class PsList(interfaces.plugins.PluginInterface): self.config["kernel"], filter_func=self.create_pid_filter(self.config.get("pid", None)), ): - pid = task.p_pid - ppid = task.p_ppid + offset = format_hints.Hex(task.vol.offset) name = utility.array_to_string(task.p_comm) - yield (0, (pid, ppid, name)) + pid = task.p_pid + uid = task.p_uid + gid = task.p_gid + start_time_seconds = task.p_start.tv_sec + start_time_microseconds = task.p_start.tv_usec + start_time = datetime.datetime.fromtimestamp(start_time_seconds + start_time_microseconds / 1e6) + + + ppid = task.p_ppid + + yield (0, (offset, name, pid, uid, gid, start_time, ppid)) @classmethod def list_tasks_allproc( @@ -310,5 +321,5 @@ class PsList(interfaces.plugins.PluginInterface): def run(self): return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str)], self._generator() + [("OFFSET", format_hints.Hex), ("NAME", str), ("PID", int), ("UID", int), ("GID", int), ("Start Time", datetime.datetime), ("PPID", int)], self._generator() ) From 396daa528c1028e559ac626571f8894e35450f3e Mon Sep 17 00:00:00 2001 From: cpuu Date: Wed, 10 May 2023 10:07:09 +0900 Subject: [PATCH 363/526] Apply Black Python linter to mac.pslist plugin Applied the Black Python linter to the mac PsList plugin, resulting in more readable and consistent code formatting. --- volatility3/framework/plugins/mac/pslist.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index f88715c9c..dbed09818 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -114,11 +114,12 @@ class PsList(interfaces.plugins.PluginInterface): gid = task.p_gid start_time_seconds = task.p_start.tv_sec start_time_microseconds = task.p_start.tv_usec - start_time = datetime.datetime.fromtimestamp(start_time_seconds + start_time_microseconds / 1e6) - + start_time = datetime.datetime.fromtimestamp( + start_time_seconds + start_time_microseconds / 1e6 + ) ppid = task.p_ppid - + yield (0, (offset, name, pid, uid, gid, start_time, ppid)) @classmethod @@ -321,5 +322,14 @@ class PsList(interfaces.plugins.PluginInterface): def run(self): return renderers.TreeGrid( - [("OFFSET", format_hints.Hex), ("NAME", str), ("PID", int), ("UID", int), ("GID", int), ("Start Time", datetime.datetime), ("PPID", int)], self._generator() + [ + ("OFFSET", format_hints.Hex), + ("NAME", str), + ("PID", int), + ("UID", int), + ("GID", int), + ("Start Time", datetime.datetime), + ("PPID", int), + ], + self._generator(), ) From 7f9afccf6f44fb69cd1039ab478854317bad5d5b Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 10 May 2023 08:49:46 +0100 Subject: [PATCH 364/526] Linux: apply black formating to maple tree parsing --- .../symbols/linux/extensions/__init__.py | 46 +++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 67a4cd2d0..7398a9ecd 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -294,6 +294,7 @@ class fs_struct(objects.StructType): raise AttributeError("Unable to find the root mount") + class maple_tree(objects.StructType): # include/linux/maple_tree.h # Mask for Maple Tree Flags @@ -322,7 +323,12 @@ class maple_tree(objects.StructType): ) def _parse_maple_tree_node( - self, maple_tree_entry, parent, expected_maple_tree_depth, seen=set(), current_depth=1 + self, + maple_tree_entry, + parent, + expected_maple_tree_depth, + seen=set(), + current_depth=1, ): """Recursively parse Maple Tree Nodes and yield all non empty slots""" @@ -384,13 +390,21 @@ class maple_tree(objects.StructType): for slot in node.mr64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( - slot, pointer, expected_maple_tree_depth, seen, current_depth + 1 + slot, + pointer, + expected_maple_tree_depth, + seen, + current_depth + 1, ) elif node_type == self.MAPLE_ARANGE_64: for slot in node.ma64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( - slot, pointer, expected_maple_tree_depth, seen, current_depth + 1 + slot, + pointer, + expected_maple_tree_depth, + seen, + current_depth + 1, ) else: # unkown maple node type @@ -398,13 +412,16 @@ class maple_tree(objects.StructType): f"Unkown Maple Tree node type {node_type} at offset {hex(pointer)}." ) + class mm_struct(objects.StructType): def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct.""" - if not self.has_member('mmap'): - raise AttributeError("get_mmap_iter called on mm_struct where no mmap member exists.") - + if not self.has_member("mmap"): + raise AttributeError( + "get_mmap_iter called on mm_struct where no mmap member exists." + ) + if not self.mmap: return @@ -420,9 +437,11 @@ class mm_struct(objects.StructType): def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mm_mt member of an mm_struct.""" - - if not self.has_member('mm_mt'): - raise AttributeError("get_maple_tree_iter called on mm_struct where no mm_mt member exists.") + + if not self.has_member("mm_mt"): + raise AttributeError( + "get_maple_tree_iter called on mm_struct where no mm_mt member exists." + ) symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): @@ -430,20 +449,21 @@ class mm_struct(objects.StructType): vma = self._context.object( symbol_table_name + constants.BANG + "vm_area_struct", layer_name=self.vol.native_layer_name, - offset=vma_pointer + offset=vma_pointer, ) yield vma def get_vma_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" - if self.has_member('mmap'): + if self.has_member("mmap"): yield from self.get_mmap_iter() - elif self.has_member('mm_mt'): + elif self.has_member("mm_mt"): yield from self.get_maple_tree_iter() else: raise AttributeError("Unable to find mmap or mm_mt in mm_struct") - + + class super_block(objects.StructType): # include/linux/kdev_t.h MINORBITS = 20 From 834372a6f89dc4c49c0a2fb563c1dd1cea1392ae Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 10 May 2023 09:12:03 +0100 Subject: [PATCH 365/526] Linux: apply black v23.3.0 formating to volatility3/framework/symbols/linux/extensions/__init__.py --- .../symbols/linux/extensions/__init__.py | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7398a9ecd..eecb200f9 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -34,10 +34,8 @@ class module(generic.GenericIntelProcess): def get_init_size(self): if self.has_member("init_layout"): return self.init_layout.size - elif self.has_member("init_size"): return self.init_size - raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module" ) @@ -45,10 +43,8 @@ class module(generic.GenericIntelProcess): def get_core_size(self): if self.has_member("core_layout"): return self.core_layout.size - elif self.has_member("core_size"): return self.core_size - raise AttributeError( "module -> get_core_size: Unable to determine core size of module" ) @@ -58,7 +54,6 @@ class module(generic.GenericIntelProcess): return self.core_layout.base elif self.has_member("module_core"): return self.module_core - raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): @@ -66,7 +61,6 @@ class module(generic.GenericIntelProcess): return self.init_layout.base elif self.has_member("module_init"): return self.module_init - raise AttributeError("module -> get_module_core: Unable to get module init") def get_name(self): @@ -88,7 +82,6 @@ class module(generic.GenericIntelProcess): idx = 0 while arr[idx]: idx = idx + 1 - return idx def get_sections(self): @@ -97,7 +90,6 @@ class module(generic.GenericIntelProcess): num_sects = self.sect_attrs.nsections else: num_sects = self._get_sect_count(self.sect_attrs.grp) - arr = self._context.object( self.get_symbol_table().name + constants.BANG + "array", layer_name=self.vol.layer_name, @@ -116,7 +108,6 @@ class module(generic.GenericIntelProcess): prefix = "Elf64_" else: prefix = "Elf32_" - elf_table_name = intermed.IntermediateSymbolTable.create( self.context, self.config_path, @@ -155,7 +146,6 @@ class module(generic.GenericIntelProcess): return self.kallsyms.symtab elif self.has_member("symtab"): return self.symtab - raise AttributeError("module -> symtab: Unable to get symtab") @property @@ -164,7 +154,6 @@ class module(generic.GenericIntelProcess): return int(self.kallsyms.num_symtab) elif self.has_member("num_symtab"): return int(self.num_symtab) - raise AttributeError( "module -> num_symtab: Unable to determine number of symbols" ) @@ -177,7 +166,6 @@ class module(generic.GenericIntelProcess): # Older kernels elif self.has_member("strtab"): return self.strtab - raise AttributeError("module -> strtab: Unable to get strtab") @@ -195,19 +183,15 @@ class task_struct(generic.GenericIntelProcess): pgd = self.mm.pgd except exceptions.InvalidAddressException: return None - if not isinstance(parent_layer, linear.LinearlyMappedLayer): raise TypeError( "Parent layer is not a translation layer, unable to construct process layer" ) - dtb, layer_name = parent_layer.translate(pgd) if not dtb: return None - if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.pid}" - # Add the constructed layer and return the name return self._add_process_layer( self._context, dtb, config_prefix, preferred_name @@ -229,7 +213,6 @@ class task_struct(generic.GenericIntelProcess): vollog.info( f"adding vma: {start:x} {self.mm.brk:x} | {end:x} {self.mm.start_brk:x}" ) - yield (start, end - start) @property @@ -282,7 +265,6 @@ class fs_struct(objects.StructType): return self.root elif self.root.has_member("dentry"): return self.root.dentry - raise AttributeError("Unable to find the root dentry") def get_root_mnt(self): @@ -291,7 +273,6 @@ class fs_struct(objects.StructType): return self.rootmnt elif self.root.has_member("mnt"): return self.root.mnt - raise AttributeError("Unable to find the root mount") @@ -340,7 +321,6 @@ class maple_tree(objects.StructType): return else: seen.add(maple_tree_entry) - # check if we have exceeded the expected depth of this maple tree. # e.g. when current_depth is larger than expected_maple_tree_depth there may be an issue. # it is normal that expected_maple_tree_depth is equal to current_depth. @@ -349,7 +329,6 @@ class maple_tree(objects.StructType): f"The depth for the maple tree at {hex(self.vol.offset)} is {expected_maple_tree_depth}, however when parsing the nodes " f"a depth of {current_depth} was reached. This is unexpected and may lead to incorrect results." ) - # parse the mte to extract the pointer value, node type, and leaf status pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) node_type = ( @@ -421,10 +400,8 @@ class mm_struct(objects.StructType): raise AttributeError( "get_mmap_iter called on mm_struct where no mmap member exists." ) - if not self.mmap: return - yield self.mmap seen = {self.mmap.vol.offset} @@ -442,7 +419,6 @@ class mm_struct(objects.StructType): raise AttributeError( "get_maple_tree_iter called on mm_struct where no mm_mt member exists." ) - symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): # convert pointer to vm_area_struct and yield @@ -569,7 +545,6 @@ class vm_area_struct(objects.StructType): retval = retval + char else: retval = retval + "-" - return retval # only parse the rwx bits @@ -583,7 +558,6 @@ class vm_area_struct(objects.StructType): def get_page_offset(self) -> int: if self.vm_file == 0: return 0 - return self.vm_pgoff << constants.linux.PAGE_SHIFT def get_name(self, context, task): @@ -600,7 +574,6 @@ class vm_area_struct(objects.StructType): fname = "[vdso]" else: fname = "Anonymous Mapping" - return fname # used by malfind @@ -611,10 +584,8 @@ class vm_area_struct(objects.StructType): if flags_str == "rwx": ret = True - elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True - return ret @@ -624,12 +595,10 @@ class qstr(objects.StructType): str_length = self.len + 1 # Maximum length should include null terminator else: str_length = 255 - try: ret = objects.utility.pointer_to_string(self.name, str_length) except (exceptions.InvalidAddressException, ValueError): ret = "" - return ret @@ -660,7 +629,6 @@ class dentry(objects.StructType): """ if self.vol.offset == old_dentry: return True - return self.d_ancestor(old_dentry) def d_ancestor(self, ancestor_dentry): @@ -678,10 +646,8 @@ class dentry(objects.StructType): ): if current_dentry.d_parent == ancestor_dentry.vol.offset: return current_dentry - dentry_seen.add(current_dentry.vol.offset) current_dentry = current_dentry.d_parent - return None @@ -738,12 +704,10 @@ class list_head(objects.StructType, collections.abc.Iterable): link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: return - if not sentinel: yield self._context.object( symbol_type, layer, offset=self.vol.offset - relative_offset ) - seen = {self.vol.offset} while link.vol.offset not in seen: obj = self._context.object( @@ -869,7 +833,6 @@ class mount(objects.StructType): peer = current_mnt.get_peer_under_root(self.mnt_ns, root) if peer and peer.vol.offset != 0: return peer.mnt_group_id - mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.mnt_master return 0 @@ -885,12 +848,10 @@ class mount(objects.StructType): current_mnt.mnt.mnt_root, root ): return current_mnt - mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.next_peer() if current_mnt.vol.offset == self.vol.offset: break - return None def is_path_reachable(self, current_dentry, root): @@ -907,7 +868,6 @@ class mount(objects.StructType): current_dentry = current_mnt.mnt_mountpoint mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.mnt_parent - return current_mnt.mnt.vol.offset == root.mnt and current_dentry.is_subdir( root.dentry ) @@ -968,7 +928,6 @@ class kobject(objects.StructType): ret = refcnt.counter else: ret = refcnt.refs.counter - return ret @@ -987,7 +946,6 @@ class mnt_namespace(objects.StructType): if not self._context.symbol_space.has_type(mnt_type): # Old kernels ~ 2.6 mnt_type = table_name + constants.BANG + "vfsmount" - for mount in self.list.to_list(mnt_type, "mnt_list"): yield mount @@ -1012,7 +970,6 @@ class socket(objects.StructType): ) if not module_names: raise ValueError(f"No module using the symbol table {symbol_table}") - kernel_module_name = module_names[0] kernel = self._context.modules[kernel_module_name] return kernel @@ -1022,7 +979,6 @@ class socket(objects.StructType): kernel = self._get_vol_kernel() except ValueError: return 0 - socket_alloc = linux.LinuxUtilities.container_of( self.vol.offset, "socket_alloc", "socket", kernel ) @@ -1048,7 +1004,6 @@ class sock(objects.StructType): def get_inode(self): if not self.sk_socket: return 0 - return self.sk_socket.get_inode() def get_protocol(self): @@ -1058,7 +1013,6 @@ class sock(objects.StructType): # Return the generic socket state if self.has_member("sk"): return self.sk.sk_socket.get_state() - return self.sk_socket.get_state() @@ -1066,7 +1020,6 @@ class unix_sock(objects.StructType): def get_name(self): if not self.addr: return - sockaddr_un = self.addr.name.cast("sockaddr_un") saddr = str(utility.array_to_string(sockaddr_un.sun_path)) return saddr @@ -1102,7 +1055,6 @@ class inet_sock(objects.StructType): protocol = IP_PROTOCOLS.get(self.sk.sk_protocol) if self.get_family() == "AF_INET6": protocol = IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) - return protocol def get_state(self): @@ -1133,7 +1085,6 @@ class inet_sock(objects.StructType): dport_le = sk_common.skc_dport else: return - return socket_module.htons(dport_le) def get_src_addr(self): @@ -1152,7 +1103,6 @@ class inet_sock(objects.StructType): saddr = self.pinet6.saddr else: return - parent_layer = self._context.layers[self.vol.layer_name] try: addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) @@ -1161,7 +1111,6 @@ class inet_sock(objects.StructType): f"Unable to read socket src address from {saddr.vol.offset:#x}" ) return - return socket_module.inet_ntop(family, addr_bytes) def get_dst_addr(self): @@ -1183,7 +1132,6 @@ class inet_sock(objects.StructType): addr_size = 16 else: return - parent_layer = self._context.layers[self.vol.layer_name] try: addr_bytes = parent_layer.read(daddr.vol.offset, addr_size) @@ -1192,7 +1140,6 @@ class inet_sock(objects.StructType): f"Unable to read socket dst address from {daddr.vol.offset:#x}" ) return - return socket_module.inet_ntop(family, addr_bytes) From b3c348820f341ea43a5bb215a5841349f46fb16c Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 10 May 2023 09:18:05 +0100 Subject: [PATCH 366/526] Linux: apply black v23.3.0 formating to volatility3/framework/symbols/linux/__init__.py --- volatility3/framework/symbols/linux/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 7a22241a5..c012f9cc7 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -53,6 +53,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): # Only found in 6.1+ kernels self.optional_set_type_class("maple_tree", extensions.maple_tree) + class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" From 7516346b649d7678c2996d1f4e02c48637fb050f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 10 May 2023 10:55:19 +0200 Subject: [PATCH 367/526] Adjust framework versioning --- volatility3/framework/plugins/linux/mountinfo.py | 5 ++++- volatility3/framework/symbols/linux/__init__.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index e4081dc83..da743bb60 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -50,6 +50,9 @@ class MountInfo(plugins.PluginInterface): requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) + ), requirements.ListRequirement( name="pids", description="Filter on specific process IDs.", @@ -86,7 +89,7 @@ class MountInfo(plugins.PluginInterface): if not mnt_root: return None - path_root = linux.LinuxUtilities._get_path_mnt(task, mnt) + path_root = linux.LinuxUtilities.get_path_mnt(task, mnt) if not path_root: return None diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index dbdf1b777..9ae8479b6 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -55,7 +55,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 0, 0) + _version = (2, 1, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -79,7 +79,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt) @classmethod - def _get_path_mnt(cls, task, mnt) -> str: + def get_path_mnt(cls, task, mnt) -> str: """Returns the mount point pathname relative to the task's root directory. Args: From e28b42731e663baf09051978a669d47e72506aaf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 10 May 2023 21:55:16 +0100 Subject: [PATCH 368/526] Core: Change readthedocs python version Attempting to fix #953 --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 4d21d9b40..3f10db145 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -14,6 +14,6 @@ formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: 3.7 + version: 3.11 install: - requirements: doc/requirements.txt From 3f1797f7e5b47a9bfd03f521604cd014012a0c69 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 10 May 2023 21:57:50 +0100 Subject: [PATCH 369/526] Core: Change readthedocs build platform Fixes #953 --- .readthedocs.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 3f10db145..e7c2b25d5 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,8 +12,12 @@ sphinx: # Optionally build your docs in additional formats such as PDF and ePub formats: all +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # Optionally set the version of Python and requirements required to build your docs python: - version: 3.11 install: - requirements: doc/requirements.txt From 8e56cb39d13c22575ba5133ffbc6b83344bed0ba Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 10 May 2023 22:07:38 +0100 Subject: [PATCH 370/526] Docs: Fix the documentation dependencies --- doc/requirements.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 93d6ea70a..b715e59f5 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,8 @@ # These packages are required for building the documentation. -sphinx>=4.0.0 +sphinx>=4.0.0,<7 sphinx_autodoc_typehints>=1.4.0 sphinx-rtd-theme>=0.4.3 + +yara-python +pycryptodome +pefile From 1a7ec07c28ee4830f14a0dfaa003aaa5f82eeffb Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 13 Dec 2022 16:09:57 +0000 Subject: [PATCH 371/526] add linux.vmayarascan based on windows.vadtarascan --- .../framework/plugins/linux/vmayarascan.py | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 volatility3/framework/plugins/linux/vmayarascan.py diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py new file mode 100644 index 000000000..e9482089a --- /dev/null +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -0,0 +1,121 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from typing import Iterable, List, Tuple + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins import yarascan +from volatility3.plugins.linux import pslist + +class VmaYaraScan(interfaces.plugins.PluginInterface): + """Scans all virtual memory areas for tasks using yara.""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.BooleanRequirement( + name="wide", + description="Match wide (unicode) strings", + default=False, + optional=True, + ), + requirements.StringRequirement( + name="yara_rules", description="Yara rules (as a string)", optional=True + ), + requirements.URIRequirement( + name="yara_file", description="Yara rules (as a file)", optional=True + ), + # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code + # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later + requirements.URIRequirement( + name="yara_compiled_file", + description="Yara compiled rules (as a file)", + optional=True, + ), + requirements.IntRequirement( + name="max_size", + default=0x40000000, + description="Set the maximum size (default is 1GB)", + optional=True, + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), + ] + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + rules = yarascan.YaraScan.process_yara_options(dict(self.config)) + + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + for task in pslist.PsList.list_tasks( + context=self.context, + vmlinux_module_name=self.config["kernel"], + filter_func=filter_func, + ): + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + continue + + proc_layer = self.context.layers[proc_layer_name] + for offset, rule_name, name, value in proc_layer.scan( + context=self.context, + scanner=yarascan.YaraScanner(rules=rules), + sections=self.get_vma_maps(task), + ): + yield 0, ( + format_hints.Hex(offset), + task.tgid, + rule_name, + name, + value, + ) + + @staticmethod + def get_vma_maps( + task: interfaces.objects.ObjectInterface, + ) -> Iterable[Tuple[int, int]]: + """Creates a map of start/end addresses for each virtual memory area in a task. + + Args: + task: The task object of which to read the vmas from + + Returns: + An iterable of tuples containing start and end addresses for each descriptor + """ + if task.mm: + for vma in task.mm.get_mmap_iter(): + vm_size = vma.vm_end - vma.vm_start + yield (vma.vm_start, vm_size) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("PID", int), + ("Rule", str), + ("Component", str), + ("Value", bytes), + ], + self._generator(), + ) From 2279a83e3b53f92ae21cbd8ec8acd0dfa389458b Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 4 Jan 2023 09:32:39 +0000 Subject: [PATCH 372/526] liniting for linux.vmayarascan --- volatility3/framework/plugins/linux/vmayarascan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index e9482089a..3efbd2ae1 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -10,6 +10,7 @@ from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan from volatility3.plugins.linux import pslist + class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" From 57e8234e6c795391ad99f4a76216bb3ef3db808a Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 4 Jan 2023 09:36:20 +0000 Subject: [PATCH 373/526] remove unused variable in linux.vmayarascan --- volatility3/framework/plugins/linux/vmayarascan.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 3efbd2ae1..c7a48cc14 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -64,8 +64,6 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): ] def _generator(self): - kernel = self.context.modules[self.config["kernel"]] - rules = yarascan.YaraScan.process_yara_options(dict(self.config)) filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) From b54bbfb88f600e660d1ca6e8e7cec77138179c1f Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 1 Feb 2023 11:27:44 +0000 Subject: [PATCH 374/526] Update linux.vmayarascan to pull requirements from the generic yarascan plugin --- .../framework/plugins/linux/vmayarascan.py | 79 ++++++++++--------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index c7a48cc14..f0d42f6e3 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # @@ -15,68 +15,71 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [ - requirements.ModuleRequirement( - name="kernel", - description="Linux kernel", - architectures=["Intel32", "Intel64"], - ), - requirements.BooleanRequirement( - name="wide", - description="Match wide (unicode) strings", - default=False, - optional=True, - ), - requirements.StringRequirement( - name="yara_rules", description="Yara rules (as a string)", optional=True - ), - requirements.URIRequirement( - name="yara_file", description="Yara rules (as a file)", optional=True - ), - # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code - # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later - requirements.URIRequirement( - name="yara_compiled_file", - description="Yara compiled rules (as a file)", - optional=True, - ), - requirements.IntRequirement( - name="max_size", - default=0x40000000, - description="Set the maximum size (default is 1GB)", - optional=True, - ), - requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) - ), - requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) - ), + # create a list of requirements for vmayarascan + vmayarascan_requirements = [ requirements.ListRequirement( name="pid", element_type=int, description="Process IDs to include (all other processes are excluded)", optional=True, ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(1, 1, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), ] + # get base yarascan requirements + yarascan_requirements = yarascan.YaraScan.get_requirements() + + # remove TranslationLayerRequirement from the base yarascan requirements + # if this is not removed automagic will not find both the TranslationLayerRequirement + # for YaraScan and the ModuleRequirement for VmaYaraScan + yarascan_requirements = [ + requirement + for requirement in yarascan_requirements + if not isinstance(requirement, requirements.TranslationLayerRequirement) + ] + + # return the combined requirements + return yarascan_requirements + vmayarascan_requirements + def _generator(self): + # use yarascan to parse the yara options provided and create the rules rules = yarascan.YaraScan.process_yara_options(dict(self.config)) + # filter based on the pid option if provided filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for task in pslist.PsList.list_tasks( context=self.context, vmlinux_module_name=self.config["kernel"], filter_func=filter_func, ): + + # attempt to create a process layer for each task and skip those + # that cannot (e.g. kernel threads) proc_layer_name = task.add_process_layer() if not proc_layer_name: continue + # get the proc_layer object from the context proc_layer = self.context.layers[proc_layer_name] + + # scan the process layer with the yarascanner for offset, rule_name, name, value in proc_layer.scan( context=self.context, scanner=yarascan.YaraScanner(rules=rules), From 09b0c8e406ec1aaf49a87a99fc86523fce36ff69 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 12 May 2023 13:30:32 +0100 Subject: [PATCH 375/526] Linux: Update linux.vmayarascan and yarascan so that command line options are taken from the base yarascan plugin --- .../framework/plugins/linux/vmayarascan.py | 16 +++------------- volatility3/framework/plugins/yarascan.py | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index f0d42f6e3..8e4174c12 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -31,7 +31,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="yarascan", plugin=yarascan.YaraScan, version=(1, 1, 0) + name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) ), requirements.VersionRequirement( name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) @@ -43,17 +43,8 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): ), ] - # get base yarascan requirements - yarascan_requirements = yarascan.YaraScan.get_requirements() - - # remove TranslationLayerRequirement from the base yarascan requirements - # if this is not removed automagic will not find both the TranslationLayerRequirement - # for YaraScan and the ModuleRequirement for VmaYaraScan - yarascan_requirements = [ - requirement - for requirement in yarascan_requirements - if not isinstance(requirement, requirements.TranslationLayerRequirement) - ] + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() # return the combined requirements return yarascan_requirements + vmayarascan_requirements @@ -69,7 +60,6 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): vmlinux_module_name=self.config["kernel"], filter_func=filter_func, ): - # attempt to create a process layer for each task and skip those # that cannot (e.g. kernel threads) proc_layer_name = task.add_process_layer() diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 1c8467689..11c708607 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -61,19 +61,31 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (1, 2, 0) # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string # or something that makes more sense @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [ + """Returns the requirements needed to run yarascan directly, combining the TranslationLayerRequirement + and the requirements from get_yarascan_option_requirements.""" + return cls.get_yarascan_option_requirements() + [ requirements.TranslationLayerRequirement( name="primary", description="Memory layer for the kernel", architectures=["Intel32", "Intel64"], - ), + ) + ] + + @classmethod + def get_yarascan_option_requirements( + cls, + ) -> List[interfaces.configuration.RequirementInterface]: + """Returns the requirements needed for the command lines options used by yarascan. This can + then also be used by other plugins that are using yarascan. This does not include a + TranslationLayerRequirement or a ModuleRequirement.""" + return [ requirements.BooleanRequirement( name="insensitive", description="Makes the search case insensitive", From cd03c52fd30d7caf57aaa38a7b32197e4d158ca2 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 12 May 2023 13:37:15 +0100 Subject: [PATCH 376/526] Linux: Update linux.vmayarascan to use task.mm.get_vma_iter() which means it will work on linux kernels 6.1 and above --- volatility3/framework/plugins/linux/vmayarascan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 8e4174c12..eda0d7dca 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -96,7 +96,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): An iterable of tuples containing start and end addresses for each descriptor """ if task.mm: - for vma in task.mm.get_mmap_iter(): + for vma in task.mm.get_vma_iter(): vm_size = vma.vm_end - vma.vm_start yield (vma.vm_start, vm_size) From 3d9efc3b3466a91dcbaf5aacfbc302b816bba316 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 17 May 2023 14:14:33 +0100 Subject: [PATCH 377/526] Added linux capabilities plugin --- .../framework/constants/linux/__init__.py | 45 ++++ .../framework/plugins/linux/capabilities.py | 218 ++++++++++++++++++ .../framework/symbols/linux/__init__.py | 2 + .../symbols/linux/extensions/__init__.py | 108 ++++++++- 4 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 volatility3/framework/plugins/linux/capabilities.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 1b133eb42..a802e0ada 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -234,3 +234,48 @@ BLUETOOTH_PROTOCOLS = ( "HIDP", "AVDTP", ) + +# Ref: include/uapi/linux/capability.h +CAPABILITIES = ( + "chown", + "dac_override", + "dac_read_search", + "fowner", + "fsetid", + "kill", + "setgid", + "setuid", + "setpcap", + "linux_immutable", + "net_bind_service", + "net_broadcast", + "net_admin", + "net_raw", + "ipc_lock", + "ipc_owner", + "sys_module", + "sys_rawio", + "sys_chroot", + "sys_ptrace", + "sys_pacct", + "sys_admin", + "sys_boot", + "sys_nice", + "sys_resource", + "sys_time", + "sys_tty_config", + "mknod", + "lease", + "audit_write", + "audit_control", + "setfcap", + "mac_override", + "mac_admin", + "syslog", + "wake_alarm", + "block_suspend", + "audit_read", + "perfmon", + "bpf", + "checkpoint_restore", +) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py new file mode 100644 index 000000000..8bcd80eef --- /dev/null +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -0,0 +1,218 @@ +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import Iterable, List, Tuple, Dict + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols.linux import extensions +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class Capabilities(plugins.PluginInterface): + """Lists process capabilities""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pids", + description="Filter on specific process IDs.", + element_type=int, + optional=True, + ), + requirements.BooleanRequirement( + name="inheritable", + description="Show only inheritable capabilities in human-readable strings.", + optional=True, + ), + requirements.BooleanRequirement( + name="permitted", + description="Show only permitted capabilities in human-readable strings.", + optional=True, + ), + requirements.BooleanRequirement( + name="effective", + description="Show only effective capabilities in human-readable strings.", + optional=True, + ), + requirements.BooleanRequirement( + name="bounding", + description="Show only bounding capabilities in human-readable strings.", + optional=True, + ), + requirements.BooleanRequirement( + name="ambient", + description="Show only ambient capabilities in human-readable strings.", + optional=True, + ), + ] + + def _check_capabilities_support(self): + """Checks that the framework supports at least as much capabilities as + the kernel being analysed. Otherwise, it shows a warning for the + developers. + """ + vmlinux = self.context.modules[self.config["kernel"]] + + kernel_cap_last_cap = vmlinux.object(object_type="int", offset=kernel_cap_last_cap) + vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() + if kernel_cap_last_cap > vol2_last_cap: + vollog.warning("Developers: The supported Linux capabilities of this plugin are outdated for this kernel") + + @staticmethod + def _decode_cap(cap: interfaces.objects.ObjectInterface) -> str: + """Returns a textual representation of the capability set. + The format is a comma-separated list of capabilitites. In order to + summarize the output and if all the capabilities are enabled, instead of + the individual capabilities, the special name "all" will be shown. + + Args: + cap: Kernel capability object. Usually a 'kernel_cap_struct' struct + + Returns: + str: A string with a comma separated list of decoded capabilities + """ + if isinstance(cap, renderers.NotAvailableValue): + return cap + + cap_value = cap.get_capabilities() + if cap_value == 0: + return "-" + + CAP_FULL = 0xffffffff + if cap_value == CAP_FULL: + return "all" + + return ", ".join(cap.enumerate_capabilities()) + + @classmethod + def get_task_capabilities(cls, task: interfaces.objects.ObjectInterface) -> Dict: + """Returns a dict with the task basic information along with its capabilities + + Args: + task: A task object from where to get the fields. + + Returns: + dict: A dict with the task basic information along with its capabilities + """ + task_cred = task.real_cred + fields = { + "common": [ + utility.array_to_string(task.comm), + int(task.pid), + int(task.tgid), + int(task.parent.pid), + int(task.cred.euid), + ], + "capabilities": [ + task_cred.cap_inheritable, + task_cred.cap_permitted, + task_cred.cap_effective, + task_cred.cap_bset, + ] + } + + # Ambient capabilities were added in kernels 4.3.6 + if task_cred.has_member("cap_ambient"): + fields["capabilities"].append(task_cred.cap_ambient) + else: + fields["capabilities"].append(renderers.NotAvailableValue()) + + return fields + + def get_tasks_capabilities(self, tasks: List[interfaces.objects.ObjectInterface]) -> Iterable[Dict]: + """Yields a dict for each task containing the task's basic information along with its capabilities + + Args: + tasks: An iterable with the tasks to process. + + Yields: + Iterable[Dict]: A dict for each task containing the task's basic information along with its capabilities + """ + for task in tasks: + if task.is_kernel_thread: + continue + + yield self.get_task_capabilities(task) + + def _generator(self, tasks: Iterable[interfaces.objects.ObjectInterface]) -> Iterable[Tuple[int, Tuple]]: + for fields in self.get_tasks_capabilities(tasks): + selected_fields = fields["common"] + cap_inh, cap_prm, cap_eff, cap_bnd, cap_amb = fields["capabilities"] + + if self.config.get("inheritable"): + selected_fields.append(self._decode_cap(cap_inh)) + elif self.config.get("permitted"): + selected_fields.append(self._decode_cap(cap_prm)) + elif self.config.get("effective"): + selected_fields.append(self._decode_cap(cap_eff)) + elif self.config.get("bounding"): + selected_fields.append(self._decode_cap(cap_bnd)) + elif self.config.get("ambient"): + selected_fields.append(self._decode_cap(cap_amb)) + else: + # Raw values + selected_fields.append(format_hints.Hex(cap_inh.get_capabilities())) + selected_fields.append(format_hints.Hex(cap_prm.get_capabilities())) + selected_fields.append(format_hints.Hex(cap_eff.get_capabilities())) + selected_fields.append(format_hints.Hex(cap_bnd.get_capabilities())) + + # Ambient capabilities were added in kernels 4.3.6 + if isinstance(cap_amb, renderers.NotAvailableValue): + selected_fields.append(cap_amb) + else: + selected_fields.append(format_hints.Hex(cap_amb.get_capabilities())) + + yield 0, selected_fields + + def run(self): + pids = self.config.get("pids") + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks(self.context, self.config["kernel"], filter_func=pid_filter) + + columns = [ + ("Name", str), + ("Tid", int), + ("Pid", int), + ("PPid", int), + ("EUID", int), + ] + + if self.config.get("inheritable"): + columns.append(("cap_inheritable", str)) + elif self.config.get("permitted"): + columns.append(("cap_permitted", str)) + elif self.config.get("effective"): + columns.append(("cap_effective", str)) + elif self.config.get("bounding"): + columns.append(("cap_bounding", str)) + elif self.config.get("ambient"): + columns.append(("cap_ambient", str)) + else: + columns.append(("cap_inheritable", format_hints.Hex)) + columns.append(("cap_permitted", format_hints.Hex)) + columns.append(("cap_effective", format_hints.Hex)) + columns.append(("cap_bounding", format_hints.Hex)) + columns.append(("cap_ambient", format_hints.Hex)) + + return renderers.TreeGrid(columns, self._generator(tasks)) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 36314b2c6..0bab9dedf 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -28,6 +28,8 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("fs_struct", extensions.fs_struct) self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) + self.set_type_class("cred", extensions.cred) + self.set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9ac98b5da..8a8785bd0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -5,7 +5,7 @@ import collections.abc import logging import socket as socket_module -from typing import Generator, Iterable, Iterator, Optional, Tuple +from typing import Generator, Iterable, Iterator, Optional, Tuple, List from volatility3.framework import constants from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY @@ -1428,3 +1428,109 @@ class bpf_prog(objects.StructType): # kernel < 3.18.140 raise AttributeError("Unable to find the BPF type") + +class cred(objects.StructType): + # struct cred was added in kernels 2.6.29 + def _get_cred_int_value(self, member: str) -> int: + """Helper to obtain the right cred member value for the current kernel. + + Args: + member (str): The requested cred member name to obtain its value + + Raises: + AttributeError: When the requested cred member doesn't exist + AttributeError: When the cred implementation is not supported. + + Returns: + int: The cred member value + """ + if not self.has_member(member): + raise AttributeError(f"struct cred doesn't have a '{member}' member") + + cred_val = self.member(member) + if hasattr(cred_val, "val"): + # From kernels 3.5.7 on it is a 'kuid_t' type + value = cred_val.val + elif isinstance(cred_val, objects.Integer): + # From at least 2.6.30 and until 3.5.7 it was a 'uid_t' type which was an 'unsigned int' + value = cred_val + else: + raise AttributeError("Kernel struct cred is not supported") + + return int(value) + + @property + def euid(self): + """Returns the effective user ID + + Returns: + int: the effective user ID value + """ + return self._get_cred_int_value("euid") + + +class kernel_cap_struct(objects.StructType): + # struct kernel_cap_struct was added in kernels 2.5.0 + @classmethod + def get_last_cap_value(cls) -> int: + """Returns the latest capability ID supported by the framework. + + Returns: + int: The latest supported capability ID supported by the framework. + """ + return len(constants.CAPABILITIES) - 1 + + @classmethod + def capabilities_to_string(cls, capabilities_bitfield: int) -> List[str]: + """Translates a capability bitfield to a list of capability strings. + + Args: + capabilities_bitfield (int): The capability bitfield value. + + Returns: + List[str]: A list of capability strings. + """ + + capabilities = [] + for bit, name in enumerate(constants.CAPABILITIES): + if capabilities_bitfield & (1 << bit) != 0: + capabilities.append(name) + + return capabilities + + def get_capabilities(self) -> int: + """Returns the capability bitfield value + + Returns: + int: The capability bitfield value. + """ + # In kernels 2.6.25.20 the kernel_cap_struct::cap became and array + cap_value = self.cap[0] if isinstance(self.cap, objects.Array) else self.cap + return int(cap_value & 0xffffffff) + + def enumerate_capabilities(self) -> List[str]: + """Returns the list of capability strings. + + Returns: + List[str]: The list of capability strings. + """ + capabilities_value = self.get_capabilities() + return self.capabilities_to_string(capabilities_value) + + def has_capability(self, capability: str) -> bool: + """Checks if the given capability string is enabled. + + Args: + capability (str): A string representing the capability i.e. dac_read_search + + Raises: + AttributeError: If the fiven capability is unknown to the framework. + + Returns: + bool: "True" if the given capability is enabled. + """ + if capability not in constants.CAPABILITIES: + raise AttributeError(f"Unknown capability with name '{capability}'") + + cap_value = 1 << constants.CAPABILITIES.index(capability) + return cap_value & self.get_capabilities() != 0 From d5d318d3cfd43f4d33fdb9252b120b5ed8666a12 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 17 May 2023 15:26:24 +0100 Subject: [PATCH 378/526] Fix wrong constants module --- .../framework/symbols/linux/extensions/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 8a8785bd0..f626db434 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,6 +13,7 @@ from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES +from volatility3.framework.constants.linux import CAPABILITIES from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility @@ -1478,7 +1479,7 @@ class kernel_cap_struct(objects.StructType): Returns: int: The latest supported capability ID supported by the framework. """ - return len(constants.CAPABILITIES) - 1 + return len(CAPABILITIES) - 1 @classmethod def capabilities_to_string(cls, capabilities_bitfield: int) -> List[str]: @@ -1492,7 +1493,7 @@ class kernel_cap_struct(objects.StructType): """ capabilities = [] - for bit, name in enumerate(constants.CAPABILITIES): + for bit, name in enumerate(CAPABILITIES): if capabilities_bitfield & (1 << bit) != 0: capabilities.append(name) @@ -1529,8 +1530,8 @@ class kernel_cap_struct(objects.StructType): Returns: bool: "True" if the given capability is enabled. """ - if capability not in constants.CAPABILITIES: + if capability not in CAPABILITIES: raise AttributeError(f"Unknown capability with name '{capability}'") - cap_value = 1 << constants.CAPABILITIES.index(capability) + cap_value = 1 << CAPABILITIES.index(capability) return cap_value & self.get_capabilities() != 0 From 4f5ca6116c73c456a9e55c56a41882b5bcccf7ac Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 17 May 2023 16:07:56 +0100 Subject: [PATCH 379/526] Add black recommendations --- .../framework/plugins/linux/capabilities.py | 24 +++++++++++++------ .../symbols/linux/extensions/__init__.py | 3 ++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 8bcd80eef..ed08c4c05 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -74,10 +74,14 @@ class Capabilities(plugins.PluginInterface): """ vmlinux = self.context.modules[self.config["kernel"]] - kernel_cap_last_cap = vmlinux.object(object_type="int", offset=kernel_cap_last_cap) + kernel_cap_last_cap = vmlinux.object( + object_type="int", offset=kernel_cap_last_cap + ) vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() if kernel_cap_last_cap > vol2_last_cap: - vollog.warning("Developers: The supported Linux capabilities of this plugin are outdated for this kernel") + vollog.warning( + "Developers: The supported Linux capabilities of this plugin are outdated for this kernel" + ) @staticmethod def _decode_cap(cap: interfaces.objects.ObjectInterface) -> str: @@ -99,7 +103,7 @@ class Capabilities(plugins.PluginInterface): if cap_value == 0: return "-" - CAP_FULL = 0xffffffff + CAP_FULL = 0xFFFFFFFF if cap_value == CAP_FULL: return "all" @@ -129,7 +133,7 @@ class Capabilities(plugins.PluginInterface): task_cred.cap_permitted, task_cred.cap_effective, task_cred.cap_bset, - ] + ], } # Ambient capabilities were added in kernels 4.3.6 @@ -140,7 +144,9 @@ class Capabilities(plugins.PluginInterface): return fields - def get_tasks_capabilities(self, tasks: List[interfaces.objects.ObjectInterface]) -> Iterable[Dict]: + def get_tasks_capabilities( + self, tasks: List[interfaces.objects.ObjectInterface] + ) -> Iterable[Dict]: """Yields a dict for each task containing the task's basic information along with its capabilities Args: @@ -155,7 +161,9 @@ class Capabilities(plugins.PluginInterface): yield self.get_task_capabilities(task) - def _generator(self, tasks: Iterable[interfaces.objects.ObjectInterface]) -> Iterable[Tuple[int, Tuple]]: + def _generator( + self, tasks: Iterable[interfaces.objects.ObjectInterface] + ) -> Iterable[Tuple[int, Tuple]]: for fields in self.get_tasks_capabilities(tasks): selected_fields = fields["common"] cap_inh, cap_prm, cap_eff, cap_bnd, cap_amb = fields["capabilities"] @@ -188,7 +196,9 @@ class Capabilities(plugins.PluginInterface): def run(self): pids = self.config.get("pids") pid_filter = pslist.PsList.create_pid_filter(pids) - tasks = pslist.PsList.list_tasks(self.context, self.config["kernel"], filter_func=pid_filter) + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=pid_filter + ) columns = [ ("Name", str), diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f626db434..0051cdc2f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1430,6 +1430,7 @@ class bpf_prog(objects.StructType): # kernel < 3.18.140 raise AttributeError("Unable to find the BPF type") + class cred(objects.StructType): # struct cred was added in kernels 2.6.29 def _get_cred_int_value(self, member: str) -> int: @@ -1507,7 +1508,7 @@ class kernel_cap_struct(objects.StructType): """ # In kernels 2.6.25.20 the kernel_cap_struct::cap became and array cap_value = self.cap[0] if isinstance(self.cap, objects.Array) else self.cap - return int(cap_value & 0xffffffff) + return int(cap_value & 0xFFFFFFFF) def enumerate_capabilities(self) -> List[str]: """Returns the list of capability strings. From f0bd9787160c8bca29e52090dc484dea213004e7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 20 May 2023 12:17:09 +0200 Subject: [PATCH 380/526] Fix typo --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0051cdc2f..4b1c53683 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1526,7 +1526,7 @@ class kernel_cap_struct(objects.StructType): capability (str): A string representing the capability i.e. dac_read_search Raises: - AttributeError: If the fiven capability is unknown to the framework. + AttributeError: If the given capability is unknown to the framework. Returns: bool: "True" if the given capability is enabled. From bea981180d5044aa979d39e6f2e06e13909513a1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 22 May 2023 01:32:00 +0100 Subject: [PATCH 381/526] Prevent potential dentry pointer memory smear --- volatility3/framework/symbols/linux/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 36314b2c6..3ddffb49a 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -200,8 +200,11 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): Returns: str: A file (or sock pipe) pathname relative to the task's root directory. """ + + # Memory smear protection: Check that both the file and dentry pointers are valids. try: dentry = filp.get_dentry() + dentry.is_root() except exceptions.InvalidAddressException: return "" From 0f42f0eaf1bbc47b86a180289a5d393cbcafc6d7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 22 May 2023 01:39:14 +0100 Subject: [PATCH 382/526] fix typo --- volatility3/framework/symbols/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3ddffb49a..339f6417f 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -201,7 +201,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): str: A file (or sock pipe) pathname relative to the task's root directory. """ - # Memory smear protection: Check that both the file and dentry pointers are valids. + # Memory smear protection: Check that both the file and dentry pointers are valid. try: dentry = filp.get_dentry() dentry.is_root() From b9e5cfb393c35d005235996804be6c6853a34b73 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 22 May 2023 15:05:39 +0100 Subject: [PATCH 383/526] Core: Protect from clearing a non-existant cache Issue kindly raised by @garanews, thanks! 5:D --- volatility3/framework/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index c7b23a9c3..9c17846a8 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -224,4 +224,7 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: def clear_cache(complete=False): - os.unlink(os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)) + try: + os.unlink(os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)) + except FileNotFoundError: + vollog.log(constants.LOGLEVEL_VVVV, "Attempting to clear a non-existant cache") From aa04b8ca3d6ba9db3d7658436927a2430cc6371e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 16 Jun 2023 18:03:07 +0100 Subject: [PATCH 384/526] Windows: Fix VAD offset canonicalization #969 --- volatility3/framework/plugins/windows/vadinfo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 812affe86..abc6142fe 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -198,6 +198,7 @@ class VadInfo(interfaces.plugins.PluginInterface): def _generator(self, procs): kernel = self.context.modules[self.config["kernel"]] + kernel_layer = self.context.layers[kernel.layer_name] def passthrough(_: interfaces.objects.ObjectInterface) -> bool: return False @@ -229,7 +230,7 @@ class VadInfo(interfaces.plugins.PluginInterface): ( proc.UniqueProcessId, process_name, - format_hints.Hex(vad.vol.offset), + format_hints.Hex(kernel_layer.canonicalize(vad.vol.offset)), format_hints.Hex(vad.get_start()), format_hints.Hex(vad.get_end()), vad.get_tag(), From a9698e4e731841932a17153847b264f0ceeb70a5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 16:31:48 +0100 Subject: [PATCH 385/526] Documentation: minor fixes/updates --- doc/source/conf.py | 144 ++++++++++++------ doc/source/getting-started-mac-tutorial.rst | 8 +- .../framework/plugins/linux/sockstat.py | 5 +- .../symbols/linux/extensions/__init__.py | 32 ++-- 4 files changed, 121 insertions(+), 68 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 895219b25..8b467ec1d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -21,57 +21,72 @@ import sphinx.ext.apidoc def setup(app): - volatility_directory = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'volatility3')) + volatility_directory = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "volatility3") + ) source_dir = os.path.abspath(os.path.dirname(__file__)) - sphinx.ext.apidoc.main(argv = ['-e', '-M', '-f', '-T', '-o', source_dir, volatility_directory]) + sphinx.ext.apidoc.main( + argv=["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory] + ) # Go through the volatility3.framework.plugins files and change them to volatility3.plugins for dir, _, files in os.walk(os.path.dirname(__file__)): for filename in files: - if filename.startswith('volatility3.framework.plugins') and filename != 'volatility3.framework.plugins.rst': + if ( + filename.startswith("volatility3.framework.plugins") + and filename != "volatility3.framework.plugins.rst" + ): # Change all volatility3.framework.plugins to volatility3.plugins in the file # Rename the file - new_filename = filename.replace('volatility3.framework.plugins', 'volatility3.plugins') + new_filename = filename.replace( + "volatility3.framework.plugins", "volatility3.plugins" + ) replace_string = b"Submodules\n----------\n\n.. toctree::\n\n" submodules = replace_string # If file already exists, read out the subpackages entries from it add them to the new list if os.path.exists(os.path.join(dir, new_filename)): - with open(os.path.join(dir, new_filename), 'rb') as newfile: + with open(os.path.join(dir, new_filename), "rb") as newfile: data = newfile.read() index = data.find(replace_string) if index > -1: submodules = data[index:] - with open(os.path.join(dir, new_filename), 'wb') as newfile: + with open(os.path.join(dir, new_filename), "wb") as newfile: with open(os.path.join(dir, filename), "rb") as oldfile: line = oldfile.read() - correct_plugins = line.replace(b'volatility3.framework.plugins', b'volatility3.plugins') - correct_submodules = correct_plugins.replace(replace_string, submodules) + correct_plugins = line.replace( + b"volatility3.framework.plugins", b"volatility3.plugins" + ) + correct_submodules = correct_plugins.replace( + replace_string, submodules + ) newfile.write(correct_submodules) os.remove(os.path.join(dir, filename)) - elif filename == 'volatility3.framework.rst': + elif filename == "volatility3.framework.rst": with open(os.path.join(dir, filename), "rb") as contents: lines = contents.readlines() plugins_seen = False with open(os.path.join(dir, filename), "wb") as contents: for line in lines: - if b'volatility3.framework.plugins' in line: + if b"volatility3.framework.plugins" in line: plugins_seen = True - if plugins_seen and line == b'': - contents.write(b' volatility3.plugins') + if plugins_seen and line == b"": + contents.write(b" volatility3.plugins") contents.write(line) - elif filename == 'volatility3.plugins.rst': + elif filename == "volatility3.plugins.rst": with open(os.path.join(dir, filename), "rb") as contents: lines = contents.readlines() - with open(os.path.join(dir, 'volatility3.framework.plugins.rst'), "rb") as contents: + with open( + os.path.join(dir, "volatility3.framework.plugins.rst"), "rb" + ) as contents: real_lines = contents.readlines() # Process real_lines for line_index in range(len(real_lines)): - if b'Submodules' in real_lines[line_index]: + if b"Submodules" in real_lines[line_index]: break else: line_index = len(real_lines) @@ -82,36 +97,52 @@ def setup(app): for line in lines: contents.write(line) for line in submodule_lines: - contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins')) + contents.write( + line.replace( + b"volatility3.framework.plugins", b"volatility3.plugins" + ) + ) # Clear up the framework.plugins page - with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "rb") as contents: + with open( + os.path.join(os.path.dirname(__file__), "volatility3.framework.plugins.rst"), + "rb", + ) as contents: real_lines = contents.readlines() - with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "wb") as contents: + with open( + os.path.join(os.path.dirname(__file__), "volatility3.framework.plugins.rst"), + "wb", + ) as contents: for line in real_lines: - if b'volatility3.framework.plugins.' not in line: + if b"volatility3.framework.plugins." not in line: contents.write(line) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('../..')) +sys.path.insert(0, os.path.abspath("../..")) from volatility3.framework import constants # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '2.0' +needs_sphinx = "2.0" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.autosectionlabel' + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", + "sphinx.ext.autosectionlabel", ] autosectionlabel_prefix_document = True @@ -119,7 +150,7 @@ autosectionlabel_prefix_document = True try: import sphinx_autodoc_typehints - extensions.append('sphinx_autodoc_typehints') + extensions.append("sphinx_autodoc_typehints") except ImportError: # If the autodoc typehints extension isn't available, carry on regardless pass @@ -128,17 +159,17 @@ except ImportError: # templates_path = ['tools/templates'] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'Volatility 3' -copyright = '2012-2022, Volatility Foundation' +project = "Volatility 3" +copyright = "2012-2022, Volatility Foundation" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -147,7 +178,7 @@ copyright = '2012-2022, Volatility Foundation' # The full version, including alpha/beta/rc tags. release = constants.PACKAGE_VERSION # The short X.Y version. -version = ".".join(release.split('.')[0:2]) +version = ".".join(release.split(".")[0:2]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -180,7 +211,7 @@ add_module_names = False # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -196,8 +227,8 @@ pygments_style = 'sphinx' # html_theme = 'pydoctheme' # html_theme_options = {'collapsiblesidebar': True} # html_theme_path = ['tools'] -html_theme = 'sphinx_rtd_theme' -html_theme_options = {'logo_only': True} +html_theme = "sphinx_rtd_theme" +html_theme_options = {"logo_only": True} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -216,17 +247,17 @@ html_theme_options = {'logo_only': True} # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = '_static/vol.png' +html_logo = "_static/vol.png" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -html_favicon = '_static/favicon.ico' +html_favicon = "_static/favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied @@ -275,17 +306,15 @@ html_static_path = ['_static'] # html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'Volatilitydoc' +htmlhelp_basename = "Volatilitydoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # 'preamble': '', } @@ -294,7 +323,13 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'Volatility.tex', 'Volatility 3 Documentation', 'Volatility Foundation', 'manual'), + ( + "index", + "Volatility.tex", + "Volatility 3 Documentation", + "Volatility Foundation", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of @@ -321,7 +356,15 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [('vol-cli', 'volatility', 'Volatility 3 Documentation', ['Volatility Foundation'], 1)] +man_pages = [ + ( + "vol-cli", + "volatility", + "Volatility 3 Documentation", + ["Volatility Foundation"], + 1, + ) +] # If true, show URL addresses after external links. # man_show_urls = False @@ -332,8 +375,15 @@ man_pages = [('vol-cli', 'volatility', 'Volatility 3 Documentation', ['Volatilit # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Volatility', 'Volatility 3 Documentation', 'Volatility Foundation', 'Volatility', - 'Memory forensics framework.', 'Miscellaneous'), + ( + "index", + "Volatility", + "Volatility 3 Documentation", + "Volatility Foundation", + "Volatility", + "Memory forensics framework.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. @@ -349,10 +399,14 @@ texinfo_documents = [ # texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {"python": ("http://docs.python.org/", None)} # -- Autodoc options ------------------------------------------------------- # autodoc_member_order = 'groupwise' -autodoc_default_options = {'members': True, 'inherited-members': True, 'show-inheritance': True} -autoclass_content = 'both' +autodoc_default_options = { + "members": True, + "inherited-members": True, + "show-inheritance": True, +} +autoclass_content = "both" diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index cfb0afa9a..42e58c0d5 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -78,10 +78,10 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-mac-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. mac.pslist -~~~~~~~~~~~~ +~~~~~~~~~~ .. code-block:: shell-session @@ -107,7 +107,7 @@ mac.pslist ``mac.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. mac.pstree -~~~~~~~~~~~~ +~~~~~~~~~~ .. code-block:: shell-session @@ -128,7 +128,7 @@ mac.pstree ``mac.pstree`` helps us to display the parent child relationships between processes. mac.ifconfig -~~~~~~~~~~ +~~~~~~~~~~~~ .. code-block:: shell-session diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index f06b3ad8e..fa67122ba 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -83,7 +83,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock: Kernel generic `sock` object Returns a tuple with: - sock: The respective kernel's *_sock object for that socket family + sock: The respective kernel's \*_sock object for that socket family sock_stat: A tuple with the source and destination (address and port) along with its state string socket_filter: A dictionary with information about the socket filter """ @@ -501,8 +501,7 @@ class Sockstat(plugins.PluginInterface): family: Socket family string (AF_UNIX, AF_INET, etc) sock_type: Socket type string (STREAM, DGRAM, etc) protocol: Protocol string (UDP, TCP, etc) - sock_fields: A tuple with the *_sock object, the sock stats and the - extended info dictionary + sock_fields: A tuple with the \*_sock object, the sock stats and the extended info dictionary """ vmlinux = context.modules[symbol_table] diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9ac98b5da..64c038f5a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -794,7 +794,7 @@ class mount(objects.StructType): """Gets the fs where we are mounted on Returns: - A 'mount *' + A mount pointer """ return self.mnt_parent @@ -802,7 +802,7 @@ class mount(objects.StructType): """Gets the dentry of the mountpoint Returns: - A 'dentry *' + A dentry pointer """ return self.mnt_mountpoint @@ -839,7 +839,7 @@ class mount(objects.StructType): """Returns the root of the mounted tree Returns: - A 'dentry *' + A dentry pointer """ vfsmnt = self.get_vfsmnt_current() dentry = vfsmnt.mnt_root @@ -850,7 +850,7 @@ class mount(objects.StructType): """Returns the parent root of the mounted tree Returns: - A 'dentry *' + A dentry pointer """ return self.get_mnt_parent().get_dentry_current() @@ -970,17 +970,17 @@ class vfsmount(objects.StructType): """Helper to make sure it is comparing two pointers to 'vfsmount'. Depending on the kernel version, the calling object (self) could be - a 'vfsmount *' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust + a 'vfsmount \*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust in the framework "auto" dereferencing ability to assure that when we reach this point 'self' will be a 'vfsmount' already and self.vol.offset - a 'vfsmount *' and not a 'vfsmount **'. The argument must be a 'vfsmount *'. + a 'vfsmount \*' and not a 'vfsmount \*\*'. The argument must be a 'vfsmount \*'. Typically, it's called from do_get_path(). Args: - vfsmount_ptr (vfsmount *): A pointer to a 'vfsmount' + vfsmount_ptr (vfsmount \*): A pointer to a 'vfsmount' Raises: - exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount *' + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \*' Returns: bool: 'True' if the given argument points to the the same 'vfsmount' @@ -1010,7 +1010,7 @@ class vfsmount(objects.StructType): """Returns the current fs where we are mounted on Returns: - A 'vfsmount *' + A vfsmount pointer """ return self.get_mnt_parent() @@ -1018,8 +1018,8 @@ class vfsmount(objects.StructType): """Gets the parent fs (vfsmount) to where it's mounted on Returns: - For kernels < 3.3.8: A 'vfsmount *' - For kernels >= 3.3.8: A 'vfsmount' + For kernels < 3.3.8: A vfsmount pointer + For kernels >= 3.3.8: A vfsmount object """ if self._is_kernel_prior_to_struct_mount(): return self.get_mnt_parent() @@ -1030,7 +1030,7 @@ class vfsmount(objects.StructType): """Returns the root of the mounted tree Returns: - A 'dentry *' + A dentry pointer """ if self._is_kernel_prior_to_struct_mount(): return self.get_mnt_mountpoint() @@ -1041,7 +1041,7 @@ class vfsmount(objects.StructType): """Returns the parent root of the mounted tree Returns: - A 'dentry *' + A dentry pointer """ if self._is_kernel_prior_to_struct_mount(): return self.get_mnt_mountpoint() @@ -1052,8 +1052,8 @@ class vfsmount(objects.StructType): """Gets the mnt_parent member. Returns: - For kernels < 3.3.8: A 'vfsmount *' - For kernels >= 3.3.8: A 'mount *' + For kernels < 3.3.8: A vfsmount pointer + For kernels >= 3.3.8: A mount pointer """ if self._is_kernel_prior_to_struct_mount(): return self.mnt_parent @@ -1064,7 +1064,7 @@ class vfsmount(objects.StructType): """Gets the dentry of the mountpoint Returns: - A 'dentry *' + A dentry pointer """ if self.has_member("mnt_mountpoint"): return self.mnt_mountpoint From 1ff2d80b44cc919474f502d37840c4bf5fe0e6f1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 17:01:57 +0100 Subject: [PATCH 386/526] Documentation: Update basics - memory layer information --- doc/source/basics.rst | 73 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/doc/source/basics.rst b/doc/source/basics.rst index d493c61b3..1b8e64780 100644 --- a/doc/source/basics.rst +++ b/doc/source/basics.rst @@ -1,7 +1,7 @@ Volatility 3 Basics =================== -Volatility splits memory analysis down to several components: +Volatility splits memory analysis down to several components. The main ones are: * Memory layers * Templates and Objects @@ -13,22 +13,65 @@ which acts as a container for all the various layers and tables necessary to con Memory layers ------------- -A memory layer is a body of data that can be accessed by requesting data at a specific address. Memory is seen as -sequential when accessed through sequential addresses, however, there is no obligation for the data to be stored -sequentially, and modern processors tend to store the memory in a paged format. Moreover, there is no need for the data -to be stored in an easily accessible format, it could be encoded or encrypted or more, it could be the combination of -two other sources. These are typically handled by programs that process file formats, or the memory manager of the -processor, but these are all translations (either in the geometric or linguistic sense) of the original data. +A memory layer is a body of data that can be accessed by requesting data at a specific address. At its lowest level +this data is stored on a phyiscal medium (RAM) and very early computers addresses locations in memory directly. However, +as the size of memory increased and it became more difficult to manage memory most architectures moved to a "paged" model +of memory, where the available memory is cut into specific fixed-sized pages. To help further, programs can ask for any address +and the processor will look up their (virtual) address in a map, to find out where the (physical) address that it lives at is, +in the actual memory of the system. -In Volatility 3 this is represented by a directed graph, whose end nodes are -:py:class:`DataLayers ` and whose internal nodes are -specifically called a :py:class:`TranslationLayer `. -In this way, a raw memory image in the LiME file format and a page file can be -combined to form a single Intel virtual memory layer. When requesting addresses from the Intel layer, it will use the -Intel memory mapping algorithm, along with the address of the directory table base or page table map, to translate that +Volatility can work with these layers as long as it knows the map (so, for example that virtual address `1` looks up at physical +address `9`). The automagic that runs at the start of every volatility session often locates the kernel's memory map, and creates +a kernel virtual layer, which allows for kernel addresses to be looked up and the correct data returned. There can, however, be +several maps, and in general there is a different map for each process (although a portion of the operating system's memory is +usually mapped to the same location across all processes). The maps may take the same address but point to a different part of +physical memory. It also means that two processes could theoretically share memory, but having an virtual address mapped to the +same physical address as another process. See the worked example below for more information. + +To translate an address on a layer, call :py:meth:`layer.mapping(offset, length, ignore_errors) ` and it will return a list of chunks without overlap, in order, +for the requested range. If a portion cannot be mapped, an exception will be thrown unless `ignore_errors` is true. Each +chunk will contain the original offset of the chunk, the translated offset, the original size and the translated size of +the chunk, as well as the lower layer the chunk lives within. + +Worked example +^^^^^^^^^^^^^^ + +The operating system and two programs may all appear to have access to all of physical memory, but actually the maps they each have +mean they each see something different: + +.. code-block:: + :caption: Memory mapping example + + Operating system map Physical Memory + 1 -> 9 1 - Free + 2 -> 3 2 - OS.4, Process 1.4, Process 2.4 + 3 -> 7 3 - OS.2 + 4 -> 2 4 - Free + 5 - Free + Process 1 map 6 - Process 1.2, Process 2.3 + 1 -> 12 7 - OS.3 + 2 -> 6 8 - Process1.3 + 3 -> 8 9 - OS.1 + 4 -> 2 10 - Process2.1 + 11 - Free + Process 2 map 12 - Process1.1 + 1 -> 10 13 - Free + 2 -> 15 14 - Free + 3 -> 6 15 - Process2.2 + 4 -> 2 16 - Free + +In this example, part of the operating system is visible across all processes (although not all processes can write to the memory, there +is a permissions model for intel addressing which is not discussed further here).) + +In Volatility 3 mappings are represented by a directed graph of layers, whose end nodes are +:py:class:`DataLayers ` and whose internal nodes are :py:class:`TranslationLayers `. +In this way, a raw memory image in the LiME file format and a page file can be combined to form a single Intel virtual +memory layer. When requesting addresses from the Intel layer, it will use the Intel memory mapping algorithm, along +with the address of the directory table base or page table map, to translate that address into a physical address, which will then either be directed towards the swap layer or the LiME layer. Should it -be directed towards the LiME layer, the LiME file format algorithm will be translated to determine where within the file -the data is stored and that will be returned. +be directed towards the LiME layer, the LiME file format algorithm will be translate the new address to determine where +within the file the data is stored. When the :py:meth:`layer.read() ` +method is called, the translation is done automatically and the correct data gathered and combined. .. note:: Volatility 2 had a similar concept, called address spaces, but these could only stack linearly one on top of another. From 50b5d2232b7e1519156292875fb9ebc625cde4ac Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 19:00:03 +0100 Subject: [PATCH 387/526] Core: Add type parameter to object_from_symbol --- API_CHANGES.md | 4 ++++ volatility3/framework/constants/__init__.py | 4 ++-- volatility3/framework/contexts/__init__.py | 15 +++++++++++---- volatility3/framework/interfaces/context.py | 2 ++ 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index 98a08f09d..61d8781fb 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,10 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.5.0 +===== +Add in support for specifying a type override for object_from_symbol + 2.4.0 ===== Add a `get_size()` method to Windows VAD structures and fix several off-by-one issues when calculating VAD sizes. diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3a6b24ea8..de1674885 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 4 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_MINOR = 5 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index ecce5041c..81b516765 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -272,8 +272,9 @@ class Module(interfaces.context.ModuleInterface): symbol_name: str, native_layer_name: Optional[str] = None, absolute: bool = False, + object_type: Optional[Union[str, interfaces.objects.ObjectInterface]] = None, **kwargs, - ) -> "interfaces.objects.ObjectInterface": + ) -> interfaces.objects.ObjectInterface: """Returns an object based on a specific symbol (containing type and offset information) and the layer_name of the Module. This will throw a ValueError if the symbol does not contain an associated type, or if @@ -284,6 +285,7 @@ class Module(interfaces.context.ModuleInterface): symbol_name: Name of the symbol (within the module) to construct native_layer_name: Name of the layer in which constructed objects are made (for pointers) absolute: whether the symbol's address is absolute or relative to the module + object_type: Override for the type from the symobl to use (or if the symbol type is missing) """ if constants.BANG not in symbol_name: symbol_name = self.symbol_table_name + constants.BANG + symbol_name @@ -299,8 +301,13 @@ class Module(interfaces.context.ModuleInterface): if not absolute: offset += self._offset - if symbol_val.type is None: - raise TypeError(f"Symbol {symbol_val.name} has no associated type") + if object_type is None: + if symbol_val.type is None: + raise TypeError( + f"Symbol {symbol_val.name} has no associated type and no object_type specified" + ) + else: + object_type = symbol_val.type # Ensure we don't use a layer_name other than the module's, why would anyone do that? if "layer_name" in kwargs: @@ -308,7 +315,7 @@ class Module(interfaces.context.ModuleInterface): # Since type may be a template, we don't just call our own module method return self._context.object( - object_type=symbol_val.type, + object_type=object_type, layer_name=self._layer_name, offset=offset, native_layer_name=native_layer_name or self._native_layer_name, diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 7e385746d..03f2d9f1b 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -253,6 +253,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol_name: str, native_layer_name: Optional[str] = None, absolute: bool = False, + object_type: Optional[Union[str, interfaces.objects.ObjectInterface]] = None, **kwargs, ) -> "interfaces.objects.ObjectInterface": """Returns an object created using the symbol_table_name and layer_name @@ -262,6 +263,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol_name: The name of a symbol (that must be present in the module's symbol table). The symbol's associated type will be used to construct an object at the symbol's offset. native_layer_name: The native layer for objects that reference a different layer (if not the default provided during module construction) absolute: A boolean specifying whether the offset is absolute within the layer, or relative to the start of the module + object_type: Override for the type from the symobl to use (or if the symbol type is missing) Returns: The constructed object From c8e53ff16a0feab2f9d036fc3e173fe1969d8621 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 19:02:51 +0100 Subject: [PATCH 388/526] Core: Fix small typing issue in previous patch --- volatility3/framework/contexts/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 81b516765..73868a58f 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -272,9 +272,9 @@ class Module(interfaces.context.ModuleInterface): symbol_name: str, native_layer_name: Optional[str] = None, absolute: bool = False, - object_type: Optional[Union[str, interfaces.objects.ObjectInterface]] = None, + object_type: Optional[Union[str, "interfaces.objects.ObjectInterface"]] = None, **kwargs, - ) -> interfaces.objects.ObjectInterface: + ) -> "interfaces.objects.ObjectInterface": """Returns an object based on a specific symbol (containing type and offset information) and the layer_name of the Module. This will throw a ValueError if the symbol does not contain an associated type, or if From 66598f5b631a959d7cd73b8e929e95d122ca9a58 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 19:05:22 +0100 Subject: [PATCH 389/526] Core: Second fix is the charm... --- volatility3/framework/interfaces/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 03f2d9f1b..29cb41379 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -253,7 +253,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol_name: str, native_layer_name: Optional[str] = None, absolute: bool = False, - object_type: Optional[Union[str, interfaces.objects.ObjectInterface]] = None, + object_type: Optional[Union[str, "interfaces.objects.ObjectInterface"]] = None, **kwargs, ) -> "interfaces.objects.ObjectInterface": """Returns an object created using the symbol_table_name and layer_name From 6c0ce1d130092ac496c85eec294580a689c6ce2d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 20:58:36 +0100 Subject: [PATCH 390/526] Volshell: Typo Fixes #958 --- volatility3/cli/volshell/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index df369c53e..ea9e65d9b 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -480,7 +480,7 @@ class Volshell(interfaces.plugins.PluginInterface): accessor = resources.ResourceAccessor() with accessor.open(url=location) as fp: self.__console.runsource( - io.TextIOWrapper(fp.read(), encoding="utf-8"), symbol="exec" + io.TextIOWrapper(fp, encoding="utf-8").read(), symbol="exec" ) print("\nCode complete") From 241cf832f9b3676fbcf4e724c646dbfe46c24eb2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jul 2023 00:07:40 +0200 Subject: [PATCH 391/526] Plugin parameters removed. Instead, it now shows each capability set in their textual representation. The dictionary was replaced by a dataclass. Last but not leas, CAP_FULL moved to constants. --- .../framework/constants/linux/__init__.py | 2 + .../framework/plugins/linux/capabilities.py | 179 ++++++++---------- .../symbols/linux/extensions/__init__.py | 4 +- 3 files changed, 83 insertions(+), 102 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index a802e0ada..e57fa30d4 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -279,3 +279,5 @@ CAPABILITIES = ( "bpf", "checkpoint_restore", ) + +CAP_FULL = 0xFFFFFFFF diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index ed08c4c05..9a59f31c1 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -3,19 +3,50 @@ # import logging +from dataclasses import dataclass, astuple, fields from typing import Iterable, List, Tuple, Dict -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions +from volatility3.framework.constants.linux import CAP_FULL from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility -from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.linux import extensions from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) +@dataclass +class TaskData: + """Stores basic information about a task""" + + comm: str + pid: int + tgid: int + ppid: int + euid: int + + +@dataclass +class CapabilitiesData: + """Stores each set of capabilties for a task""" + + cap_inheritable: interfaces.objects.ObjectInterface + cap_permitted: interfaces.objects.ObjectInterface + cap_effective: interfaces.objects.ObjectInterface + cap_bset: interfaces.objects.ObjectInterface + cap_ambient: interfaces.objects.ObjectInterface + + def astuple(self) -> Tuple: + """Returns a shallow copy of the capability sets in a tuple. + + Otherwise, when dataclasses.astuple() performs a deep-copy recursion on + ObjectInterface will take a substantial amount of time. + """ + return tuple(getattr(self, field.name) for field in fields(self)) + + class Capabilities(plugins.PluginInterface): """Lists process capabilities""" @@ -40,43 +71,26 @@ class Capabilities(plugins.PluginInterface): element_type=int, optional=True, ), - requirements.BooleanRequirement( - name="inheritable", - description="Show only inheritable capabilities in human-readable strings.", - optional=True, - ), - requirements.BooleanRequirement( - name="permitted", - description="Show only permitted capabilities in human-readable strings.", - optional=True, - ), - requirements.BooleanRequirement( - name="effective", - description="Show only effective capabilities in human-readable strings.", - optional=True, - ), - requirements.BooleanRequirement( - name="bounding", - description="Show only bounding capabilities in human-readable strings.", - optional=True, - ), - requirements.BooleanRequirement( - name="ambient", - description="Show only ambient capabilities in human-readable strings.", - optional=True, - ), ] - def _check_capabilities_support(self): + def _check_capabilities_support( + self, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ): """Checks that the framework supports at least as much capabilities as the kernel being analysed. Otherwise, it shows a warning for the developers. """ - vmlinux = self.context.modules[self.config["kernel"]] - kernel_cap_last_cap = vmlinux.object( - object_type="int", offset=kernel_cap_last_cap - ) + vmlinux = context.modules[vmlinux_module_name] + + try: + kernel_cap_last_cap = vmlinux.object_from_symbol(symbol_name="cap_last_cap") + except exceptions.SymbolError: + # It should be a kernel < 3.2 + return + vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() if kernel_cap_last_cap > vol2_last_cap: vollog.warning( @@ -103,7 +117,6 @@ class Capabilities(plugins.PluginInterface): if cap_value == 0: return "-" - CAP_FULL = 0xFFFFFFFF if cap_value == CAP_FULL: return "all" @@ -119,33 +132,32 @@ class Capabilities(plugins.PluginInterface): Returns: dict: A dict with the task basic information along with its capabilities """ + task_data = TaskData( + comm=utility.array_to_string(task.comm), + pid=int(task.pid), + tgid=int(task.tgid), + ppid=int(task.parent.pid), + euid=int(task.cred.euid), + ) + task_cred = task.real_cred - fields = { - "common": [ - utility.array_to_string(task.comm), - int(task.pid), - int(task.tgid), - int(task.parent.pid), - int(task.cred.euid), - ], - "capabilities": [ - task_cred.cap_inheritable, - task_cred.cap_permitted, - task_cred.cap_effective, - task_cred.cap_bset, - ], - } + capabilities_data = CapabilitiesData( + cap_inheritable=task_cred.cap_inheritable, + cap_permitted=task_cred.cap_permitted, + cap_effective=task_cred.cap_effective, + cap_bset=task_cred.cap_bset, + cap_ambient=renderers.NotAvailableValue(), + ) # Ambient capabilities were added in kernels 4.3.6 if task_cred.has_member("cap_ambient"): - fields["capabilities"].append(task_cred.cap_ambient) - else: - fields["capabilities"].append(renderers.NotAvailableValue()) + capabilities_data.cap_ambient = task_cred.cap_ambient - return fields + return task_data, capabilities_data + @classmethod def get_tasks_capabilities( - self, tasks: List[interfaces.objects.ObjectInterface] + cls, tasks: List[interfaces.objects.ObjectInterface] ) -> Iterable[Dict]: """Yields a dict for each task containing the task's basic information along with its capabilities @@ -156,44 +168,23 @@ class Capabilities(plugins.PluginInterface): Iterable[Dict]: A dict for each task containing the task's basic information along with its capabilities """ for task in tasks: - if task.is_kernel_thread: - continue - - yield self.get_task_capabilities(task) + yield cls.get_task_capabilities(task) def _generator( self, tasks: Iterable[interfaces.objects.ObjectInterface] ) -> Iterable[Tuple[int, Tuple]]: - for fields in self.get_tasks_capabilities(tasks): - selected_fields = fields["common"] - cap_inh, cap_prm, cap_eff, cap_bnd, cap_amb = fields["capabilities"] + for task_fields, capabilities_fields in self.get_tasks_capabilities(tasks): + task_fields = astuple(task_fields) - if self.config.get("inheritable"): - selected_fields.append(self._decode_cap(cap_inh)) - elif self.config.get("permitted"): - selected_fields.append(self._decode_cap(cap_prm)) - elif self.config.get("effective"): - selected_fields.append(self._decode_cap(cap_eff)) - elif self.config.get("bounding"): - selected_fields.append(self._decode_cap(cap_bnd)) - elif self.config.get("ambient"): - selected_fields.append(self._decode_cap(cap_amb)) - else: - # Raw values - selected_fields.append(format_hints.Hex(cap_inh.get_capabilities())) - selected_fields.append(format_hints.Hex(cap_prm.get_capabilities())) - selected_fields.append(format_hints.Hex(cap_eff.get_capabilities())) - selected_fields.append(format_hints.Hex(cap_bnd.get_capabilities())) + capabilities_text = tuple( + self._decode_cap(cap) for cap in capabilities_fields.astuple() + ) - # Ambient capabilities were added in kernels 4.3.6 - if isinstance(cap_amb, renderers.NotAvailableValue): - selected_fields.append(cap_amb) - else: - selected_fields.append(format_hints.Hex(cap_amb.get_capabilities())) - - yield 0, selected_fields + yield 0, task_fields + capabilities_text def run(self): + self._check_capabilities_support(self.context, self.config["kernel"]) + pids = self.config.get("pids") pid_filter = pslist.PsList.create_pid_filter(pids) tasks = pslist.PsList.list_tasks( @@ -206,23 +197,11 @@ class Capabilities(plugins.PluginInterface): ("Pid", int), ("PPid", int), ("EUID", int), + ("cap_inheritable", str), + ("cap_permitted", str), + ("cap_effective", str), + ("cap_bounding", str), + ("cap_ambient", str), ] - if self.config.get("inheritable"): - columns.append(("cap_inheritable", str)) - elif self.config.get("permitted"): - columns.append(("cap_permitted", str)) - elif self.config.get("effective"): - columns.append(("cap_effective", str)) - elif self.config.get("bounding"): - columns.append(("cap_bounding", str)) - elif self.config.get("ambient"): - columns.append(("cap_ambient", str)) - else: - columns.append(("cap_inheritable", format_hints.Hex)) - columns.append(("cap_permitted", format_hints.Hex)) - columns.append(("cap_effective", format_hints.Hex)) - columns.append(("cap_bounding", format_hints.Hex)) - columns.append(("cap_ambient", format_hints.Hex)) - return renderers.TreeGrid(columns, self._generator(tasks)) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 4b1c53683..641f77728 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,7 +13,7 @@ from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES -from volatility3.framework.constants.linux import CAPABILITIES +from volatility3.framework.constants.linux import CAPABILITIES, CAP_FULL from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility @@ -1508,7 +1508,7 @@ class kernel_cap_struct(objects.StructType): """ # In kernels 2.6.25.20 the kernel_cap_struct::cap became and array cap_value = self.cap[0] if isinstance(self.cap, objects.Array) else self.cap - return int(cap_value & 0xFFFFFFFF) + return cap_value & CAP_FULL def enumerate_capabilities(self) -> List[str]: """Returns the list of capability strings. From 239e164ce6b2237b1b5569bba9e540b5a5694a45 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 9 Jul 2023 23:50:13 +0100 Subject: [PATCH 392/526] Windows: Fix strings plugin missing kernel pages @eve-mem spotted that we were only recording the first kernel page in any contiguous set of kernel pages, thus missing some results. This should now be fixed. --- volatility3/framework/plugins/windows/strings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 32f3df4c8..0eaa65884 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -149,9 +149,9 @@ class Strings(interfaces.plugins.PluginInterface): for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors=True): offset, _, mapped_offset, mapped_size, maplayer = mapval for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): - cur_set = reverse_map.get(mapped_offset >> 12, set()) + cur_set = reverse_map.get(val >> 12, set()) cur_set.add(("kernel", offset)) - reverse_map[mapped_offset >> 12] = cur_set + reverse_map[val >> 12] = cur_set if progress_callback: progress_callback( (offset * 100) / layer.maximum_address, From 63c7719763d8d5c5080840bad740d7fa6be14816 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jul 2023 01:01:30 +0200 Subject: [PATCH 393/526] Fixed returned types. An empty string is now returned when no capability is enabled. --- .../framework/plugins/linux/capabilities.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 9a59f31c1..aa3edfe4b 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -114,8 +114,8 @@ class Capabilities(plugins.PluginInterface): return cap cap_value = cap.get_capabilities() - if cap_value == 0: - return "-" + if not cap_value: + return "" if cap_value == CAP_FULL: return "all" @@ -123,14 +123,16 @@ class Capabilities(plugins.PluginInterface): return ", ".join(cap.enumerate_capabilities()) @classmethod - def get_task_capabilities(cls, task: interfaces.objects.ObjectInterface) -> Dict: + def get_task_capabilities( + cls, task: interfaces.objects.ObjectInterface + ) -> Tuple[TaskData, CapabilitiesData]: """Returns a dict with the task basic information along with its capabilities Args: task: A task object from where to get the fields. Returns: - dict: A dict with the task basic information along with its capabilities + A tuple with the task basic information and its capabilities """ task_data = TaskData( comm=utility.array_to_string(task.comm), @@ -158,14 +160,14 @@ class Capabilities(plugins.PluginInterface): @classmethod def get_tasks_capabilities( cls, tasks: List[interfaces.objects.ObjectInterface] - ) -> Iterable[Dict]: + ) -> Iterable[Tuple[TaskData, CapabilitiesData]]: """Yields a dict for each task containing the task's basic information along with its capabilities Args: tasks: An iterable with the tasks to process. Yields: - Iterable[Dict]: A dict for each task containing the task's basic information along with its capabilities + A tuple for each task containing the task's basic information and its capabilities """ for task in tasks: yield cls.get_task_capabilities(task) From 38e9b6baa87dbfb9ad8795621aa97a5fd1c61b30 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jul 2023 01:08:46 +0200 Subject: [PATCH 394/526] Removed Dict type from the imports --- volatility3/framework/plugins/linux/capabilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index aa3edfe4b..0eb9e3b7a 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -4,7 +4,7 @@ import logging from dataclasses import dataclass, astuple, fields -from typing import Iterable, List, Tuple, Dict +from typing import Iterable, List, Tuple from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.constants.linux import CAP_FULL From 42fe37ed52cf62b5eebe073420e9a6245307eb60 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jul 2023 01:12:20 +0200 Subject: [PATCH 395/526] Adjust docstring to the new retuned type. --- volatility3/framework/plugins/linux/capabilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 0eb9e3b7a..84d1d543f 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -126,7 +126,7 @@ class Capabilities(plugins.PluginInterface): def get_task_capabilities( cls, task: interfaces.objects.ObjectInterface ) -> Tuple[TaskData, CapabilitiesData]: - """Returns a dict with the task basic information along with its capabilities + """Returns a tuple with the task basic information along with its capabilities Args: task: A task object from where to get the fields. @@ -161,7 +161,7 @@ class Capabilities(plugins.PluginInterface): def get_tasks_capabilities( cls, tasks: List[interfaces.objects.ObjectInterface] ) -> Iterable[Tuple[TaskData, CapabilitiesData]]: - """Yields a dict for each task containing the task's basic information along with its capabilities + """Yields a tuple for each task containing the task's basic information along with its capabilities Args: tasks: An iterable with the tasks to process. From f8fc5d5e58495d9f45d900cd9041ba37d0d078dc Mon Sep 17 00:00:00 2001 From: cpuu Date: Wed, 12 Jul 2023 14:43:38 +0900 Subject: [PATCH 396/526] Update pslist.py list modules in code in alphabetical order --- volatility3/framework/plugins/mac/pslist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index dbed09818..88045a277 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -4,13 +4,13 @@ import datetime import logging -from typing import Callable, Iterable, List, Dict +from typing import Callable, Dict, Iterable, List -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility -from volatility3.framework.symbols import mac from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import mac vollog = logging.getLogger(__name__) From 9ba3d9ba4e5ed4e239c81a59c50de0ca5db08ef1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 19 Jul 2023 22:48:06 +0200 Subject: [PATCH 397/526] Several fixes: * The capabilities array was to have a 64bit bitwise. * Capabilities set has to be tested with the kernel maximum. We can't use the plugin capabilities set, otherwise we can have wrong interpretations when we tried to compress the list of capabilities to "all" * Supports kernels >= 6.3. They changed the kernel_cap_struct::cap type again to a u64 type. --- .../framework/constants/linux/__init__.py | 2 -- .../framework/plugins/linux/capabilities.py | 3 +- .../symbols/linux/extensions/__init__.py | 31 ++++++++++++++++--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index e57fa30d4..a802e0ada 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -279,5 +279,3 @@ CAPABILITIES = ( "bpf", "checkpoint_restore", ) - -CAP_FULL = 0xFFFFFFFF diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 84d1d543f..518f52603 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -7,7 +7,6 @@ from dataclasses import dataclass, astuple, fields from typing import Iterable, List, Tuple from volatility3.framework import interfaces, renderers, exceptions -from volatility3.framework.constants.linux import CAP_FULL from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -117,7 +116,7 @@ class Capabilities(plugins.PluginInterface): if not cap_value: return "" - if cap_value == CAP_FULL: + if cap_value == cap.get_kernel_cap_full(): return "all" return ", ".join(cap.enumerate_capabilities()) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0137b2cc4..bc2c6e27b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,7 +13,7 @@ from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES -from volatility3.framework.constants.linux import CAPABILITIES, CAP_FULL +from volatility3.framework.constants.linux import CAPABILITIES from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility @@ -1482,6 +1482,21 @@ class kernel_cap_struct(objects.StructType): """ return len(CAPABILITIES) - 1 + def get_kernel_cap_full(self) -> int: + """Return the maximum value allowed for this kernel for a capability + + Returns: + int: _description_ + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + try: + cap_last_cap = vmlinux.object_from_symbol(symbol_name="cap_last_cap") + except exceptions.SymbolError: + # It should be a kernel < 3.2, let's use our list of capabilities + cap_last_cap = self.get_last_cap_value() + + return (1 << cap_last_cap + 1) - 1 + @classmethod def capabilities_to_string(cls, capabilities_bitfield: int) -> List[str]: """Translates a capability bitfield to a list of capability strings. @@ -1506,9 +1521,17 @@ class kernel_cap_struct(objects.StructType): Returns: int: The capability bitfield value. """ - # In kernels 2.6.25.20 the kernel_cap_struct::cap became and array - cap_value = self.cap[0] if isinstance(self.cap, objects.Array) else self.cap - return cap_value & CAP_FULL + + if isinstance(self.cap, objects.Array): + # In 2.6.25.x <= kernels < 6.3 kernel_cap_struct::cap is an array + # to become a 64bit bitfield + cap_value = (self.cap[1] << 32) | self.cap[0] + else: + # In kernels < 2.6.25.x kernel_cap_struct::cap was a u32 + # In kernels >= 6.3 kernel_cap_struct::cap is a u64 + cap_value = self.cap + + return cap_value & self.get_kernel_cap_full() def enumerate_capabilities(self) -> List[str]: """Returns the list of capability strings. From 90da6298e0f69cbc42ef2a6e1da5863aae4d5031 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 20 Jul 2023 00:29:52 +0200 Subject: [PATCH 398/526] Added further details to the kernel_cap_struct::cap comments --- .../framework/symbols/linux/extensions/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index bc2c6e27b..527785a69 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1523,11 +1523,15 @@ class kernel_cap_struct(objects.StructType): """ if isinstance(self.cap, objects.Array): - # In 2.6.25.x <= kernels < 6.3 kernel_cap_struct::cap is an array - # to become a 64bit bitfield + # In 2.6.25.x <= kernels < 6.3 kernel_cap_struct::cap is a two + # elements __u32 array that constitutes a 64bit bitfield. + # Technically, it can also be an array of 1 element if + # _KERNEL_CAPABILITY_U32S = _LINUX_CAPABILITY_U32S_1 + # However, in the source code, that never happens. + # From 2.6.24 to 2.6.25 cap became an array of 2 elements. cap_value = (self.cap[1] << 32) | self.cap[0] else: - # In kernels < 2.6.25.x kernel_cap_struct::cap was a u32 + # In kernels < 2.6.25.x kernel_cap_struct::cap was a __u32 # In kernels >= 6.3 kernel_cap_struct::cap is a u64 cap_value = self.cap From 0a6deae6bd6de1ac3f5c5cbdefa1f201e9c16b44 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 21 Jul 2023 12:52:21 +0200 Subject: [PATCH 399/526] This will fix #985. It requires the changes in dwarf2json mentioned in this ticket. --- .../framework/symbols/linux/__init__.py | 3 +- .../symbols/linux/extensions/__init__.py | 56 +++++++++++++++---- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 5c42a436d..e9306b3d8 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -29,10 +29,11 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) - self.set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) + self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) + self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) # Mount self.set_type_class("vfsmount", extensions.vfsmount) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 527785a69..0578b629d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1472,13 +1472,13 @@ class cred(objects.StructType): class kernel_cap_struct(objects.StructType): - # struct kernel_cap_struct was added in kernels 2.5.0 + # struct kernel_cap_struct exists from 2.1.92 <= kernels < 6.3 @classmethod def get_last_cap_value(cls) -> int: """Returns the latest capability ID supported by the framework. Returns: - int: The latest supported capability ID supported by the framework. + int: The latest capability ID supported by the framework. """ return len(CAPABILITIES) - 1 @@ -1486,7 +1486,7 @@ class kernel_cap_struct(objects.StructType): """Return the maximum value allowed for this kernel for a capability Returns: - int: _description_ + int: The capability full bitfield mask """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) try: @@ -1522,17 +1522,29 @@ class kernel_cap_struct(objects.StructType): int: The capability bitfield value. """ + if not self.has_member("cap"): + raise exceptions.VolatilityException( + "Unsupported kernel capabilities implementation" + ) + if isinstance(self.cap, objects.Array): - # In 2.6.25.x <= kernels < 6.3 kernel_cap_struct::cap is a two - # elements __u32 array that constitutes a 64bit bitfield. - # Technically, it can also be an array of 1 element if - # _KERNEL_CAPABILITY_U32S = _LINUX_CAPABILITY_U32S_1 - # However, in the source code, that never happens. - # From 2.6.24 to 2.6.25 cap became an array of 2 elements. - cap_value = (self.cap[1] << 32) | self.cap[0] + if len(self.cap) == 1: + # At least in the vanilla kernel, from 2.6.24 to 2.6.25 + # kernel_cap_struct::cap become a two elements array. + # However, in some distros or custom kernel can techically + # be _KERNEL_CAPABILITY_U32S = _LINUX_CAPABILITY_U32S_1 + # Leaving this code here for the sake of ensuring completeness. + cap_value = self.cap[0] + elif len(self.cap) == 2: + # In 2.6.25.x <= kernels < 6.3 kernel_cap_struct::cap is a two + # elements __u32 array that constitutes a 64bit bitfield. + cap_value = (self.cap[1] << 32) | self.cap[0] + else: + raise exceptions.VolatilityException( + "Unsupported kernel capabilities implementation" + ) else: - # In kernels < 2.6.25.x kernel_cap_struct::cap was a __u32 - # In kernels >= 6.3 kernel_cap_struct::cap is a u64 + # In kernels < 2.6.25.x kernel_cap_struct::cap is a __u32 cap_value = self.cap return cap_value & self.get_kernel_cap_full() @@ -1563,3 +1575,23 @@ class kernel_cap_struct(objects.StructType): cap_value = 1 << CAPABILITIES.index(capability) return cap_value & self.get_capabilities() != 0 + + +class kernel_cap_t(kernel_cap_struct): + # In kernels 6.3 kernel_cap_struct became the kernel_cap_t typedef + def get_capabilities(self) -> int: + """Returns the capability bitfield value + + Returns: + int: The capability bitfield value. + """ + + if self.has_member("val"): + # In kernels >= 6.3 kernel_cap_t::val is a u64 + cap_value = self.val + else: + raise exceptions.VolatilityException( + "Unsupported kernel capabilities implementation" + ) + + return cap_value & self.get_kernel_cap_full() From 8373b5ed5ac8fe73a67c2d0c8b4f69367d60e9d0 Mon Sep 17 00:00:00 2001 From: cstation Date: Sat, 22 Jul 2023 18:52:39 +0200 Subject: [PATCH 400/526] Push ELF export limit to a constant --- volatility3/framework/constants/linux/__init__.py | 2 ++ volatility3/framework/plugins/linux/elfs.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 1b133eb42..ba7181db8 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -234,3 +234,5 @@ BLUETOOTH_PROTOCOLS = ( "HIDP", "AVDTP", ) + +ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 23bdf1c3a..c6334c977 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -105,8 +105,9 @@ class Elfs(plugins.PluginInterface): real_size = end - start - if real_size < 0 or real_size > 100000000: - continue + # Check if ELF has a legitimate size + if real_size < 0 or real_size > constants.linux.ELF_MAX_EXTRACTION_SIZE: + raise ValueError(f"The claimed size of the ELF is invalid: {real_size}") sections[start] = real_size From d53714fac84ce3c31fef6cd2b5b5dc6201ec5315 Mon Sep 17 00:00:00 2001 From: cstation Date: Sat, 22 Jul 2023 17:03:57 +0000 Subject: [PATCH 401/526] Fix linting --- volatility3/framework/constants/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index e5132f6ce..6e8883f19 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -280,4 +280,4 @@ CAPABILITIES = ( "checkpoint_restore", ) -ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 \ No newline at end of file +ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 From d872abaf302291cbffed1695bccbeb2805aa4c1e Mon Sep 17 00:00:00 2001 From: xabrouck Date: Thu, 27 Jul 2023 09:28:53 +0200 Subject: [PATCH 402/526] fix bug in snappy lib loading and make it work on macOS. --- volatility3/framework/layers/avml.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index c9c682ac4..1f4a7e053 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -19,11 +19,17 @@ vollog = logging.getLogger(__name__) try: # TODO: Find library for windows if needed try: - # Linux/Mac + # Linux lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.so.1") except OSError: lib_snappy = None + try: + # macOS + lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.1.dylib") + except OSError: + lib_snappy = None + try: if not lib_snappy: # Windows 64 @@ -31,7 +37,7 @@ try: except OSError: lib_snappy = None - if lib_snappy: + if not lib_snappy: # Windows 32 lib_snappy = ctypes.cdll.LoadLibrary("snappy32") From a226b90b4de512f3777210a361297b7a81324d53 Mon Sep 17 00:00:00 2001 From: xabrouck Date: Thu, 27 Jul 2023 09:37:57 +0200 Subject: [PATCH 403/526] previous commit would break linux snappy support --- volatility3/framework/layers/avml.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index 1f4a7e053..c825464cc 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -25,8 +25,9 @@ try: lib_snappy = None try: - # macOS - lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.1.dylib") + if not lib_snappy: + # macOS + lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.1.dylib") except OSError: lib_snappy = None From 5d4c70d5174ff354176288fcd17ca2d1acaf2f56 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 3 Aug 2023 15:16:36 +0100 Subject: [PATCH 404/526] Documentation: Fix requirements -> get_requirements #993 --- doc/source/using-as-a-library.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index c63adcfc3..fb012f1ae 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -67,9 +67,10 @@ return a dictionary of plugin names and the plugin classes. Determine what configuration options a plugin requires ------------------------------------------------------ -For each plugin class, we can call the classmethod `requirements` on it, which will return a list of objects that -adhere to the :py:class:`~volatility3.framework.interfaces.configuration.RequirementInterface` method. The various -types of Requirement are split roughly in two, +For each plugin class, we can call the classmethod +:py:func:`~volatility3.framework.interfaces.configuration.ConfigurableInterface.get_requirements` on it, which will +return a list of objects that adhere to the :py:class:`~volatility3.framework.interfaces.configuration.RequirementInterface` +method. The various types of Requirement are split roughly in two, :py:class:`~volatility3.framework.interfaces.configuration.SimpleTypeRequirement` (such as integers, booleans, floats and strings) and more complex requirements (such as lists, choices, multiple requirements, translation layer requirements or symbol table requirements). A requirement just specifies a type of data and a name, and must be From 7c82da4f5044a4bf35028c01924e659ccac5e828 Mon Sep 17 00:00:00 2001 From: xabrouck Date: Mon, 14 Aug 2023 11:53:14 +0200 Subject: [PATCH 405/526] check IoC of dirty bit in PTEs from executable VMAs. this can for example detect code injected using ptrace(). this can also detect injected code that was reset to the original code (malware uninstalled before memory dump happened). --- volatility3/framework/layers/intel.py | 9 +++++++++ volatility3/framework/plugins/linux/malfind.py | 2 +- .../framework/symbols/linux/extensions/__init__.py | 11 ++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 478eb168f..e2d89540d 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -110,6 +110,11 @@ class Intel(linear.LinearlyMappedLayer): def _page_is_valid(entry: int) -> bool: """Returns whether a particular page is valid based on its entry.""" return bool(entry & 1) + + @staticmethod + def _page_is_dirty(entry: int) -> bool: + """Returns whether a particular page is dirty based on its entry.""" + return bool(entry & (1<<6)) def canonicalize(self, addr: int) -> int: """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" @@ -259,6 +264,10 @@ class Intel(linear.LinearlyMappedLayer): except exceptions.InvalidAddressException: return False + def is_dirty(self, offset: int) -> bool: + """Returns whether the page at offset is marked dirty""" + return self._page_is_dirty(self._translate_entry(offset)[0]) + def mapping( self, offset: int, length: int, ignore_errors: bool = False ) -> Iterable[Tuple[int, int, int, int, str]]: diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 1fd005de8..332d5ede1 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -47,7 +47,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.mm.get_vma_iter(): - if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]": + if vma.is_suspicious(proc_layer) and vma.get_name(self.context, task) != "[vdso]": data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 527785a69..d2e6197ef 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -578,7 +578,7 @@ class vm_area_struct(objects.StructType): return fname # used by malfind - def is_suspicious(self): + def is_suspicious(self, proclayer): ret = False flags_str = self.get_protection() @@ -587,6 +587,15 @@ class vm_area_struct(objects.StructType): ret = True elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True + elif "x" in flags_str: + for i in range(self.vm_start,self.vm_end,constants.linux.PAGE_SHIFT): + try: + if proclayer.is_dirty(i): + vollog.warning(f"Found malicious (dirty+exec) page at {hex(i)} !") + ret = True + break + except (exceptions.PagedInvalidAddressException, exceptions.InvalidAddressException): + pass return ret From 581c493f4fdb685053b408029dd56f18a3acda78 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Aug 2023 21:35:11 +0100 Subject: [PATCH 406/526] Add in action to make old tickets stale. --- .github/workflows/stale.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 000000000..925a9f624 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,23 @@ +name: Close inactive issues +on: + schedule: + - cron: "30 1 * * *" + +jobs: + close-issues: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v5 + with: + days-before-issue-stale: 200 + days-before-issue-close: 60 + stale-issue-label: "stale" + stale-issue-message: "This issue is stale because it has been open for 200 days with no activity." + close-issue-message: "This issue was closed because it has been inactive for 60 days since being marked as stale." + days-before-pr-stale: -1 + days-before-pr-close: -1 + repo-token: ${{ secrets.GITHUB_TOKEN }} + exempt-issue-labels: "enhancement,plugin-request,question" From 804d68d94d507747d42f018baa3255dfe8635b4d Mon Sep 17 00:00:00 2001 From: Eve <120014766+eve-mem@users.noreply.github.com> Date: Thu, 17 Aug 2023 13:38:23 +0100 Subject: [PATCH 407/526] Linux: fix bug where get_process_memory_sections fails with 6.1+ kernels --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 527785a69..a40a87d5c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -203,7 +203,7 @@ class task_struct(generic.GenericIntelProcess): ) -> Generator[Tuple[int, int], None, None]: """Returns a list of sections based on the memory manager's view of this task's virtual memory.""" - for vma in self.mm.get_mmap_iter(): + for vma in self.mm.get_vma_iter(): start = int(vma.vm_start) end = int(vma.vm_end) From 6f7f1284adbbcfacd759ec0d931859e0789cfc8a Mon Sep 17 00:00:00 2001 From: xabrouck Date: Fri, 18 Aug 2023 11:37:04 +0200 Subject: [PATCH 408/526] Fix bug with PAGE_SHIFT that wasn't shifted, also greatly increases performance Use black Better logging --- volatility3/framework/layers/intel.py | 6 +++--- .../framework/plugins/linux/malfind.py | 13 ++++++++++-- .../symbols/linux/extensions/__init__.py | 21 +++++++++++++------ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index e2d89540d..046203fa6 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -110,11 +110,11 @@ class Intel(linear.LinearlyMappedLayer): def _page_is_valid(entry: int) -> bool: """Returns whether a particular page is valid based on its entry.""" return bool(entry & 1) - + @staticmethod def _page_is_dirty(entry: int) -> bool: """Returns whether a particular page is dirty based on its entry.""" - return bool(entry & (1<<6)) + return bool(entry & (1 << 6)) def canonicalize(self, addr: int) -> int: """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" @@ -267,7 +267,7 @@ class Intel(linear.LinearlyMappedLayer): def is_dirty(self, offset: int) -> bool: """Returns whether the page at offset is marked dirty""" return self._page_is_dirty(self._translate_entry(offset)[0]) - + def mapping( self, offset: int, length: int, ignore_errors: bool = False ) -> Iterable[Tuple[int, int, int, int, str]]: diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 332d5ede1..8a21afc03 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -3,7 +3,7 @@ # from typing import List - +import logging from volatility3.framework import constants, interfaces from volatility3.framework import renderers from volatility3.framework.configuration import requirements @@ -11,6 +11,8 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) + class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" @@ -47,7 +49,14 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.mm.get_vma_iter(): - if vma.is_suspicious(proc_layer) and vma.get_name(self.context, task) != "[vdso]": + vma_name = vma.get_name(self.context, task) + vollog.debug( + f"Injections : processing PID {task.pid} : VMA {vma_name} : {hex(vma.vm_start)}-{hex(vma.vm_end)}" + ) + if ( + vma.is_suspicious(proc_layer) + and vma.get_name(self.context, task) != "[vdso]" + ): data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d2e6197ef..616e54e70 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -578,7 +578,7 @@ class vm_area_struct(objects.StructType): return fname # used by malfind - def is_suspicious(self, proclayer): + def is_suspicious(self, proclayer=None): ret = False flags_str = self.get_protection() @@ -587,15 +587,24 @@ class vm_area_struct(objects.StructType): ret = True elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True - elif "x" in flags_str: - for i in range(self.vm_start,self.vm_end,constants.linux.PAGE_SHIFT): + elif proclayer and "x" in flags_str: + for i in range(self.vm_start, self.vm_end, 1 << constants.linux.PAGE_SHIFT): try: if proclayer.is_dirty(i): - vollog.warning(f"Found malicious (dirty+exec) page at {hex(i)} !") + vollog.warning( + f"Found malicious (dirty+exec) page at {hex(i)} !" + ) + # We do not attempt to find other dirty+exec pages once we have found one ret = True break - except (exceptions.PagedInvalidAddressException, exceptions.InvalidAddressException): - pass + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ) as excp: + vollog.debug(f"Unable to translate address {hex(i)} : {excp}") + # Abort as it is likely that other addresses in the same range will also fail + ret = False + break return ret From 9b5c4b04158756133fdb74d3d20ded7d2079d776 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 22 Aug 2023 15:56:18 +1000 Subject: [PATCH 409/526] fix comment typo --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0578b629d..0144e31b7 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1531,7 +1531,7 @@ class kernel_cap_struct(objects.StructType): if len(self.cap) == 1: # At least in the vanilla kernel, from 2.6.24 to 2.6.25 # kernel_cap_struct::cap become a two elements array. - # However, in some distros or custom kernel can techically + # However, in some distros or custom kernel can technically # be _KERNEL_CAPABILITY_U32S = _LINUX_CAPABILITY_U32S_1 # Leaving this code here for the sake of ensuring completeness. cap_value = self.cap[0] From 5c80a66d6dd9b3bc2400a4a8062ed30add10ce4b Mon Sep 17 00:00:00 2001 From: 616c696365 <616c696365@localhost.com> Date: Wed, 30 Aug 2023 20:12:12 +0100 Subject: [PATCH 410/526] Windows: Update pslist.py, add friendly option --- .../framework/plugins/windows/pslist.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 88697e71a..806bb678e 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -50,6 +50,12 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): default=False, optional=True, ), + requirements.BooleanRequirement( + name="friendly", + description="Display process name in dump filename", + default=False, + optional=True, + ), ] @classmethod @@ -60,6 +66,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pe_table_name: str, proc: interfaces.objects.ObjectInterface, open_method: Type[interfaces.plugins.FileHandlerInterface], + friendly: bool = False, ) -> interfaces.plugins.FileHandlerInterface: """Extracts the complete data for a process as a FileHandlerInterface @@ -90,9 +97,20 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset=peb.ImageBaseAddress, layer_name=proc_layer_name, ) - file_handle = open_method( - f"pid.{proc.UniqueProcessId}.{peb.ImageBaseAddress:#x}.dmp" + + process_name = proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", ) + if friendly: + file_handle = open_method( + f"{proc.UniqueProcessId}.{process_name}.{peb.ImageBaseAddress:#x}.dmp" + ) + else: + file_handle = open_method( + f"pid.{proc.UniqueProcessId}.{peb.ImageBaseAddress:#x}.dmp" + ) for offset, data in dos_header.reconstruct(): file_handle.seek(offset) file_handle.write(data) @@ -243,6 +261,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pe_table_name, proc, self.open, + self.config["friendly"], ) file_output = "Error outputting file" if file_handle: From b4c6b661f01fc3dde54362a4f55be4d89e4cc6e5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Sep 2023 21:11:36 +0100 Subject: [PATCH 411/526] Core: Include only volatility3 in distributions packages Fixes #951 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 936a12af2..cfcda3d5c 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setuptools.setup( include_package_data=True, exclude_package_data={"": ["development", "development.*"], "development": ["*"]}, packages=setuptools.find_namespace_packages( - exclude=["development", "development.*"] + include=["volatility3"] ), entry_points={ "console_scripts": [ From da203f7d6828fdeca1fd8f4d85361e65813804eb Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Sep 2023 21:33:32 +0100 Subject: [PATCH 412/526] Documentation: Improve library documentation Fixes #993. --- doc/source/conf.py | 2 +- doc/source/using-as-a-library.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 8b467ec1d..d601c1eee 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -27,7 +27,7 @@ def setup(app): source_dir = os.path.abspath(os.path.dirname(__file__)) sphinx.ext.apidoc.main( - argv=["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory] + ["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory] ) # Go through the volatility3.framework.plugins files and change them to volatility3.plugins diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index fb012f1ae..4acf35f98 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -54,6 +54,12 @@ also be included, which can be found in `volatility3.constants.PLUGINS_PATH`. volatility3.plugins.__path__ = + constants.PLUGINS_PATH failures = framework.import_files(volatility3.plugins, True) +.. note:: + + Volatility uses the `volatility3.plugins` namespace for all plugins (including those in `volatility3.framework.plugins`). + Please ensure you only use `volatility3.plugins` and only ever import plugins from this namespace. + This ensures the ability of users to override core plugins without needing write access to the framework directory. + Once the plugins have been imported, we can interrogate which plugins are available. The :py:func:`~volatility3.framework.list_plugins` call will return a dictionary of plugin names and the plugin classes. From 52207e09332f4322c33139aee63d9e104e0a05f6 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 4 Sep 2023 17:44:24 +0100 Subject: [PATCH 413/526] Add extra debug information for layer stacker to show file size of the physical file --- volatility3/framework/automagic/stacker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/automagic/stacker.py b/volatility3/framework/automagic/stacker.py index e611b5f06..d966d99fa 100644 --- a/volatility3/framework/automagic/stacker.py +++ b/volatility3/framework/automagic/stacker.py @@ -156,6 +156,9 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): self._cached = context.config.get(path, None), context.config.branch( path ) + vollog.debug( + f"physical_layer maximum_address: {physical_layer.maximum_address}" + ) vollog.debug(f"Stacked layers: {stacked_layers}") @classmethod From 455487dbb65644ac8c633fdcda6351bbc3011019 Mon Sep 17 00:00:00 2001 From: xabrouck Date: Tue, 5 Sep 2023 16:20:46 +0200 Subject: [PATCH 414/526] Making cred optional as it didn't exist in old 2.6 kernels --- volatility3/framework/symbols/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 5c42a436d..f96302684 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -28,7 +28,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("fs_struct", extensions.fs_struct) self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) - self.set_type_class("cred", extensions.cred) + self.optional_set_type_class("cred", extensions.cred) self.set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) From 05df365936a5965171632c7b0b0dbd1bee6c08a9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 5 Sep 2023 18:23:48 +0100 Subject: [PATCH 415/526] Core: Fix missing packages in setup.py Fixes #1002. --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index cfcda3d5c..44ece8484 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ with open("README.md", "r", encoding="utf-8") as fh: def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -20,6 +20,7 @@ def get_install_requires(): requirements.append(stripped_line) return requirements + setuptools.setup( name="volatility3", description="Memory forensics framework", @@ -39,9 +40,8 @@ setuptools.setup( python_requires=">=3.7.0", include_package_data=True, exclude_package_data={"": ["development", "development.*"], "development": ["*"]}, - packages=setuptools.find_namespace_packages( - include=["volatility3"] - ), + packages=setuptools.find_namespace_packages(where="volatility3"), + package_dir={"": "volatility3"}, entry_points={ "console_scripts": [ "vol = volatility3.cli:main", From 627e2fbab92b30c389be91688fed01f4620c30a0 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 5 Sep 2023 22:27:53 +0200 Subject: [PATCH 416/526] Adding install.yml workflow --- .github/workflows/install.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/install.yml diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml new file mode 100644 index 000000000..e7f9936b6 --- /dev/null +++ b/.github/workflows/install.yml @@ -0,0 +1,29 @@ +name: Test install Volatility3 +on: [push, pull_request] +jobs: + + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.7"] + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup python-pip + run: python -m pip install --upgrade pip + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Install volatility3 + run: pip install . + + - name: Run volatility3 + run: vol --help \ No newline at end of file From 2a67aa639a63b35b2203da07b64be4603d8d8ead Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 5 Sep 2023 22:29:25 +0200 Subject: [PATCH 417/526] Removing Python version --- .github/workflows/install.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index e7f9936b6..51768cc56 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -1,19 +1,14 @@ -name: Test install Volatility3 +name: Install Volatility3 test on: [push, pull_request] jobs: - build: + install: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.7"] steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - name: Setup python-pip run: python -m pip install --upgrade pip From 9982a44131f76987cbd55513a16d3582d27bb505 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 5 Sep 2023 22:36:40 +0200 Subject: [PATCH 418/526] Adding matrix strategy for hosts and python version --- .github/workflows/install.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 51768cc56..9ac14e6c6 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -2,13 +2,20 @@ name: Install Volatility3 test on: [push, pull_request] jobs: - install: - runs-on: ubuntu-latest + install_test: + runs-on: ${{ matrix.host }} + strategy: + matrix: + fail-fast: false + host: [ ubuntu-latest, macOS-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v3 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} - name: Setup python-pip run: python -m pip install --upgrade pip From ef341d54d6edb665b629799a76207f0f4a113f7b Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 5 Sep 2023 22:37:45 +0200 Subject: [PATCH 419/526] Fixing fail-fast strategy --- .github/workflows/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 9ac14e6c6..0a5fec25e 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -5,8 +5,8 @@ jobs: install_test: runs-on: ${{ matrix.host }} strategy: + fail-fast: false matrix: - fail-fast: false host: [ ubuntu-latest, macOS-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: From 9d2fd4051731ac696718cf40bb6e6543c6a1f40f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 5 Sep 2023 23:07:57 +0100 Subject: [PATCH 420/526] Revert "Core: Include only volatility3 in distributions packages" This reverts commit b4c6b661f01fc3dde54362a4f55be4d89e4cc6e5. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cfcda3d5c..936a12af2 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setuptools.setup( include_package_data=True, exclude_package_data={"": ["development", "development.*"], "development": ["*"]}, packages=setuptools.find_namespace_packages( - include=["volatility3"] + exclude=["development", "development.*"] ), entry_points={ "console_scripts": [ From 803c56e3c4c6495b2725b77cc7d045e39c98a9bd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 5 Sep 2023 23:51:17 +0100 Subject: [PATCH 421/526] Core: include the volatility3 package and all volatility3 subpackages --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 44ece8484..c2c55067d 100644 --- a/setup.py +++ b/setup.py @@ -37,11 +37,12 @@ setuptools.setup( "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, + packages=setuptools.find_namespace_packages( + include=["volatility3", "volatility3.*"] + ), + package_dir={"volatility3": "volatility3"}, python_requires=">=3.7.0", include_package_data=True, - exclude_package_data={"": ["development", "development.*"], "development": ["*"]}, - packages=setuptools.find_namespace_packages(where="volatility3"), - package_dir={"": "volatility3"}, entry_points={ "console_scripts": [ "vol = volatility3.cli:main", From 47ddf5d0e5142d6deeb071225ebb2d8bc366d381 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 6 Sep 2023 20:34:13 +0100 Subject: [PATCH 422/526] Core: Bump the version after 2.5.0 release branch --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index de1674885..c3ebaca27 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 370b6774ea3de8fb98bf625ed30b4347d68e2b54 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Sun, 17 Sep 2023 23:21:56 +0200 Subject: [PATCH 423/526] Remove macOS from matrix hosts --- .github/workflows/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 0a5fec25e..cc2a7fd3e 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - host: [ ubuntu-latest, macOS-latest, windows-latest ] + host: [ ubuntu-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v3 From 43bef641dc9b79f6ba3f5c0e345e9d76ffcd1318 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Sep 2023 20:37:38 +0100 Subject: [PATCH 424/526] Revert "Linux: Fix slight issue in envvars renaming" This reverts commit 60292c2da1efa886d0f46bd720af537a6069a623. --- volatility3/framework/plugins/linux/envars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 758943312..c4e3ed3c9 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -1,4 +1,4 @@ -from volatility3.plugins.linux import envvars +from volatility3.plugins import envvars import logging vollog = logging.getLogger(__name__) From 56cb5ea93ba74819ba4fe3a9bec39220d6d60d43 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Sep 2023 20:37:51 +0100 Subject: [PATCH 425/526] Revert "Linux: Rename linux.envars to linux.envvars" This reverts commit d9a365d96fcd990c7faba32ab7aa63523203e9f8. --- volatility3/framework/plugins/linux/envars.py | 121 +++++++++++++++++- .../framework/plugins/linux/envvars.py | 121 ------------------ 2 files changed, 115 insertions(+), 127 deletions(-) delete mode 100644 volatility3/framework/plugins/linux/envvars.py diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index c4e3ed3c9..5cbf0f502 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -1,12 +1,121 @@ -from volatility3.plugins import envvars +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import logging +from volatility3.framework import exceptions, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist + vollog = logging.getLogger(__name__) -class Envars(envvars.Envvars): - def run(self, *args, **kwargs): - vollog.warning( - "The linux.envars plugin has been renamed to linux.envvars and will only be accessible through the new name in a future release" +class Envars(plugins.PluginInterface): + """Lists processes with their environment variables""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + ] + + def _generator(self, tasks): + """Generates a listing of processes along with environment variables""" + + # walk the process list and return the envars + for task in tasks: + pid = task.pid + + # get process name as string + name = utility.array_to_string(task.comm) + + # try and get task parent + try: + ppid = task.parent.pid + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." + ) + ppid = 0 + + # kernel threads never have an mm as they do not have userland mappings + try: + mm = task.mm + except exceptions.InvalidAddressException: + # no mm so cannot get envars + vollog.debug( + f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." + ) + mm = None + continue + + # if mm exists attempt to get envars + if mm: + # get process layer to read envars from + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + vollog.debug( + f"Unable to construct process layer for task {pid} {name}, will not extract any envars." + ) + continue + proc_layer = self.context.layers[proc_layer_name] + + # get the size of the envars with sanity checking + envars_size = task.mm.env_end - task.mm.env_start + if not (0 < envars_size <= 8192): + vollog.debug( + f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." + ) + continue + + # attempt to read all envars data + try: + envar_data = proc_layer.read(task.mm.env_start, envars_size) + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." + ) + continue + + # parse envar data, envars are null terminated, keys and values are separated by '=' + envar_data = envar_data.rstrip(b"\x00") + for envar_pair in envar_data.split(b"\x00"): + try: + key, value = envar_pair.decode().split("=", 1) + except ValueError: + vollog.debug( + f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" + ) + continue + yield (0, (pid, ppid, name, key, value)) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + return renderers.TreeGrid( + [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), ) - return super().run(*args, **kwargs) diff --git a/volatility3/framework/plugins/linux/envvars.py b/volatility3/framework/plugins/linux/envvars.py deleted file mode 100644 index 1d6c8b784..000000000 --- a/volatility3/framework/plugins/linux/envvars.py +++ /dev/null @@ -1,121 +0,0 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -import logging - -from volatility3.framework import exceptions, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins -from volatility3.framework.objects import utility -from volatility3.plugins.linux import pslist - -vollog = logging.getLogger(__name__) - - -class Envvars(plugins.PluginInterface): - """Lists processes with their environment variables""" - - _required_framework_version = (2, 0, 0) - - @classmethod - def get_requirements(cls): - # Since we're calling the plugin, make sure we have the plugin's requirements - return [ - requirements.ModuleRequirement( - name="kernel", - description="Linux kernel", - architectures=["Intel32", "Intel64"], - ), - requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) - ), - requirements.ListRequirement( - name="pid", - description="Filter on specific process IDs", - element_type=int, - optional=True, - ), - ] - - def _generator(self, tasks): - """Generates a listing of processes along with environment variables""" - - # walk the process list and return the envars - for task in tasks: - pid = task.pid - - # get process name as string - name = utility.array_to_string(task.comm) - - # try and get task parent - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." - ) - ppid = 0 - - # kernel threads never have an mm as they do not have userland mappings - try: - mm = task.mm - except exceptions.InvalidAddressException: - # no mm so cannot get envars - vollog.debug( - f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." - ) - mm = None - continue - - # if mm exists attempt to get envars - if mm: - # get process layer to read envars from - proc_layer_name = task.add_process_layer() - if proc_layer_name is None: - vollog.debug( - f"Unable to construct process layer for task {pid} {name}, will not extract any envars." - ) - continue - proc_layer = self.context.layers[proc_layer_name] - - # get the size of the envars with sanity checking - envars_size = task.mm.env_end - task.mm.env_start - if not (0 < envars_size <= 8192): - vollog.debug( - f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." - ) - continue - - # attempt to read all envars data - try: - envar_data = proc_layer.read(task.mm.env_start, envars_size) - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." - ) - continue - - # parse envar data, envars are null terminated, keys and values are separated by '=' - envar_data = envar_data.rstrip(b"\x00") - for envar_pair in envar_data.split(b"\x00"): - try: - key, value = envar_pair.decode().split("=", 1) - except ValueError: - vollog.debug( - f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" - ) - continue - yield (0, (pid, ppid, name, key, value)) - - def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - - return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], - self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ), - ) From a46c9d9d8ecf0a36352c672c2193227c59cd33c1 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 29 Sep 2023 10:16:20 +0100 Subject: [PATCH 426/526] Linux: add padded read when getting magic for elf extension to help with smear and missing pages --- .../framework/symbols/linux/extensions/elf.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 416a7e4d2..a05885a7b 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -33,14 +33,18 @@ class elf(objects.StructType): layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() # We read the MAGIC: (0x0 to 0x4) 0x7f 0x45 0x4c 0x46 - magic = self._context.object( - symbol_table_name + constants.BANG + "unsigned long", - layer_name=layer_name, - offset=object_info.offset, - ) + magic = self._context.layers[layer_name].read(object_info.offset, 4, True) # Check validity - if magic != 0x464C457F: + if ( + magic[0] == 0x7F + and magic[1] == 0x45 # E + and magic[2] == 0x4C # L + and magic[3] == 0x46 # F + ): + self._valid_magic = True + else: + self._valid_magic = False return None # We need to read the EI_CLASS (0x4 offset) @@ -72,7 +76,10 @@ class elf(objects.StructType): """ Determine whether it is a valid object """ - return self._type_prefix is not None and self._hdr is not None + if self._valid_magic: + return self._type_prefix is not None and self._hdr is not None + else: + return False def __getattr__(self, name): # Just redirect to the corresponding header From 41a02fbf5b5bd860f35d53c7ed34a97bfa68f3cd Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 3 Oct 2023 07:13:28 +0100 Subject: [PATCH 427/526] Linux: use try/except in linux elf extension to catch paged and invalid addresses rather than crashing --- .../framework/symbols/linux/extensions/elf.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index a05885a7b..8b42b3075 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -3,9 +3,12 @@ # from typing import Dict, Tuple +import logging from volatility3.framework import constants -from volatility3.framework import objects, interfaces +from volatility3.framework import objects, interfaces, exceptions + +vollog = logging.getLogger(__name__) class elf(objects.StructType): @@ -33,20 +36,29 @@ class elf(objects.StructType): layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() # We read the MAGIC: (0x0 to 0x4) 0x7f 0x45 0x4c 0x46 - magic = self._context.layers[layer_name].read(object_info.offset, 4, True) - - # Check validity - if ( - magic[0] == 0x7F - and magic[1] == 0x45 # E - and magic[2] == 0x4C # L - and magic[3] == 0x46 # F - ): - self._valid_magic = True - else: + try: + magic = self._context.object( + symbol_table_name + constants.BANG + "unsigned long", + layer_name=layer_name, + offset=object_info.offset, + ) + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ) as excp: + vollog.debug( + f"Unable to check magic bytes for ELF file at offset {hex(object_info.offset)} in layer {layer_name}: {excp}" + ) self._valid_magic = False return None + # Check validity + if magic != 0x464C457F: # e.g. ELF + self._valid_magic = False + return None + else: + self._valid_magic = True + # We need to read the EI_CLASS (0x4 offset) ei_class = self._context.object( symbol_table_name + constants.BANG + "unsigned char", From 93b297283292ff7080f539cc483e48c28271dfe8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Oct 2023 22:19:17 +0100 Subject: [PATCH 428/526] Renderers: Allow nodes to be turned into dictionaries --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/renderers/__init__.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index c3ebaca27..09dded076 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 534686022..43bb59a21 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -10,7 +10,7 @@ import collections import collections.abc import datetime import logging -from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar, Union +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, TypeVar, Union from volatility3.framework import interfaces from volatility3.framework.interfaces import renderers @@ -96,6 +96,10 @@ class TreeNode(interfaces.renderers.TreeNode): # if isinstance(val, datetime.datetime): # tznaive = val.tzinfo is None or val.tzinfo.utcoffset(val) is None + def asdict(self) -> Dict[str, Any]: + """Returns the contents of the node as a dictionary""" + return self._values._asdict() + @property def values(self) -> List[interfaces.renderers.BaseTypes]: """Returns the list of values from the particular node, based on column From 310b6508db305b46288e5d0f530ca169ce020726 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 11 Oct 2023 09:09:15 +0100 Subject: [PATCH 429/526] vmware: Add warning when no metadata file is found for a vmem file. --- volatility3/framework/layers/vmware.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 0bc1a350b..5467c70c0 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -232,6 +232,9 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): ) if not vmss_success and not vmsn_success: + vollog.warning( + f"No metadata file alongside VMEM file! A VMSS or VMSN file is required to correctly process a VMEM file. These should be placed in the same directory with the same file name, e.g. sample.vmem and sample.vmsn.", + ) return None new_layer_name = context.layers.free_layer_name("VmwareLayer") context.config[ From 9cc73b6a0388321ee6de9f6c82a124e263282d78 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 12 Oct 2023 09:57:24 -0500 Subject: [PATCH 430/526] issue #1017 - check for valid root node type --- volatility3/framework/layers/registry.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc8ce1f4c..a2d3b0fd4 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -171,7 +171,12 @@ class RegistryHive(linear.LinearlyMappedLayer): node (default) or a list of nodes from root to the current node (if return_list is true). """ - node_key = [self.get_node(self.root_cell_offset)] + root_node = self.get_node(self.root_cell_offset) + if not root_node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"): + raise RegistryFormatException( + self.name, "Encountered {} instead of _CM_KEY_NODE".format(root_node.vol.type_name) + ) + node_key = [root_node] if key.endswith("\\"): key = key[:-1] key_array = key.split("\\") From 95c468c4f8aeaa00e658901ee1bd23054756d8a2 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 12 Oct 2023 10:16:56 -0500 Subject: [PATCH 431/526] issue #1019 - for subkeys, return the modified time of the subkey itself, not its parent key --- volatility3/framework/plugins/windows/registry/printkey.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 537bfc943..70a288b63 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -153,6 +153,9 @@ class PrintKey(interfaces.plugins.PluginInterface): vollog.debug(excp) key_node_name = renderers.UnreadableValue() + # if the item is a subkey, use the LastWriteTime of that subkey + last_write_time = conversion.wintime_to_datetime(node.LastWriteTime.QuadPart) + yield ( depth, ( From e45380f4f579e3c54f1475312efc2aff47341b2a Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 13 Oct 2023 09:29:46 +0100 Subject: [PATCH 432/526] Update vmware layer metadata file warning to be less doom and gloom. --- volatility3/framework/layers/vmware.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 5467c70c0..622ff0250 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -4,6 +4,7 @@ import contextlib import logging import struct +import os from typing import Any, Dict, List, Optional from volatility3.framework import constants, exceptions, interfaces @@ -232,8 +233,10 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): ) if not vmss_success and not vmsn_success: + vmem_file_basename = os.path.basename(location) + example_vmss_file_basename = os.path.basename(vmss) vollog.warning( - f"No metadata file alongside VMEM file! A VMSS or VMSN file is required to correctly process a VMEM file. These should be placed in the same directory with the same file name, e.g. sample.vmem and sample.vmsn.", + f"No metadata file found alongside VMEM file. A VMSS or VMSN file may be required to correctly process a VMEM file. These should be placed in the same directory with the same file name, e.g. {vmem_file_basename} and {example_vmss_file_basename}.", ) return None new_layer_name = context.layers.free_layer_name("VmwareLayer") From 63ace6099508664b51b671f95a84bd891736e0d3 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Oct 2023 15:54:10 +0100 Subject: [PATCH 433/526] Core: Add (optional) sanitization to the FileHandler class --- volatility3/framework/interfaces/plugins.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 0de109c5e..29395aadf 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -43,7 +43,7 @@ class FileHandlerInterface(io.RawIOBase): return self._preferred_filename @preferred_filename.setter - def preferred_filename(self, filename): + def preferred_filename(self, filename: str): """Sets the preferred filename""" if self.closed: raise IOError("FileHandler name cannot be changed once closed") @@ -57,6 +57,18 @@ class FileHandlerInterface(io.RawIOBase): def close(self): """Method that commits the file and fixes the final filename for use""" + @staticmethod + def sanitize_filename(filename: str) -> str: + """Sanititizes the filename to ensure only a specific whitelist of characters is allowed through""" + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]\{\}!$%^:#~?<>,|" + result = "" + for char in filename: + if char in allowed: + result += char + else: + result += "?" + return result + def __enter__(self): return self From 14fa5ad771a668b8f4f717674412803cf7229c50 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Oct 2023 16:27:22 +0100 Subject: [PATCH 434/526] Documentation: Add in a CITATION.cff file Initial addition of a CITATION file, no version information was included because keeping it up to date with the version in the repo automatically would be tricky and not doing so would lead it to becoming out of sync. Fixes #1013. --- CITATION.cff | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..c36c3b7d5 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,37 @@ +# This CITATION.cff file was generated with cffinit. +# Visit https://bit.ly/cffinit to generate yours today! + +cff-version: 1.2.0 +title: Volatility 3 +message: >- + If you reference this software, please feel free to cite + it using the information below. +type: software +authors: + - name: Volatility Foundation + country: US + website: 'https://www.volatilityfoundation.org/' +identifiers: + - type: url + value: 'https://github.com/volatilityfoundation/volatility3' + description: Volatility 3 source code respository +repository-code: 'https://github.com/volatilityfoundation/volatility3' +url: 'https://github.com/volatilityfoundation/volatility3' +abstract: >- + Volatility is the world's most widely used framework for + extracting digital artifacts from volatile memory (RAM) + samples. The extraction techniques are performed + completely independent of the system being investigated + but offer visibility into the runtime state of the system. + The framework is intended to introduce people to the + techniques and complexities associated with extracting + digital artifacts from volatile memory samples and provide + a platform for further work into this exciting area of + research. +keywords: + - malware + - forensics + - memory + - python + - ram + - volatility From 5d43071f572a4c2aa5cbe573cb5b400e8d27607f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Oct 2023 15:54:10 +0100 Subject: [PATCH 435/526] Core: Add (optional) sanitization to the FileHandler class --- volatility3/framework/interfaces/plugins.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 0de109c5e..29395aadf 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -43,7 +43,7 @@ class FileHandlerInterface(io.RawIOBase): return self._preferred_filename @preferred_filename.setter - def preferred_filename(self, filename): + def preferred_filename(self, filename: str): """Sets the preferred filename""" if self.closed: raise IOError("FileHandler name cannot be changed once closed") @@ -57,6 +57,18 @@ class FileHandlerInterface(io.RawIOBase): def close(self): """Method that commits the file and fixes the final filename for use""" + @staticmethod + def sanitize_filename(filename: str) -> str: + """Sanititizes the filename to ensure only a specific whitelist of characters is allowed through""" + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]\{\}!$%^:#~?<>,|" + result = "" + for char in filename: + if char in allowed: + result += char + else: + result += "?" + return result + def __enter__(self): return self From 7323bd3a591a4d989fde5837d51a0a9c2d9061f3 Mon Sep 17 00:00:00 2001 From: 616c696365 <616c696365@localhost.com> Date: Wed, 18 Oct 2023 19:14:02 +0100 Subject: [PATCH 436/526] windows.pslist process name added to dumped file by default --- .../framework/plugins/windows/pslist.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 806bb678e..e7a0d5dd4 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -50,12 +50,6 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): default=False, optional=True, ), - requirements.BooleanRequirement( - name="friendly", - description="Display process name in dump filename", - default=False, - optional=True, - ), ] @classmethod @@ -66,7 +60,6 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pe_table_name: str, proc: interfaces.objects.ObjectInterface, open_method: Type[interfaces.plugins.FileHandlerInterface], - friendly: bool = False, ) -> interfaces.plugins.FileHandlerInterface: """Extracts the complete data for a process as a FileHandlerInterface @@ -103,14 +96,13 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): max_length=proc.ImageFileName.vol.count, errors="replace", ) - if friendly: - file_handle = open_method( + + file_handle = open_method( + open_method.sanitize_filename( f"{proc.UniqueProcessId}.{process_name}.{peb.ImageBaseAddress:#x}.dmp" ) - else: - file_handle = open_method( - f"pid.{proc.UniqueProcessId}.{peb.ImageBaseAddress:#x}.dmp" - ) + ) + for offset, data in dos_header.reconstruct(): file_handle.seek(offset) file_handle.write(data) @@ -261,7 +253,6 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pe_table_name, proc, self.open, - self.config["friendly"], ) file_output = "Error outputting file" if file_handle: From fcaba2d95a79a7bec70cde265199ab892b6c0948 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Wed, 18 Oct 2023 14:08:50 -0500 Subject: [PATCH 437/526] issue #1017 - black updates --- volatility3/framework/layers/registry.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index a2d3b0fd4..9841d2bb0 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -174,7 +174,10 @@ class RegistryHive(linear.LinearlyMappedLayer): root_node = self.get_node(self.root_cell_offset) if not root_node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"): raise RegistryFormatException( - self.name, "Encountered {} instead of _CM_KEY_NODE".format(root_node.vol.type_name) + self.name, + "Encountered {} instead of _CM_KEY_NODE".format( + root_node.vol.type_name + ), ) node_key = [root_node] if key.endswith("\\"): From a497216bebd8e5a9d3ebe4fd039b673d90346ce7 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Wed, 18 Oct 2023 14:09:42 -0500 Subject: [PATCH 438/526] issue #1019 - black updates --- volatility3/framework/plugins/windows/registry/printkey.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 70a288b63..f66e55f4b 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -154,7 +154,9 @@ class PrintKey(interfaces.plugins.PluginInterface): key_node_name = renderers.UnreadableValue() # if the item is a subkey, use the LastWriteTime of that subkey - last_write_time = conversion.wintime_to_datetime(node.LastWriteTime.QuadPart) + last_write_time = conversion.wintime_to_datetime( + node.LastWriteTime.QuadPart + ) yield ( depth, From 2d3b3158dc953a89898cd3e403415e1725f4cbc2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 21 Oct 2023 23:34:17 +0200 Subject: [PATCH 439/526] correct sql query and strip identifier --- volatility3/framework/automagic/symbol_cache.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 29f2cfd08..2c605f915 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -429,8 +429,7 @@ class SqliteCache(CacheManagerInterface): progress_callback(0, "Reading remote ISF list") cursor = self._database.cursor() cursor.execute( - f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', {self.cache_period})" - ) + f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', '{self.cache_period}')" ) remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, "Reading remote ISF list") for operating_system in constants.OS_CATEGORIES: @@ -438,9 +437,11 @@ class SqliteCache(CacheManagerInterface): {}, operating_system=operating_system ) for identifier, location in identifiers: + identifier = identifier.rstrip() + identifier = identifier[:-1] if identifier.endswith(b"\x00") else identifier # Linux banners dumped by dwarf2json end with "\x00\n". If not stripped, the banner cannot match. cursor.execute( - "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - (location, identifier, operating_system, False), + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + (identifier, location, operating_system, False), ) progress_callback(100, "Reading remote ISF list") self._database.commit() From c20baf9d1fd346fe9f809227d063d4cf56717879 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 23 Oct 2023 17:49:03 +0200 Subject: [PATCH 440/526] black formatting --- volatility3/framework/automagic/symbol_cache.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 2c605f915..22f1c94f3 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -429,7 +429,8 @@ class SqliteCache(CacheManagerInterface): progress_callback(0, "Reading remote ISF list") cursor = self._database.cursor() cursor.execute( - f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', '{self.cache_period}')" ) + f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', '{self.cache_period}')" + ) remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, "Reading remote ISF list") for operating_system in constants.OS_CATEGORIES: @@ -438,9 +439,11 @@ class SqliteCache(CacheManagerInterface): ) for identifier, location in identifiers: identifier = identifier.rstrip() - identifier = identifier[:-1] if identifier.endswith(b"\x00") else identifier # Linux banners dumped by dwarf2json end with "\x00\n". If not stripped, the banner cannot match. + identifier = ( + identifier[:-1] if identifier.endswith(b"\x00") else identifier + ) # Linux banners dumped by dwarf2json end with "\x00\n". If not stripped, the banner cannot match. cursor.execute( - "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", (identifier, location, operating_system, False), ) progress_callback(100, "Reading remote ISF list") From 3bb1285dbfb3200567f50662ceed663283c16d02 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 Oct 2023 13:53:55 +0000 Subject: [PATCH 441/526] Layers: Make removal of a layer more efficient, as noted in #809 --- volatility3/framework/interfaces/layers.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 68592f8cb..e2a68780a 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -678,16 +678,12 @@ class LayerContainer(collections.abc.Mapping): name: The name of the layer to delete """ for layer in self._layers: - depend_list = [ - superlayer - for superlayer in self._layers - if name in self._layers[layer].dependencies - ] - if depend_list: + if name in self._layers[layer].dependencies: raise exceptions.LayerException( self._layers[layer].name, - f"Layer {self._layers[layer].name} is depended upon: {', '.join(depend_list)}", + f"Layer {self._layers[layer].name} is depended upon by {layer}", ) + # Otherwise, wipe out the layer self._layers[name].destroy() del self._layers[name] From b9c4a1d8a76bd40b7a0ee40ea91c0e32da99000e Mon Sep 17 00:00:00 2001 From: Leron Gray Date: Tue, 31 Oct 2023 13:52:26 -0500 Subject: [PATCH 442/526] fix guid and pdb_name for #1026 --- volatility3/framework/symbols/windows/pdbutil.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index a43933ccf..bdcf25fa1 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -9,6 +9,7 @@ import lzma import os import re import struct +from pathlib import PureWindowsPath from typing import Any, Dict, Generator, List, Optional, Tuple, Union from urllib import parse, request @@ -226,13 +227,12 @@ class PDBUtility(interfaces.configuration.VersionableInterface): return None pdb_name = debug_entry.PdbFileName.decode("utf-8").strip("\x00") + + # Let pathlib do the filename extraction. This will likely always be a Windows path though. + pdb_name = PureWindowsPath(pdb_name).name + age = debug_entry.Age - guid = "{:08x}{:04x}{:04x}{}".format( - debug_entry.Signature_Data1, - debug_entry.Signature_Data2, - debug_entry.Signature_Data3, - binascii.hexlify(debug_entry.Signature_Data4).decode("utf-8"), - ) + guid = debug_entry.Signature_String[:32] # Removes the Age from the GUID return guid, age, pdb_name @classmethod From c2b008969c44d9e3a0c8bfa8911ff6facbb53fea Mon Sep 17 00:00:00 2001 From: Leron Gray Date: Sun, 5 Nov 2023 16:25:43 -0600 Subject: [PATCH 443/526] update pefile requirements --- requirements-dev.txt | 2 +- requirements-minimal.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9db14d441..c9b615cd8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ # The following packages are required for core functionality. -pefile>=2017.8.1 +pefile>=2023.2.7 # The following packages are optional. # If certain packages are not necessary, place a comment (#) at the start of the line. diff --git a/requirements-minimal.txt b/requirements-minimal.txt index 31ac02814..c030b332d 100644 --- a/requirements-minimal.txt +++ b/requirements-minimal.txt @@ -1,2 +1,2 @@ # These packages are required for core functionality. -pefile>=2017.8.1 #foo \ No newline at end of file +pefile>=2023.2.7 #foo \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 99e0786cc..ddd1088c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # The following packages are required for core functionality. -pefile>=2017.8.1 +pefile>=2023.2.7 # The following packages are optional. # If certain packages are not necessary, place a comment (#) at the start of the line. From ebe19bf179068952a612ad94e29d08851b70b429 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 10 Nov 2023 20:30:38 +0000 Subject: [PATCH 444/526] Linux: update maple tree extension to fix issue #1032 --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3fb772135..c3e50fce4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -301,7 +301,11 @@ class maple_tree(objects.StructType): self.ma_flags & self.MT_FLAGS_HEIGHT_MASK ) >> self.MT_FLAGS_HEIGHT_OFFSET yield from self._parse_maple_tree_node( - self.ma_root, maple_tree_offset, expected_maple_tree_depth + self.ma_root, + maple_tree_offset, + expected_maple_tree_depth, + seen=set(), + current_depth=1, ) def _parse_maple_tree_node( From 9fe7d479f6e3afe6db9f9eeb3b723b93bc4f7f19 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 10 Nov 2023 20:58:07 +0000 Subject: [PATCH 445/526] Linux: update bash plugin to only get memory sections once --- volatility3/framework/plugins/linux/bash.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index b7dc2c16b..ce4567ca6 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -75,11 +75,16 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): bang_addrs = [] + # get task memory sections to be used by scanners + task_memory_sections = [ + section for section in task.get_process_memory_sections(heap_only=True) + ] + # find '#' values on the heap for address in proc_layer.scan( self.context, scanners.BytesScanner(b"#"), - sections=task.get_process_memory_sections(heap_only=True), + sections=task_memory_sections, ): bang_addrs.append(struct.pack(pack_format, address)) @@ -89,7 +94,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): for address, _ in proc_layer.scan( self.context, scanners.MultiStringScanner(bang_addrs), - sections=task.get_process_memory_sections(heap_only=True), + sections=task_memory_sections, ): hist = self.context.object( bash_table_name + constants.BANG + "hist_entry", From f7d5e722a2ef98370b45c3d3aaa811f13d5de480 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 13 Nov 2023 07:05:30 +0000 Subject: [PATCH 446/526] Linux: add member presence checks to linux.checks_afinfo _check_afinfo function to stop crashes when none of the required members are present --- .../framework/plugins/linux/check_afinfo.py | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index 90e714eaa..61c642f0e 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -51,10 +51,24 @@ class Check_afinfo(plugins.PluginInterface): yield check, addr def _check_afinfo(self, var_name, var, op_members, seq_members): - for hooked_member, hook_address in self._check_members( - var.seq_fops, var_name, op_members - ): - yield var_name, hooked_member, hook_address + # check if object has a least one of the members used for analysis by this function + required_members = ["seq_fops", "seq_ops", "seq_show"] + for member in required_members: + vollog.debug(f"{var_name}: {member} :{var.has_member(member)}") + has_required_member = any( + [var.has_member(member) for member in required_members] + ) + if not has_required_member: + vollog.warning( + f"This plugin requires the seq_fops, seq_ops, or seq_show members to be to check for hooks. These members are not present in the {var_name} object at {hex(var.vol.offset)}. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." + ) + return + + if var.has_member("seq_fops"): + for hooked_member, hook_address in self._check_members( + var.seq_fops, var_name, op_members + ): + yield var_name, hooked_member, hook_address # newer kernels if var.has_member("seq_ops"): @@ -64,8 +78,10 @@ class Check_afinfo(plugins.PluginInterface): yield var_name, hooked_member, hook_address # this is the most commonly hooked member by rootkits, so a force a check on it - elif not self._is_known_address(var.seq_show): - yield var_name, "show", var.seq_show + else: + if var.has_member("seq_show"): + if not self._is_known_address(var.seq_show): + yield var_name, "show", var.seq_show def _generator(self): vmlinux = self.context.modules[self.config["kernel"]] From 5ab0e4f83f7feb5444ce8d691e58bf56e495db1a Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 14 Nov 2023 07:22:00 +0000 Subject: [PATCH 447/526] Linux: remove _valid_magic from linux elf extension, check for _type_prefix and _hdr attrs instead --- volatility3/framework/symbols/linux/extensions/elf.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 8b42b3075..629a05da5 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -49,15 +49,11 @@ class elf(objects.StructType): vollog.debug( f"Unable to check magic bytes for ELF file at offset {hex(object_info.offset)} in layer {layer_name}: {excp}" ) - self._valid_magic = False return None # Check validity if magic != 0x464C457F: # e.g. ELF - self._valid_magic = False return None - else: - self._valid_magic = True # We need to read the EI_CLASS (0x4 offset) ei_class = self._context.object( @@ -88,7 +84,7 @@ class elf(objects.StructType): """ Determine whether it is a valid object """ - if self._valid_magic: + if hasattr(self, "_type_prefix") and hasattr(self, "_hdr"): return self._type_prefix is not None and self._hdr is not None else: return False From 83de6274eff15f6a3dc9d5c96eecbe3d871f6b48 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 14 Nov 2023 16:53:08 +0100 Subject: [PATCH 448/526] fix symbols and function calls --- .../symbols/linux/extensions/__init__.py | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3fb772135..bb515f7de 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -71,11 +71,11 @@ class module(generic.GenericIntelProcess): def _get_sect_count(self, grp): """Try to determine the number of valid sections""" arr = self._context.object( - self.get_symbol_table().name + constants.BANG + "array", + self.get_symbol_table_name() + constants.BANG + "array", layer_name=self.vol.layer_name, offset=grp.attrs, subtype=self._context.symbol_space.get_type( - self.get_symbol_table().name + constants.BANG + "pointer" + self.get_symbol_table_name() + constants.BANG + "pointer" ), count=25, ) @@ -92,11 +92,11 @@ class module(generic.GenericIntelProcess): else: num_sects = self._get_sect_count(self.sect_attrs.grp) arr = self._context.object( - self.get_symbol_table().name + constants.BANG + "array", + self.get_symbol_table_name() + constants.BANG + "array", layer_name=self.vol.layer_name, offset=self.sect_attrs.attrs.vol.offset, subtype=self._context.symbol_space.get_type( - self.get_symbol_table().name + constants.BANG + "module_sect_attr" + self.get_symbol_table_name() + constants.BANG + "module_sect_attr" ), count=num_sects, ) @@ -105,13 +105,14 @@ class module(generic.GenericIntelProcess): yield attr def get_symbols(self): - if symbols.symbol_table_is_64bit(self._context, self.get_symbol_table().name): + """Get module symbols""" + if symbols.symbol_table_is_64bit(self._context, self.get_symbol_table_name()): prefix = "Elf64_" else: prefix = "Elf32_" elf_table_name = intermed.IntermediateSymbolTable.create( - self.context, - self.config_path, + self._context, + self._context.modules["kernel"].config_path, "linux", "elf", native_types=None, @@ -119,7 +120,7 @@ class module(generic.GenericIntelProcess): ) syms = self._context.object( - self.get_symbol_table().name + constants.BANG + "array", + self.get_symbol_table_name() + constants.BANG + "array", layer_name=self.vol.layer_name, offset=self.section_symtab, subtype=self._context.symbol_space.get_type( @@ -127,18 +128,39 @@ class module(generic.GenericIntelProcess): ), count=self.num_symtab + 1, ) + if self.section_strtab: for sym in syms: - sym.set_cached_strtab(self.section_strtab) - yield sym + try: + sym_offset = self.section_strtab + sym.st_name + sym_name = self._context.layers[self.vol.layer_name].read( + sym_offset, sym.st_size + ) + except exceptions.PagedInvalidAddressException: + continue + + if sym_name: + # Normalize sym_value + mask = self._context.layers[self.vol.layer_name].address_mask + sym_value = sym.st_value & mask + # Stop at first null byte (strtab is a null terminated strings list) + sym_name = sym_name.split(b"\x00")[0].decode("latin-1") + yield (sym_name, sym_value, sym_offset) def get_symbol(self, wanted_sym_name): - """Get value for a given symbol name""" - for sym in self.get_symbols(): - sym_name = sym.get_name() - sym_addr = sym.st_value + """Get symbol value for a given symbol name""" + for sym_name, sym_value, sym_offset in self.get_symbols(): if wanted_sym_name == sym_name: - return sym_addr + return sym_value + + return None + + def get_symbol_name_from_value(self, wanted_sym_value): + """Get symbol name for a given symbol value""" + for sym_name, sym_value, sym_offset in self.get_symbols(): + if wanted_sym_value == sym_value: + return sym_name + return None @property From 2de97af061e3569e9c49a717730543b8dcf743c0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 14 Nov 2023 16:54:10 +0100 Subject: [PATCH 449/526] use refcnt for inheritance --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index bb515f7de..72e26f2fa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1154,7 +1154,7 @@ class vfsmount(objects.StructType): class kobject(objects.StructType): def reference_count(self): refcnt = self.kref.refcount - if self.has_member("counter"): + if refcnt.has_member("counter"): ret = refcnt.counter else: ret = refcnt.refs.counter From bf68fa8ce2d3997fe83b56e4056068047c94590a Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 14 Nov 2023 17:09:36 +0100 Subject: [PATCH 450/526] use KSYM_NAME_LEN for symbol name length --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 72e26f2fa..b0190e0f9 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -134,8 +134,8 @@ class module(generic.GenericIntelProcess): try: sym_offset = self.section_strtab + sym.st_name sym_name = self._context.layers[self.vol.layer_name].read( - sym_offset, sym.st_size - ) + sym_offset, 512 + ) # 512 is the value of KSYM_NAME_LEN except exceptions.PagedInvalidAddressException: continue From 36327a4315640b914cea0d300aa00fa28ff2096d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 14 Nov 2023 17:20:16 +0100 Subject: [PATCH 451/526] check if symbol name is empty after strip --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b0190e0f9..c37223d4d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -139,12 +139,12 @@ class module(generic.GenericIntelProcess): except exceptions.PagedInvalidAddressException: continue - if sym_name: + # Stop at first null byte (strtab is a null terminated strings list) + sym_name = sym_name.split(b"\x00")[0].decode("latin-1") + if sym_name != "": # Normalize sym_value mask = self._context.layers[self.vol.layer_name].address_mask sym_value = sym.st_value & mask - # Stop at first null byte (strtab is a null terminated strings list) - sym_name = sym_name.split(b"\x00")[0].decode("latin-1") yield (sym_name, sym_value, sym_offset) def get_symbol(self, wanted_sym_name): From 4f4e2efd991e1f54af650d082c6d63cff80c38fb Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 15 Nov 2023 10:29:06 +0000 Subject: [PATCH 452/526] Linux: update warning logic in linux.checks_afinfo so that only one warning is shown and debug logs are cleaner --- .../framework/plugins/linux/check_afinfo.py | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index 61c642f0e..7fced6acd 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -53,16 +53,14 @@ class Check_afinfo(plugins.PluginInterface): def _check_afinfo(self, var_name, var, op_members, seq_members): # check if object has a least one of the members used for analysis by this function required_members = ["seq_fops", "seq_ops", "seq_show"] - for member in required_members: - vollog.debug(f"{var_name}: {member} :{var.has_member(member)}") has_required_member = any( [var.has_member(member) for member in required_members] ) if not has_required_member: - vollog.warning( - f"This plugin requires the seq_fops, seq_ops, or seq_show members to be to check for hooks. These members are not present in the {var_name} object at {hex(var.vol.offset)}. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." + vollog.debug( + f"{var_name} object at {hex(var.vol.offset)} had none of the required members: {', '.join([member for member in required_members])}" ) - return + raise exceptions.PluginRequirementException if var.has_member("seq_fops"): for hooked_member, hook_address in self._check_members( @@ -101,6 +99,12 @@ class Check_afinfo(plugins.PluginInterface): ) protocols = [tcp, udp] + # used to track the calls to _check_afinfo and the + # number of errors produced due to missing members + symbols_checked = set() + symbols_with_errors = set() + + # loop through all symbols for struct_type, global_vars in protocols: for global_var_name in global_vars: # this will lookup fail for the IPv6 protocols on kernels without IPv6 support @@ -113,10 +117,20 @@ class Check_afinfo(plugins.PluginInterface): object_type=struct_type, offset=global_var.address ) - for name, member, address in self._check_afinfo( - global_var_name, global_var, op_members, seq_members - ): - yield 0, (name, member, format_hints.Hex(address)) + symbols_checked.add(global_var_name) + try: + for name, member, address in self._check_afinfo( + global_var_name, global_var, op_members, seq_members + ): + yield 0, (name, member, format_hints.Hex(address)) + except exceptions.PluginRequirementException: + symbols_with_errors.add(global_var_name) + + # if every call to _check_afinfo failed show a warning + if symbols_checked == symbols_with_errors: + vollog.warning( + "This plugin was not able to check for hooks. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." + ) def run(self): return renderers.TreeGrid( From 1a73717b9325ae332d04a77f9ecef58f89c9417b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 17 Nov 2023 17:49:54 +0100 Subject: [PATCH 453/526] set 'module' as config string --- .../framework/symbols/linux/extensions/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index c37223d4d..48296b284 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -105,14 +105,18 @@ class module(generic.GenericIntelProcess): yield attr def get_symbols(self): - """Get module symbols""" + """Get module symbols + + Yields: + A tuple for each symbol containing the symbol name and its corresponding value + """ if symbols.symbol_table_is_64bit(self._context, self.get_symbol_table_name()): prefix = "Elf64_" else: prefix = "Elf32_" elf_table_name = intermed.IntermediateSymbolTable.create( self._context, - self._context.modules["kernel"].config_path, + "module", "linux", "elf", native_types=None, From 6372b6f367bfb765e1a1f38f30866adb966fed33 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 17 Nov 2023 17:52:58 +0100 Subject: [PATCH 454/526] symbols getters follow vol2 format --- .../symbols/linux/extensions/__init__.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 48296b284..21edfc669 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -132,37 +132,37 @@ class module(generic.GenericIntelProcess): ), count=self.num_symtab + 1, ) - if self.section_strtab: for sym in syms: + sym_arr = self._context.object( + self.get_symbol_table_name() + constants.BANG + "array", + layer_name=self.vol.native_layer_name, + offset=self.section_strtab + sym.st_name, + ) try: - sym_offset = self.section_strtab + sym.st_name - sym_name = self._context.layers[self.vol.layer_name].read( - sym_offset, 512 - ) # 512 is the value of KSYM_NAME_LEN - except exceptions.PagedInvalidAddressException: + sym_name = utility.array_to_string( + sym_arr, 512 + ) # 512 is the value of KSYM_NAME_LEN kernel constant + except exceptions.InvalidAddressException: continue - - # Stop at first null byte (strtab is a null terminated strings list) - sym_name = sym_name.split(b"\x00")[0].decode("latin-1") if sym_name != "": - # Normalize sym_value + # Normalize sym.st_value offset, which is an address pointing to the symbol value mask = self._context.layers[self.vol.layer_name].address_mask - sym_value = sym.st_value & mask - yield (sym_name, sym_value, sym_offset) + sym_address = sym.st_value & mask + yield (sym_name, sym_address) def get_symbol(self, wanted_sym_name): """Get symbol value for a given symbol name""" - for sym_name, sym_value, sym_offset in self.get_symbols(): + for sym_name, sym_address in self.get_symbols(): if wanted_sym_name == sym_name: - return sym_value + return sym_address return None - def get_symbol_name_from_value(self, wanted_sym_value): - """Get symbol name for a given symbol value""" - for sym_name, sym_value, sym_offset in self.get_symbols(): - if wanted_sym_value == sym_value: + def get_symbol_from_address(self, wanted_sym_address): + """Get symbol name for a given symbol address""" + for sym_name, sym_address in self.get_symbols(): + if wanted_sym_address == sym_address: return sym_name return None From fa2b840b5552c8ad3b4242652acdde362f9e0052 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 18 Nov 2023 15:05:11 +0100 Subject: [PATCH 455/526] Object Storage Layer for PR #1037 --- volatility3/framework/layers/objectstorage.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 volatility3/framework/layers/objectstorage.py diff --git a/volatility3/framework/layers/objectstorage.py b/volatility3/framework/layers/objectstorage.py new file mode 100644 index 000000000..28f7a1bd4 --- /dev/null +++ b/volatility3/framework/layers/objectstorage.py @@ -0,0 +1,56 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import urllib.parse +from typing import Optional, Any, List + +try: + import s3fs + HAS_S3FS = True +except ImportError: + HAS_S3FS = False + +try: + import gcsfs + HAS_GCSFS = True +except ImportError: + HAS_GCSFS = False + +from volatility3.framework import exceptions +from volatility3.framework.layers import resources + +vollog = logging.getLogger(__file__) + +class S3FileSystemHandler(resources.VolatilityHandler): + if HAS_S3FS: + @classmethod + def non_cached_schemes(cls) -> List[str]: + return ["s3"] + + @staticmethod + def default_open(req: urllib.request.Request) -> Optional[Any]: + """Handles the request if it's the s3 scheme.""" + if req.type == "s3": + object_uri = "://".join(req.full_url.split("://")[1:]) + return s3fs.S3FileSystem().open(object_uri) + else: + raise exceptions.LayerException("s3 requirement is missing.") + + +class GSFileSystemHandler(resources.VolatilityHandler): + if HAS_GCSFS: + @classmethod + def non_cached_schemes(cls) -> List[str]: + return ["gs"] + + @staticmethod + def default_open(req: urllib.request.Request) -> Optional[Any]: + """Handles the request if it's the gs scheme.""" + if req.type == "gs": + object_uri = "://".join(req.full_url.split("://")[1:]) + return gcsfs.GCSFileSystem().open(object_uri) + return None + else: + raise exceptions.LayerException("gcsfs requirement is missing.") \ No newline at end of file From f8e34642195eef8954fd0bdf4e55d63e7e296d6a Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 18 Nov 2023 15:17:54 +0100 Subject: [PATCH 456/526] Forgot to return 'None' in the s3 scheme --- volatility3/framework/layers/objectstorage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/layers/objectstorage.py b/volatility3/framework/layers/objectstorage.py index 28f7a1bd4..ce6cdb327 100644 --- a/volatility3/framework/layers/objectstorage.py +++ b/volatility3/framework/layers/objectstorage.py @@ -35,6 +35,7 @@ class S3FileSystemHandler(resources.VolatilityHandler): if req.type == "s3": object_uri = "://".join(req.full_url.split("://")[1:]) return s3fs.S3FileSystem().open(object_uri) + return None else: raise exceptions.LayerException("s3 requirement is missing.") From ceac912014f13e4262443e831bb390b928c57fd3 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 18 Nov 2023 15:47:49 +0100 Subject: [PATCH 457/526] Adding requirements --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index 99e0786cc..7e1c28595 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,7 @@ pycryptodome # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 + +# This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage +gcsfs>=2023.6.0 +s3fs>=2023.6.0 \ No newline at end of file From 6c6d036cc0b2606406af418fbb7a15d6db325cfc Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 19 Nov 2023 00:54:56 +0100 Subject: [PATCH 458/526] Adding Alternate Data Stream Scanner --- .../framework/plugins/windows/mftscan.py | 153 +++++++++++++++++- .../framework/symbols/windows/mft.json | 16 +- 2 files changed, 167 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 87416d274..f5c3ebc95 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,7 +5,7 @@ import contextlib import datetime import logging -from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed @@ -31,6 +31,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) ), + ] def _generator(self): @@ -189,3 +190,153 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ], self._generator(), ) + + +class ADS(interfaces.plugins.PluginInterface): + + """Scans for Alternate Data Stream""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + ] + + def _generator(self): + layer = self.context.layers[self.config["primary"]] + + # Yara Rule to scan for MFT Header Signatures + rules = yarascan.YaraScan.process_yara_options( + {"yara_rules": "/FILE0|FILE\*|BAAD/"} + ) + + # Read in the Symbol File + symbol_table = intermed.IntermediateSymbolTable.create( + context=self.context, + config_path=self.config_path, + sub_path="windows", + filename="mft", + class_types={"MFT_ENTRY": mft.MFTEntry,"FILE_NAME_ENTRY": mft.MFTFileName}, + ) + + # get each of the individual Field Sets + mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + header_object = symbol_table + constants.BANG + "ATTR_HEADER" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Scan the layer for Raw MFT records and parse the fields + for offset, _rule_name, _name, _value in layer.scan( + context=self.context, scanner=yarascan.YaraScanner(rules=rules) + ): + with contextlib.suppress(exceptions.PagedInvalidAddressException): + mft_record = self.context.object( + mft_object, offset=offset, layer_name=layer.name + ) + # We will update this on each pass in the next loop and use it as the new offset. + attr_base_offset = mft_record.FirstAttrOffset + + attr_header = self.context.object( + header_object, + offset=offset + attr_base_offset, + layer_name=layer.name, + ) + + # There is no field that has a count of Attributes + # Keep Attempting to read attributes until we get an invalid attr_header.AttrType + file_name = "" + while attr_header.AttrType.is_valid_choice: + + # Offset past the headers to the attribute data + attr_data_offset = ( + offset + + attr_base_offset + + self.context.symbol_space.get_type( + attribute_object + ).relative_child_offset("Attr_Data") + ) + + if attr_header.AttrType.lookup() == "FILE_NAME": + attr_data = self.context.object( + fn_object, offset=attr_data_offset, layer_name=layer.name + ) + file_name = attr_data.get_full_name() + + + # DATA Attribute (can be ADS or not) + if attr_header.AttrType.lookup() == "DATA": + if not attr_header.NonResidentFlag: + # It is a resident file + if attr_header.NameLength > 0: + attr_name_offset = ( + offset + + attr_base_offset + + attr_header.NameOffset + ) + ads_name = self._context.layers[layer.name].read( + attr_name_offset, attr_header.NameLength*2 , pad=True + ).decode('utf-16') + attr_content_offset = ( + offset + + attr_base_offset + + attr_header.ContentOffset + ) + content = self._context.layers[layer.name].read( + attr_content_offset, attr_header.ContentLength , pad=True + ) + + + # Preparing for Disassembly + architecture = layer.metadata.get("architecture", None) + disasm = interfaces.renderers.Disassembly( + content, 0, architecture.lower() + ) + + yield 0, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + attr_header.AttrType.lookup(), + file_name, + ads_name, + format_hints.HexBytes(content), + disasm, + ) + + # If there's no advancement the loop will never end, so break it now + if attr_header.Length == 0: + break + + # Update the base offset to point to the next attribute + attr_base_offset += attr_header.Length + # Get the next attribute + attr_header = self.context.object( + header_object, + offset=offset + attr_base_offset, + layer_name=layer.name, + ) + + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Record Type", str), + ("Record Number", int), + ("MFT Type", str), + ("Filename", str), + ("ADS Filename", str), + ("Hexdump", format_hints.HexBytes), + ("Disasm", interfaces.renderers.Disassembly), + ], + self._generator(), + ) \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index e5de8f3fa..616e8990d 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -300,10 +300,24 @@ "kind": "base", "name": "unsigned short" } + }, + "ContentLength": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ContentOffset": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned short" + } } }, "kind": "struct", - "size": 16 + "size": 22 },"RESIDENT_HEADER": { "fields": { "AttrSize": { From 46a6ab1721e241939af0b4c29bafdf2fcb09c405 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 19 Nov 2023 13:09:47 +0100 Subject: [PATCH 459/526] Making sure it is ADS --- .../framework/plugins/windows/mftscan.py | 77 ++++++++++--------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index f5c3ebc95..b4af2a867 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -254,8 +254,8 @@ class ADS(interfaces.plugins.PluginInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType file_name = "" + is_ads = 0 while attr_header.AttrType.is_valid_choice: - # Offset past the headers to the attribute data attr_data_offset = ( offset @@ -274,43 +274,48 @@ class ADS(interfaces.plugins.PluginInterface): # DATA Attribute (can be ADS or not) if attr_header.AttrType.lookup() == "DATA": - if not attr_header.NonResidentFlag: - # It is a resident file - if attr_header.NameLength > 0: - attr_name_offset = ( - offset - + attr_base_offset - + attr_header.NameOffset - ) - ads_name = self._context.layers[layer.name].read( - attr_name_offset, attr_header.NameLength*2 , pad=True - ).decode('utf-16') - attr_content_offset = ( - offset - + attr_base_offset - + attr_header.ContentOffset - ) - content = self._context.layers[layer.name].read( - attr_content_offset, attr_header.ContentLength , pad=True - ) + if is_ads > 0: + if not attr_header.NonResidentFlag: + # Resident files are the most interesting. + if attr_header.NameLength > 0: + attr_name_offset = ( + offset + + attr_base_offset + + attr_header.NameOffset + ) + ads_name = self._context.layers[layer.name].read( + attr_name_offset, attr_header.NameLength*2 , pad=True + ).decode('utf-16') + attr_content_offset = ( + offset + + attr_base_offset + + attr_header.ContentOffset + ) + content = self._context.layers[layer.name].read( + attr_content_offset, attr_header.ContentLength , pad=True + ) - - # Preparing for Disassembly - architecture = layer.metadata.get("architecture", None) - disasm = interfaces.renderers.Disassembly( - content, 0, architecture.lower() - ) + + # Preparing for Disassembly + architecture = layer.metadata.get("architecture", None) + disasm = interfaces.renderers.Disassembly( + content, 0, architecture.lower() + ) - yield 0, ( - format_hints.Hex(attr_data_offset), - mft_record.get_signature(), - mft_record.RecordNumber, - attr_header.AttrType.lookup(), - file_name, - ads_name, - format_hints.HexBytes(content), - disasm, - ) + yield 0, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + attr_header.AttrType.lookup(), + file_name, + ads_name, + format_hints.HexBytes(content), + disasm, + ) + else: + # The First Data Attr is the file itself not the ADS + is_ads+= 1 + # If there's no advancement the loop will never end, so break it now if attr_header.Length == 0: From fd5c8289397bfcbff816fb9ce18db987d5e5470e Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 20 Nov 2023 11:02:31 +0100 Subject: [PATCH 460/526] Renaming file, handling url parsing using urllib, changing logger and requirement condition placement. --- .../framework/layers/{objectstorage.py => cloudstorage.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename volatility3/framework/layers/{objectstorage.py => cloudstorage.py} (100%) diff --git a/volatility3/framework/layers/objectstorage.py b/volatility3/framework/layers/cloudstorage.py similarity index 100% rename from volatility3/framework/layers/objectstorage.py rename to volatility3/framework/layers/cloudstorage.py From 80483722ebc1a149f17d887648ef628abf71ed40 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 20 Nov 2023 11:57:56 +0100 Subject: [PATCH 461/526] split module functions to keep current API --- .../symbols/linux/extensions/__init__.py | 77 +++++++++++-------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 21edfc669..f5c71eb0e 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -104,64 +104,79 @@ class module(generic.GenericIntelProcess): for attr in arr: yield attr - def get_symbols(self): - """Get module symbols - - Yields: - A tuple for each symbol containing the symbol name and its corresponding value - """ - if symbols.symbol_table_is_64bit(self._context, self.get_symbol_table_name()): - prefix = "Elf64_" - else: - prefix = "Elf32_" + def get_elf_table_name(self): elf_table_name = intermed.IntermediateSymbolTable.create( self._context, - "module", + "config_name_elf_symbol_table", "linux", "elf", native_types=None, class_types=elf.class_types, ) + return elf_table_name + def get_symbols(self): + """Get symbols of the module + + Yields: + A symbol object + """ + + if not hasattr(self, "_elf_table_name"): + self._elf_table_name = self.get_elf_table_name() + if symbols.symbol_table_is_64bit(self._context, self.get_symbol_table_name()): + prefix = "Elf64_" + else: + prefix = "Elf32_" syms = self._context.object( self.get_symbol_table_name() + constants.BANG + "array", layer_name=self.vol.layer_name, offset=self.section_symtab, subtype=self._context.symbol_space.get_type( - elf_table_name + constants.BANG + prefix + "Sym" + self._elf_table_name + constants.BANG + prefix + "Sym" ), count=self.num_symtab + 1, ) if self.section_strtab: for sym in syms: - sym_arr = self._context.object( - self.get_symbol_table_name() + constants.BANG + "array", - layer_name=self.vol.native_layer_name, - offset=self.section_strtab + sym.st_name, - ) - try: - sym_name = utility.array_to_string( - sym_arr, 512 - ) # 512 is the value of KSYM_NAME_LEN kernel constant - except exceptions.InvalidAddressException: - continue - if sym_name != "": - # Normalize sym.st_value offset, which is an address pointing to the symbol value - mask = self._context.layers[self.vol.layer_name].address_mask - sym_address = sym.st_value & mask - yield (sym_name, sym_address) + yield sym + + def get_symbols_names_and_addresses(self): + """Get names and addresses for each symbol of the module + + Yields: + A tuple for each symbol containing the symbol name and its corresponding value + """ + + for sym in self.get_symbols(): + sym_arr = self._context.object( + self.get_symbol_table_name() + constants.BANG + "array", + layer_name=self.vol.native_layer_name, + offset=self.section_strtab + sym.st_name, + ) + try: + sym_name = utility.array_to_string( + sym_arr, 512 + ) # 512 is the value of KSYM_NAME_LEN kernel constant + except exceptions.InvalidAddressException: + continue + if sym_name != "": + # Normalize sym.st_value offset, which is an address pointing to the symbol value + mask = self._context.layers[self.vol.layer_name].address_mask + sym_address = sym.st_value & mask + yield (sym_name, sym_address) def get_symbol(self, wanted_sym_name): """Get symbol value for a given symbol name""" - for sym_name, sym_address in self.get_symbols(): + for sym_name, sym_address in self.get_symbols_names_and_addresses(): if wanted_sym_name == sym_name: return sym_address return None - def get_symbol_from_address(self, wanted_sym_address): + def get_symbol_by_address(self, wanted_sym_address): """Get symbol name for a given symbol address""" - for sym_name, sym_address in self.get_symbols(): + for sym_name, sym_address in self.get_symbols_names_and_addresses(): if wanted_sym_address == sym_address: return sym_name From c3dbf9714d9158b01385f78fb7434ecee64ac5b8 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 20 Nov 2023 19:12:07 +0100 Subject: [PATCH 462/526] Better variable init --- volatility3/framework/plugins/windows/mftscan.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index b4af2a867..c40f7ef73 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -59,7 +59,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields - for offset, _rule_name, _name, _value in layer.scan( + for offset, _, _, _ in layer.scan( context=self.context, scanner=yarascan.YaraScanner(rules=rules) ): with contextlib.suppress(exceptions.PagedInvalidAddressException): @@ -253,8 +253,9 @@ class ADS(interfaces.plugins.PluginInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - file_name = "" - is_ads = 0 + file_name = "N/A" + is_ads = False + # The First $DATA Attr is the 'principal' file itself not the ADS while attr_header.AttrType.is_valid_choice: # Offset past the headers to the attribute data attr_data_offset = ( @@ -274,7 +275,7 @@ class ADS(interfaces.plugins.PluginInterface): # DATA Attribute (can be ADS or not) if attr_header.AttrType.lookup() == "DATA": - if is_ads > 0: + if is_ads: if not attr_header.NonResidentFlag: # Resident files are the most interesting. if attr_header.NameLength > 0: @@ -313,8 +314,7 @@ class ADS(interfaces.plugins.PluginInterface): disasm, ) else: - # The First Data Attr is the file itself not the ADS - is_ads+= 1 + is_ads = True # If there's no advancement the loop will never end, so break it now From 24609856de629ccb7adbab46977de3c8492846e8 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 25 Nov 2023 17:04:47 +0100 Subject: [PATCH 463/526] Fixing: unused import, typo, ISF enhancement --- .../framework/plugins/windows/mftscan.py | 24 +++++++++---------- .../framework/symbols/windows/mft.json | 8 +++---- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index c40f7ef73..397ce75bc 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,7 +5,7 @@ import contextlib import datetime import logging -from volatility3.framework import constants, exceptions, interfaces, renderers, symbols +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed @@ -53,13 +53,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields - for offset, _, _, _ in layer.scan( + for offset, _rule_name, _name, _value in layer.scan( context=self.context, scanner=yarascan.YaraScanner(rules=rules) ): with contextlib.suppress(exceptions.PagedInvalidAddressException): @@ -86,8 +85,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset + attr_base_offset + self.context.symbol_space.get_type( - attribute_object - ).relative_child_offset("Attr_Data") + header_object + ).size ) # MFT Flags determine the file type or dir @@ -231,7 +230,6 @@ class ADS(interfaces.plugins.PluginInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" header_object = symbol_table + constants.BANG + "ATTR_HEADER" - attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields @@ -253,7 +251,7 @@ class ADS(interfaces.plugins.PluginInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - file_name = "N/A" + file_name = renderers.NotAvailableValue is_ads = False # The First $DATA Attr is the 'principal' file itself not the ADS while attr_header.AttrType.is_valid_choice: @@ -262,8 +260,8 @@ class ADS(interfaces.plugins.PluginInterface): offset + attr_base_offset + self.context.symbol_space.get_type( - attribute_object - ).relative_child_offset("Attr_Data") + header_object + ).size ) if attr_header.AttrType.lookup() == "FILE_NAME": @@ -272,7 +270,6 @@ class ADS(interfaces.plugins.PluginInterface): ) file_name = attr_data.get_full_name() - # DATA Attribute (can be ADS or not) if attr_header.AttrType.lookup() == "DATA": if is_ads: @@ -284,19 +281,21 @@ class ADS(interfaces.plugins.PluginInterface): + attr_base_offset + attr_header.NameOffset ) + ads_name = self._context.layers[layer.name].read( attr_name_offset, attr_header.NameLength*2 , pad=True ).decode('utf-16') + attr_content_offset = ( offset + attr_base_offset + attr_header.ContentOffset - ) + ) + content = self._context.layers[layer.name].read( attr_content_offset, attr_header.ContentLength , pad=True ) - # Preparing for Disassembly architecture = layer.metadata.get("architecture", None) disasm = interfaces.renderers.Disassembly( @@ -330,7 +329,6 @@ class ADS(interfaces.plugins.PluginInterface): layer_name=layer.name, ) - def run(self): return renderers.TreeGrid( [ diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index 616e8990d..d4f2aef7a 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -230,21 +230,21 @@ "offset": 0, "type": { "kind": "struct", - "name": "mft!ATTR_HEADER" + "name": "ATTR_HEADER" } }, "Resident_Header": { "offset": 16, "type": { "kind": "struct", - "name": "mft!RESIDENT_HEADER" + "name": "RESIDENT_HEADER" } }, "Attr_Data": { "offset": 24, "type": { "kind": "struct", - "name": "mft!ATTR_HEADER" + "name": "ATTR_HEADER" } } }, @@ -317,7 +317,7 @@ } }, "kind": "struct", - "size": 22 + "size": 24 },"RESIDENT_HEADER": { "fields": { "AttrSize": { From 62506aece60740787374fbc6f141dc3c33a34027 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 26 Nov 2023 11:23:09 +0000 Subject: [PATCH 464/526] Core: Fix up github security issues This fixes an unused import, an improper use of self and lots and lots of places where we implicitly return None. This now explicitly returns None to improve readability and prevent mixed implicit and explicit return values. This should also somewhat aid type checking by humans. --- volatility3/cli/__init__.py | 4 +-- volatility3/cli/volshell/generic.py | 6 ++-- volatility3/cli/volshell/linux.py | 4 +-- volatility3/cli/volshell/mac.py | 4 +-- volatility3/cli/volshell/windows.py | 2 +- volatility3/framework/automagic/module.py | 6 ++-- volatility3/framework/automagic/stacker.py | 2 +- .../framework/automagic/symbol_finder.py | 4 +-- volatility3/framework/layers/intel.py | 4 +-- volatility3/framework/layers/msf.py | 2 +- volatility3/framework/layers/segmented.py | 4 +-- .../framework/plugins/linux/capabilities.py | 2 +- .../framework/plugins/linux/check_syscall.py | 2 +- .../framework/plugins/linux/malfind.py | 2 +- .../framework/plugins/linux/sockstat.py | 12 ++++---- .../framework/plugins/mac/check_sysctl.py | 2 +- volatility3/framework/plugins/mac/kevents.py | 4 +-- volatility3/framework/plugins/mac/lsmod.py | 2 +- volatility3/framework/plugins/mac/malfind.py | 2 +- .../framework/plugins/windows/cachedump.py | 10 +++---- .../framework/plugins/windows/callbacks.py | 10 +++---- .../framework/plugins/windows/dumpfiles.py | 2 +- .../framework/plugins/windows/handles.py | 4 +-- .../framework/plugins/windows/lsadump.py | 6 ++-- .../framework/plugins/windows/malfind.py | 2 +- .../framework/plugins/windows/netstat.py | 4 +-- .../framework/plugins/windows/pstree.py | 4 +-- .../plugins/windows/registry/hivelist.py | 2 +- .../plugins/windows/registry/printkey.py | 2 +- .../plugins/windows/registry/userassist.py | 4 +-- .../plugins/windows/skeleton_key_check.py | 8 +++--- .../framework/renderers/format_hints.py | 3 +- .../framework/symbols/linux/__init__.py | 8 +++--- .../symbols/linux/extensions/__init__.py | 28 +++++++++---------- .../framework/symbols/linux/extensions/elf.py | 2 +- volatility3/framework/symbols/mac/__init__.py | 4 +-- .../symbols/mac/extensions/__init__.py | 10 +++---- .../symbols/windows/extensions/__init__.py | 22 +++++++-------- .../symbols/windows/extensions/registry.py | 4 +-- .../symbols/windows/extensions/services.py | 2 +- .../framework/symbols/windows/pdbutil.py | 1 - 41 files changed, 106 insertions(+), 106 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 9bfd14c6c..91bda7c66 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -662,7 +662,7 @@ class CommandLine: def close(self): # Don't overcommit if self.closed: - return + return None self.seek(0) @@ -712,7 +712,7 @@ class CommandLine: """Closes and commits the file (by moving the temporary file to the correct name""" # Don't overcommit if self._file.closed: - return + return None self._file.close() output_filename = self._get_final_filename() diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index ea9e65d9b..b95129d19 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -108,7 +108,7 @@ class Volshell(interfaces.plugins.PluginInterface): """Describes the available commands""" if args: help(*args) - return + return None variables = [] print("\nMethods:") @@ -325,7 +325,7 @@ class Volshell(interfaces.plugins.PluginInterface): (str, interfaces.objects.ObjectInterface, interfaces.objects.Template), ): print("Cannot display information about non-type object") - return + return None if not isinstance(object, str): # Mypy requires us to order things this way @@ -453,7 +453,7 @@ class Volshell(interfaces.plugins.PluginInterface): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: print("No symbol table provided") - return + return None longest_offset = longest_name = 0 table = self.context.symbol_space[symbol_table] diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 8c23bbec3..c5e555ec7 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -35,9 +35,9 @@ class Volshell(generic.Volshell): process_layer = task.add_process_layer() if process_layer is not None: self.change_layer(process_layer) - return + return None print(f"Layer for task ID {pid} could not be constructed") - return + return None print(f"No task with task ID {pid} found") def list_tasks(self): diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index b709511b1..2b32ad677 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -35,9 +35,9 @@ class Volshell(generic.Volshell): process_layer = task.add_process_layer() if process_layer is not None: self.change_layer(process_layer) - return + return None print(f"Layer for task ID {pid} could not be constructed") - return + return None print(f"No task with task ID {pid} found") def list_tasks(self, method=None): diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 652b2e66b..5c2190c02 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -32,7 +32,7 @@ class Volshell(generic.Volshell): if process.UniqueProcessId == pid: process_layer = process.add_process_layer() self.change_layer(process_layer) - return + return None print(f"No process with process ID {pid} found") def list_processes(self): diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index 2bdaf3f62..ee56a040c 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -29,9 +29,9 @@ class KernelModule(interfaces.automagic.AutomagicInterface): requirement.requirements[req], progress_callback, ) - return + return None if not requirement.unsatisfied(context, config_path): - return + return None # The requirement is unfulfilled and is a ModuleRequirement context.config[ @@ -43,7 +43,7 @@ class KernelModule(interfaces.automagic.AutomagicInterface): requirement.requirements[req].unsatisfied(context, new_config_path) and req != "offset" ): - return + return None # We now just have the offset requirement, but the layer requirement has been fulfilled. # Unfortunately we don't know the layer name requirement's exact name diff --git a/volatility3/framework/automagic/stacker.py b/volatility3/framework/automagic/stacker.py index d966d99fa..c251d3c46 100644 --- a/volatility3/framework/automagic/stacker.py +++ b/volatility3/framework/automagic/stacker.py @@ -103,7 +103,7 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): appropriate_config_path, layer_name = result context.config.merge(appropriate_config_path, subconfig) context.config[appropriate_config_path] = top_layer_name - return + return None self._cached = None new_context = context.clone() diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index f30dff456..bf1c8ff16 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -69,7 +69,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): # Bomb out early if our details haven't been configured if self.symbol_class is None: - return + return None self._requirements = self.find_requirements( context, @@ -120,7 +120,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): # Bomb out early if there's no banners if not self.banners: - return + return None mss = scanners.MultiStringScanner([x for x in self.banners if x is not None]) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 046203fa6..7d3b86a12 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -331,9 +331,9 @@ class Intel(linear.LinearlyMappedLayer): except exceptions.InvalidAddressException: if not ignore_errors: raise - return + return None yield offset, length, mapped_offset, length, layer_name - return + return None while length > 0: try: chunk_offset, page_size, layer_name = self._translate(offset) diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 76c645e92..8d84a774b 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -47,7 +47,7 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer): def read_streams(self): # Shortcut in case they've already been read if self._streams: - return + return None # Recover the root table, by recovering the root table index table... module = self.context.module(self.pdb_symbol_table, self._base_layer, offset=0) diff --git a/volatility3/framework/layers/segmented.py b/volatility3/framework/layers/segmented.py index beb667436..0d29d8bff 100644 --- a/volatility3/framework/layers/segmented.py +++ b/volatility3/framework/layers/segmented.py @@ -126,9 +126,9 @@ class NonLinearlySegmentedLayer( current_offset = logical_offset # If it starts too late then we're done if logical_offset > offset + length: - return + return None except exceptions.InvalidAddressException: - return + return None # Crop it to the amount we need left chunk_size = min(size, length + offset - logical_offset) yield logical_offset, chunk_size, mapped_offset, mapped_size, self._base_layer diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 518f52603..bfdb69aba 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -88,7 +88,7 @@ class Capabilities(plugins.PluginInterface): kernel_cap_last_cap = vmlinux.object_from_symbol(symbol_name="cap_last_cap") except exceptions.SymbolError: # It should be a kernel < 3.2 - return + return None vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() if kernel_cap_last_cap > vol2_last_cap: diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index b1d2919f9..b6634d612 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -145,7 +145,7 @@ class Check_syscall(plugins.PluginInterface): table_info = self._get_table_info(vmlinux, "sys_call_table", ptr_sz) except exceptions.SymbolError: vollog.error("Unable to find the system call table. Exiting.") - return + return None tables = [(table_name, table_info)] diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 8a21afc03..cf06ee0cc 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -44,7 +44,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer_name = task.add_process_layer() if not proc_layer_name: - return + return None proc_layer = self.context.layers[proc_layer_name] diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index fa67122ba..e9c98a227 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -147,7 +147,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): socket_filter["bpf_filter_type"] = "cBPF" if not sock_filter.has_member("prog") or not sock_filter.prog: - return + return None bpfprog = sock_filter.prog @@ -158,13 +158,13 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return # cBPF filter except AttributeError: # kernel < 3.18.140, it's a cBPF filter - return + return None BPF_PROG_TYPE_SOCKET_FILTER = 1 # eBPF filter if bpfprog_type != BPF_PROG_TYPE_SOCKET_FILTER: socket_filter["bpf_filter_type"] = f"UNK({bpfprog_type})" vollog.warning(f"Unexpected BPF type {bpfprog_type} for a socket") - return + return None socket_filter["bpf_filter_type"] = "eBPF" if not bpfprog.has_member("aux") or not bpfprog.aux: @@ -329,17 +329,17 @@ class SockHandlers(interfaces.configuration.VersionableInterface): xdp_sock = sock.cast("xdp_sock") device = xdp_sock.dev if not device: - return + return None src_addr = utility.array_to_string(device.name) src_port = dst_addr = dst_port = None bpfprog = device.xdp_prog if not bpfprog: - return + return None if not bpfprog.has_member("aux") or not bpfprog.aux: - return + return None bpfprog_aux = bpfprog.aux if bpfprog_aux.has_member("id"): diff --git a/volatility3/framework/plugins/mac/check_sysctl.py b/volatility3/framework/plugins/mac/check_sysctl.py index 165aad436..4f64eaed8 100644 --- a/volatility3/framework/plugins/mac/check_sysctl.py +++ b/volatility3/framework/plugins/mac/check_sysctl.py @@ -69,7 +69,7 @@ class Check_sysctl(plugins.PluginInterface): try: sysctl = sysctl.oid_link.sle_next.dereference() except exceptions.InvalidAddressException: - return + return None while sysctl: try: diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 3b996bc0a..2a8692b77 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -116,7 +116,7 @@ class Kevents(interfaces.plugins.PluginInterface): ) except exceptions.InvalidAddressException: - return + return None for klist in klist_array: for kn in mac.MacUtilities.walk_slist(klist, "kn_link"): @@ -140,7 +140,7 @@ class Kevents(interfaces.plugins.PluginInterface): try: p_klist = task.p_klist except exceptions.InvalidAddressException: - return + return None for kn in mac.MacUtilities.walk_slist(p_klist, "kn_link"): yield kn diff --git a/volatility3/framework/plugins/mac/lsmod.py b/volatility3/framework/plugins/mac/lsmod.py index 2979e374b..c6f57f889 100644 --- a/volatility3/framework/plugins/mac/lsmod.py +++ b/volatility3/framework/plugins/mac/lsmod.py @@ -75,7 +75,7 @@ class Lsmod(plugins.PluginInterface): try: kmod = kmod.next except exceptions.InvalidAddressException: - return + return None return # Generation finished def _generator(self): diff --git a/volatility3/framework/plugins/mac/malfind.py b/volatility3/framework/plugins/mac/malfind.py index 98b282e24..3094ada85 100644 --- a/volatility3/framework/plugins/mac/malfind.py +++ b/volatility3/framework/plugins/mac/malfind.py @@ -40,7 +40,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer_name = task.add_process_layer() if proc_layer_name is None: - return + return None proc_layer = self.context.layers[proc_layer_name] diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index a9b669add..6e667984a 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -108,12 +108,12 @@ class Cachedump(interfaces.plugins.PluginInterface): vollog.warning("Unable to locate SYSTEM hive") if sechive is None: vollog.warning("Unable to locate SECURITY hive") - return + return None bootkey = hashdump.Hashdump.get_bootkey(syshive) if not bootkey: vollog.warning("Unable to find bootkey") - return + return None kernel = self.context.modules[self.config["kernel"]] @@ -124,17 +124,17 @@ class Cachedump(interfaces.plugins.PluginInterface): lsakey = lsadump.Lsadump.get_lsa_key(sechive, bootkey, vista_or_later) if not lsakey: vollog.warning("Unable to find lsa key") - return + return None nlkm = self.get_nlkm(sechive, lsakey, vista_or_later) if not nlkm: vollog.warning("Unable to find nlkma key") - return + return None cache = hashdump.Hashdump.get_hive_key(sechive, "Cache") if not cache: vollog.warning("Unable to find cache key") - return + return None for cache_item in cache.get_values(): if cache_item.Name == "NL$Control": diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 48b2e7c62..fcc333b9f 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -157,7 +157,7 @@ class Callbacks(interfaces.plugins.PluginInterface): ) if callback_count == 0: - return + return None fast_refs = ntkrnlmp.object( object_type="array", @@ -199,7 +199,7 @@ class Callbacks(interfaces.plugins.PluginInterface): ) if callback_count == 0: - return + return None callback_list = ntkrnlmp.object(object_type="_LIST_ENTRY", offset=symbol_offset) for callback in callback_list.to_list(full_type_name, "Link"): @@ -256,7 +256,7 @@ class Callbacks(interfaces.plugins.PluginInterface): symbol_status = "exists" vollog.debug(f"symbol {symbol_name} {symbol_status}.") - return + return None @classmethod def list_bugcheck_reason_callbacks( @@ -287,7 +287,7 @@ class Callbacks(interfaces.plugins.PluginInterface): ).address except exceptions.SymbolError: vollog.debug("Cannot find KeBugCheckReasonCallbackListHead") - return + return None full_type_name = ( callback_table_name + constants.BANG + "_KBUGCHECK_REASON_CALLBACK_RECORD" @@ -343,7 +343,7 @@ class Callbacks(interfaces.plugins.PluginInterface): list_offset = ntkrnlmp.get_symbol("KeBugCheckCallbackListHead").address except exceptions.SymbolError: vollog.debug("Cannot find KeBugCheckCallbackListHead") - return + return None full_type_name = ( callback_table_name + constants.BANG + "_KBUGCHECK_CALLBACK_RECORD" diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 38d55d15d..dd82d897e 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -130,7 +130,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVV, f"The file object at {file_obj.vol.offset:#x} is not a file on disk", ) - return + return None # Depending on the type of object (DataSection, ImageSection, SharedCacheMap) we may need to # read from the memory layer or the primary layer. diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index dd7c90860..ddd9cb78e 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -285,7 +285,7 @@ class Handles(interfaces.plugins.PluginInterface): count = 0x1000 / subtype.size if not self.context.layers[virtual].is_valid(offset): - return + return None table = ntkrnlmp.object( object_type="array", @@ -335,7 +335,7 @@ class Handles(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVV, "Handle table parsing was aborted due to an invalid address exception", ) - return + return None for handle_table_entry in self._make_handle_array(TableCode, table_levels): yield handle_table_entry diff --git a/volatility3/framework/plugins/windows/lsadump.py b/volatility3/framework/plugins/windows/lsadump.py index 12589b07e..da8dee325 100644 --- a/volatility3/framework/plugins/windows/lsadump.py +++ b/volatility3/framework/plugins/windows/lsadump.py @@ -168,16 +168,16 @@ class Lsadump(interfaces.plugins.PluginInterface): lsakey = self.get_lsa_key(sechive, bootkey, vista_or_later) if not bootkey: vollog.warning("Unable to find bootkey") - return + return None if not lsakey: vollog.warning("Unable to find lsa key") - return + return None secrets_key = hashdump.Hashdump.get_hive_key(sechive, "Policy\\Secrets") if not secrets_key: vollog.warning("Unable to find secrets key") - return + return None for key in secrets_key.get_subkeys(): sec_val_key = hashdump.Hashdump.get_hive_key( diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 424925955..6ed078996 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -110,7 +110,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_id, excp.invalid_address, excp.layer_name ) ) - return + return None proc_layer = context.layers[proc_layer_name] diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index d3ce3fd2e..24eb02018 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -154,7 +154,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) else: # invalid argument. - return + return None vollog.debug(f"Current Port: {port}") # the given port serves as a shifted index into the port pool lists @@ -175,7 +175,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] if not assignment: - return + return None # the value within assignment.Entry is a) masked and b) points inside of the network object # first decode the pointer diff --git a/volatility3/framework/plugins/windows/pstree.py b/volatility3/framework/plugins/windows/pstree.py index 5c78d1682..a39fe7485 100644 --- a/volatility3/framework/plugins/windows/pstree.py +++ b/volatility3/framework/plugins/windows/pstree.py @@ -108,13 +108,13 @@ class PsTree(interfaces.plugins.PluginInterface): def yield_processes(pid, descendant: bool = False): if pid in process_pids: vollog.debug(f"Pid cycle: already processed pid {pid}") - return + return None process_pids.add(pid) if pid not in self._ancestors and not descendant: vollog.debug(f"Pid cycle: pid {pid} not in filtered tree") - return + return None proc, offset = self._processes[pid] row = ( diff --git a/volatility3/framework/plugins/windows/registry/hivelist.py b/volatility3/framework/plugins/windows/registry/hivelist.py index 91798de40..1cc76dad6 100644 --- a/volatility3/framework/plugins/windows/registry/hivelist.py +++ b/volatility3/framework/plugins/windows/registry/hivelist.py @@ -30,7 +30,7 @@ class HiveGenerator: ): if not hive.is_valid(): self._invalid = hive.vol.offset - return + return None yield hive @property diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index f66e55f4b..e248c19bc 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -74,7 +74,7 @@ class PrintKey(interfaces.plugins.PluginInterface): node_path = [hive.get_node(hive.root_cell_offset)] if not isinstance(node_path, list) or len(node_path) < 1: vollog.warning("Hive walker was not passed a valid node_path (or None)") - return + return None node = node_path[-1] key_path_items = [hive] + node_path[1:] key_path = "\\".join([k.get_name() for k in key_path_items]) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index f90724f66..70c75b50b 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -173,11 +173,11 @@ class UserAssist(interfaces.plugins.PluginInterface): if not userassist_node_path: vollog.warning("list_userassist did not find a valid node_path (or None)") - return + return None if not isinstance(userassist_node_path, list): vollog.warning("userassist_node_path did not return a list as expected") - return + return None userassist_node = userassist_node_path[-1] # iterate through the GUIDs under the userassist key for guidkey in userassist_node.get_subkeys(): diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index b697774cb..d321c2cc0 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -601,21 +601,21 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name): vollog.info("This plugin only supports 64bit Windows memory samples") - return + return None lsass_proc, proc_layer_name = self._find_lsass_proc(procs) if not lsass_proc: vollog.info( "Unable to find a valid lsass.exe process in the process list. This should never happen. Analysis cannot proceed." ) - return + return None cryptdll_base, cryptdll_size = self._find_cryptdll(lsass_proc) if not cryptdll_base: vollog.info( "Unable to find the location of cryptdll.dll inside of lsass.exe. Analysis cannot proceed." ) - return + return None # the custom type information from binary analysis cryptdll_types = self._get_cryptdll_types( @@ -649,7 +649,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): vollog.info( "Unable to find CSystems inside of cryptdll.dll. Analysis cannot proceed." ) - return + return None for csystem in csystems: if not self.context.layers[proc_layer_name].is_valid( diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index 6ec9ebab9..6120b77c9 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -59,7 +59,8 @@ class MultiTypeData(bytes): def __eq__(self, other): return ( - super(self) == super(other) + isinstance(other, self.__class__) + and super() == super(self.__class__, other) and self.converted_int == other.converted_int and self.encoding == other.encoding and self.split_nulls == other.split_nulls diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index f96302684..3d424dedd 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -243,17 +243,17 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ): # task.files can be null if not task.files: - return + return None fd_table = task.files.get_fds() if fd_table == 0: - return + return None max_fds = task.files.get_max_fds() # corruption check if max_fds > 500000: - return + return None file_type = symbol_table + constants.BANG + "file" @@ -378,7 +378,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): """ if not addr: - return + return None type_dec = vmlinux.get_type(type_name) member_offset = type_dec.relative_child_offset(member_name) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3fb772135..d1edfdfe0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -319,7 +319,7 @@ class maple_tree(objects.StructType): vollog.warning( f"The mte {hex(maple_tree_entry)} has all ready been seen, no further results will be produced for this node." ) - return + return None else: seen.add(maple_tree_entry) # check if we have exceeded the expected depth of this maple tree. @@ -402,7 +402,7 @@ class mm_struct(objects.StructType): "get_mmap_iter called on mm_struct where no mmap member exists." ) if not self.mmap: - return + return None yield self.mmap seen = {self.mmap.vol.offset} @@ -723,7 +723,7 @@ class list_head(objects.StructType, collections.abc.Iterable): try: link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: - return + return None if not sentinel: yield self._context.object( symbol_type, layer, offset=self.vol.offset - relative_offset @@ -1218,7 +1218,7 @@ class sock(objects.StructType): return self.sk_socket.get_inode() def get_protocol(self): - return + return None def get_state(self): # Return the generic socket state @@ -1230,13 +1230,13 @@ class sock(objects.StructType): class unix_sock(objects.StructType): def get_name(self): if not self.addr: - return + return None sockaddr_un = self.addr.name.cast("sockaddr_un") saddr = str(utility.array_to_string(sockaddr_un.sun_path)) return saddr def get_protocol(self): - return + return None def get_state(self): """Return a string representing the sock state.""" @@ -1295,7 +1295,7 @@ class inet_sock(objects.StructType): elif hasattr(sk_common, "skc_dport"): dport_le = sk_common.skc_dport else: - return + return None return socket_module.htons(dport_le) def get_src_addr(self): @@ -1313,7 +1313,7 @@ class inet_sock(objects.StructType): addr_size = 16 saddr = self.pinet6.saddr else: - return + return None parent_layer = self._context.layers[self.vol.layer_name] try: addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) @@ -1321,7 +1321,7 @@ class inet_sock(objects.StructType): vollog.debug( f"Unable to read socket src address from {saddr.vol.offset:#x}" ) - return + return None return socket_module.inet_ntop(family, addr_bytes) def get_dst_addr(self): @@ -1342,7 +1342,7 @@ class inet_sock(objects.StructType): daddr = sk_common.skc_v6_daddr addr_size = 16 else: - return + return None parent_layer = self._context.layers[self.vol.layer_name] try: addr_bytes = parent_layer.read(daddr.vol.offset, addr_size) @@ -1350,7 +1350,7 @@ class inet_sock(objects.StructType): vollog.debug( f"Unable to read socket dst address from {daddr.vol.offset:#x}" ) - return + return None return socket_module.inet_ntop(family, addr_bytes) @@ -1388,7 +1388,7 @@ class netlink_sock(objects.StructType): class vsock_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for vsocks - return + return None def get_state(self): # Return the generic socket state @@ -1399,7 +1399,7 @@ class packet_sock(objects.StructType): def get_protocol(self): eth_proto = socket_module.htons(self.num) if eth_proto == 0: - return + return None elif eth_proto in ETH_PROTOCOLS: return ETH_PROTOCOLS[eth_proto] else: @@ -1425,7 +1425,7 @@ class bt_sock(objects.StructType): class xdp_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for xdp_sock - return + return None def get_state(self): # xdp_sock.state is an enum diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 416a7e4d2..e3034d643 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -171,7 +171,7 @@ class elf(objects.StructType): self._find_symbols() if self._cached_symtab is None: - return + return None symtab_arr = self._context.object( self.get_symbol_table_name() + constants.BANG + "array", diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index 56ac96633..bc98e5bdc 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -169,7 +169,7 @@ class MacUtilities(interfaces.configuration.VersionableInterface): try: table_addr = task.p_fd.fd_ofiles.dereference() except exceptions.InvalidAddressException: - return + return None fds = objects.utility.array_of_pointers( table_addr, count=num_fds, subtype=file_type, context=context @@ -204,7 +204,7 @@ class MacUtilities(interfaces.configuration.VersionableInterface): try: current = queue.member(attr=list_head_member) except exceptions.InvalidAddressException: - return + return None while current: if current.vol.offset in seen: diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index c89b527e6..bf0b3d775 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -50,7 +50,7 @@ class proc(generic.GenericIntelProcess): task = self.get_task() current_map = task.map.hdr.links.next except exceptions.InvalidAddressException: - return + return None seen: Set[int] = set() @@ -138,13 +138,13 @@ class vm_map_object(objects.StructType): class vnode(objects.StructType): def _do_calc_path(self, ret, vnodeobj, vname): if vnodeobj is None: - return + return None if vname: try: ret.append(utility.pointer_to_string(vname, 255)) except exceptions.InvalidAddressException: - return + return None if int(vnodeobj.v_flag) & 0x000001 != 0 and int(vnodeobj.v_mount) != 0: if int(vnodeobj.v_mount.mnt_vnodecovered) != 0: @@ -158,7 +158,7 @@ class vnode(objects.StructType): parent = vnodeobj.v_parent parent_name = parent.v_name except exceptions.InvalidAddressException: - return + return None self._do_calc_path(ret, parent, parent_name) @@ -502,7 +502,7 @@ class queue_entry(objects.StructType): yielded = yielded + 1 if yielded == max_size: - return + return None n = ( getattr(n.member(attr=member_name), attr) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ba00a4053..d435851d7 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -91,7 +91,7 @@ class MMVAD_SHORT(objects.StructType): if vad_address in visited: vollog.log(constants.LOGLEVEL_VVV, "VAD node already seen!") - return + return None visited.add(vad_address) tag = self.get_tag() @@ -111,7 +111,7 @@ class MMVAD_SHORT(objects.StructType): constants.LOGLEVEL_VVV, f"Skipping VAD at {self.vol.offset} depth {depth} with tag {tag}", ) - return + return None if target: vad_object = self.cast(target) @@ -665,7 +665,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): ): yield entry except exceptions.InvalidAddressException: - return + return None def init_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: """Generator for DLLs in the order that they were initialized""" @@ -678,7 +678,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): ): yield entry except exceptions.InvalidAddressException: - return + return None def mem_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: """Generator for DLLs in the order that they appear in memory""" @@ -691,7 +691,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): ): yield entry except exceptions.InvalidAddressException: - return + return None def get_handle_count(self): try: @@ -841,11 +841,11 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): try: is_valid = trans_layer.is_valid(self.vol.offset) if not is_valid: - return + return None link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: - return + return None if not sentinel: yield self._context.object( @@ -860,7 +860,7 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): obj_offset = link.vol.offset - relative_offset if not trans_layer.is_valid(obj_offset): - return + return None obj = self._context.object( symbol_type, @@ -875,7 +875,7 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): try: link = getattr(link, direction).dereference() except exceptions.InvalidAddressException: - return + return None def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) @@ -905,10 +905,10 @@ class TOKEN(objects.StructType): sid = sid_and_attr.Sid.dereference().cast("_SID") # catch invalid pointers (UserAndGroupCount is too high) if sid is None: - return + return None # this mimics the windows API IsValidSid if sid.Revision & 0xF != 1 or sid.SubAuthorityCount > 15: - return + return None id_auth = "" for i in sid.IdentifierAuthority.Value: id_auth = i diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index fbd3ead8e..51be0841c 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -162,7 +162,7 @@ class CM_KEY_NODE(objects.StructType): try: signature = node.cast("string", max_length=2, encoding="latin-1") except (exceptions.InvalidAddressException, RegistryFormatException): - return + return None listjump = None if signature == "ri": @@ -220,7 +220,7 @@ class CM_KEY_NODE(objects.StructType): yield node except (exceptions.InvalidAddressException, RegistryFormatException) as excp: vollog.debug(f"Invalid address in get_values iteration: {excp}") - return + return None def get_name(self) -> interfaces.objects.ObjectInterface: """Gets the name for the current key node""" diff --git a/volatility3/framework/symbols/windows/extensions/services.py b/volatility3/framework/symbols/windows/extensions/services.py index 00fb1cc4e..e14de761d 100644 --- a/volatility3/framework/symbols/windows/extensions/services.py +++ b/volatility3/framework/symbols/windows/extensions/services.py @@ -110,7 +110,7 @@ class SERVICE_RECORD(objects.StructType): yield rec rec = rec.ServiceList.Blink.dereference() except exceptions.InvalidAddressException: - return + return None class SERVICE_HEADER(objects.StructType): diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index bdcf25fa1..3816312cd 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -2,7 +2,6 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import binascii import json import logging import lzma From 8b6ab44310e3e39cd3fa06321dd208c2e88d5cd8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 26 Nov 2023 12:30:06 +0000 Subject: [PATCH 465/526] Core: Fix array_of_pointers to act only on those pointers --- volatility3/framework/objects/utility.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index 177074cd7..0292608c1 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -44,8 +44,9 @@ def array_of_pointers( raise TypeError( "Subtype must be a valid template (or string name of an object template)" ) + # We have to clone the pointer class, or we'll be defining the pointer subtype for all future pointers subtype_pointer = context.symbol_space.get_type( symbol_table + constants.BANG + "pointer" - ) + ).clone() subtype_pointer.update_vol(subtype=subtype) return array.cast("array", count=count, subtype=subtype_pointer) From 8d5877e904834811b215f33cedec384c6f25678e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 27 Nov 2023 00:59:56 +0000 Subject: [PATCH 466/526] Simplify attribute object accesses This is a very small percentage slower, for some reason, than the previous mechanism, probably overhead from object creation/member access. However, it vastly simplifies the code and makes better use of the volatility object model. --- .../framework/plugins/windows/mftscan.py | 48 +++++++------------ .../framework/symbols/windows/mft.json | 6 +-- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 87416d274..4612077b8 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -67,9 +67,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - - attr_header = self.context.object( - header_object, + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -77,17 +76,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr_header.AttrType.is_valid_choice: - vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") - - # Offset past the headers to the attribute data - attr_data_offset = ( - offset - + attr_base_offset - + self.context.symbol_space.get_type( - attribute_object - ).relative_child_offset("Attr_Data") - ) + while attr.Attr_Header.AttrType.is_valid_choice: + vollog.debug(f"Attr Type: {attr.Attr_Header.AttrType.lookup()}") # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record @@ -97,19 +87,16 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType.lookup() == "STANDARD_INFORMATION": - attr_data = self.context.object( - si_object, offset=attr_data_offset, layer_name=layer.name - ) - + if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": + attr_data = attr.Attr_Data.cast(si_object) yield 0, ( - format_hints.Hex(attr_data_offset), + format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, renderers.NotApplicableValue(), - attr_header.AttrType.lookup(), + attr.Attr_Header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -118,10 +105,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType.lookup() == "FILE_NAME": - attr_data = self.context.object( - fn_object, offset=attr_data_offset, layer_name=layer.name - ) + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() # If we don't have a valid enum, coerce to hex so we can keep the record @@ -131,13 +116,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): permissions = hex(attr_data.Flags) yield 1, ( - format_hints.Hex(attr_data_offset), + format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, - attr_header.AttrType.lookup(), + attr.Attr_Header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -146,14 +131,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # If there's no advancement the loop will never end, so break it now - if attr_header.Length == 0: + if attr.Attr_Header.Length == 0: break # Update the base offset to point to the next attribute - attr_base_offset += attr_header.Length - # Get the next attribute - attr_header = self.context.object( - header_object, + attr_base_offset += attr.Attr_Header.Length + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index e5de8f3fa..ecbf1d2d7 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -230,21 +230,21 @@ "offset": 0, "type": { "kind": "struct", - "name": "mft!ATTR_HEADER" + "name": "ATTR_HEADER" } }, "Resident_Header": { "offset": 16, "type": { "kind": "struct", - "name": "mft!RESIDENT_HEADER" + "name": "RESIDENT_HEADER" } }, "Attr_Data": { "offset": 24, "type": { "kind": "struct", - "name": "mft!ATTR_HEADER" + "name": "ATTR_HEADER" } } }, From 7624c494e81fddfe1f4ae1b754fae8fbecb76ee2 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 28 Nov 2023 13:57:02 +0100 Subject: [PATCH 467/526] Simplify attribute object accesses like #1049 + custom class (MFTAttribute) --- .../framework/plugins/windows/mftscan.py | 71 ++++++------------- .../symbols/windows/extensions/mft.py | 22 ++++++ 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 397ce75bc..623497638 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -224,12 +224,12 @@ class ADS(interfaces.plugins.PluginInterface): config_path=self.config_path, sub_path="windows", filename="mft", - class_types={"MFT_ENTRY": mft.MFTEntry,"FILE_NAME_ENTRY": mft.MFTFileName}, + class_types={"MFT_ENTRY": mft.MFTEntry,"FILE_NAME_ENTRY": mft.MFTFileName, "ATTRIBUTE": mft.MFTAttribute}, ) # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - header_object = symbol_table + constants.BANG + "ATTR_HEADER" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields @@ -243,58 +243,32 @@ class ADS(interfaces.plugins.PluginInterface): # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - attr_header = self.context.object( - header_object, + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) # There is no field that has a count of Attributes - # Keep Attempting to read attributes until we get an invalid attr_header.AttrType + # Keep Attempting to read attributes until we get an invalid attr.AttrType file_name = renderers.NotAvailableValue is_ads = False - # The First $DATA Attr is the 'principal' file itself not the ADS - while attr_header.AttrType.is_valid_choice: - # Offset past the headers to the attribute data - attr_data_offset = ( - offset - + attr_base_offset - + self.context.symbol_space.get_type( - header_object - ).size - ) + + # The First $DATA Attr is the 'principal' file itself not the ADS + while attr.Attr_Header.AttrType.is_valid_choice: - if attr_header.AttrType.lookup() == "FILE_NAME": - attr_data = self.context.object( - fn_object, offset=attr_data_offset, layer_name=layer.name - ) + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() - - # DATA Attribute (can be ADS or not) - if attr_header.AttrType.lookup() == "DATA": + + if attr.Attr_Header.AttrType.lookup() == "DATA": if is_ads: - if not attr_header.NonResidentFlag: + if not attr.Attr_Header.NonResidentFlag: # Resident files are the most interesting. - if attr_header.NameLength > 0: - attr_name_offset = ( - offset - + attr_base_offset - + attr_header.NameOffset - ) + if attr.Attr_Header.NameLength > 0: - ads_name = self._context.layers[layer.name].read( - attr_name_offset, attr_header.NameLength*2 , pad=True - ).decode('utf-16') - - attr_content_offset = ( - offset - + attr_base_offset - + attr_header.ContentOffset - ) - - content = self._context.layers[layer.name].read( - attr_content_offset, attr_header.ContentLength , pad=True - ) + ads_name = attr.get_resident_filename() + content = attr.get_resident_filecontent() # Preparing for Disassembly architecture = layer.metadata.get("architecture", None) @@ -303,10 +277,10 @@ class ADS(interfaces.plugins.PluginInterface): ) yield 0, ( - format_hints.Hex(attr_data_offset), + format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), mft_record.RecordNumber, - attr_header.AttrType.lookup(), + attr.Attr_Header.AttrType.lookup(), file_name, ads_name, format_hints.HexBytes(content), @@ -317,18 +291,17 @@ class ADS(interfaces.plugins.PluginInterface): # If there's no advancement the loop will never end, so break it now - if attr_header.Length == 0: + if attr.Attr_Header.Length == 0: break # Update the base offset to point to the next attribute - attr_base_offset += attr_header.Length + attr_base_offset += attr.Attr_Header.Length # Get the next attribute - attr_header = self.context.object( - header_object, + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) - def run(self): return renderers.TreeGrid( [ diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 17b6c8325..1b5d5fce4 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -21,3 +21,25 @@ class MFTFileName(objects.StructType): "string", encoding="utf16", max_length=self.NameLength * 2, errors="replace" ) return output + + +class MFTAttribute(objects.StructType): + """This represents an MFT ATTRIBUTE""" + + def get_resident_filename(self) -> str: + # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data + layer = self._context.layers[self.vol.layer_name] + attr_name_offset = self.vol.offset + self.Attr_Header.NameOffset + + return self._context.layers[layer.name].read( + attr_name_offset, self.Attr_Header.NameLength*2 , pad=True + ).decode('utf-16') + + def get_resident_filecontent(self) -> bytes: + # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data + layer = self._context.layers[self.vol.layer_name] + attr_content_offset = self.vol.offset + self.Attr_Header.ContentOffset + + return self._context.layers[layer.name].read( + attr_content_offset, self.Attr_Header.ContentLength , pad=True + ) From c610497fa04de41042104fdefbfa243c0bcf76a9 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 29 Nov 2023 09:28:34 +0000 Subject: [PATCH 468/526] Windows: update vadyarascan to use generic yarascan requirements --- .../framework/plugins/windows/vadyarascan.py | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 4b30a9d8b..d795818e9 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -18,47 +18,26 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [ + # create a list of requirements for vadyarascan + vadyarascan_requirements = [ requirements.ModuleRequirement( name="kernel", description="Windows kernel", architectures=["Intel32", "Intel64"], ), - requirements.BooleanRequirement( - name="wide", - description="Match wide (unicode) strings", - default=False, - optional=True, - ), - requirements.StringRequirement( - name="yara_rules", description="Yara rules (as a string)", optional=True - ), - requirements.URIRequirement( - name="yara_file", description="Yara rules (as a file)", optional=True - ), - # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code - # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later - requirements.URIRequirement( - name="yara_compiled_file", - description="Yara compiled rules (as a file)", - optional=True, - ), - requirements.IntRequirement( - name="max_size", - default=0x40000000, - description="Set the maximum size (default is 1GB)", - optional=True, - ), requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.VersionRequirement( name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) + ), requirements.ListRequirement( name="pid", element_type=int, @@ -67,6 +46,12 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ), ] + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() + + # return the combined requirements + return yarascan_requirements + vadyarascan_requirements + def _generator(self): kernel = self.context.modules[self.config["kernel"]] From 39144ff45fc89bb3a22325afe996cf1ba2dc4584 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 29 Nov 2023 12:51:08 +0100 Subject: [PATCH 469/526] type hint get_symbols_names_and_addresses --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f5c71eb0e..1ec9c7416 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -141,7 +141,7 @@ class module(generic.GenericIntelProcess): for sym in syms: yield sym - def get_symbols_names_and_addresses(self): + def get_symbols_names_and_addresses(self) -> Tuple[str, int]: """Get names and addresses for each symbol of the module Yields: From b20643d5e01d8d8771937071ab433926e51fc18d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Nov 2023 12:03:43 +0000 Subject: [PATCH 470/526] Linux: Tidy up the elf symbol table name --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1824a5b76..c9f7f50da 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -107,7 +107,7 @@ class module(generic.GenericIntelProcess): def get_elf_table_name(self): elf_table_name = intermed.IntermediateSymbolTable.create( self._context, - "config_name_elf_symbol_table", + "elf_symbol_table", "linux", "elf", native_types=None, From 099403d07ce0651119d679accb79c8d0ff1253db Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 29 Nov 2023 20:47:14 +0000 Subject: [PATCH 471/526] Linux: fix bug with iomem plugin where an absolute address is used to make an object but the absolute flag is not set --- volatility3/framework/plugins/linux/iomem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index 785405ef3..6732084db 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -16,7 +16,7 @@ class IOMem(interfaces.plugins.PluginInterface): """Generates an output similar to /proc/iomem on a running system.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -53,7 +53,7 @@ class IOMem(interfaces.plugins.PluginInterface): # create the resource object with protection against memory smear try: - resource = vmlinux.object("resource", resource_offset) + resource = vmlinux.object("resource", resource_offset, absolute=True) except exceptions.InvalidAddressException: vollog.warning( f"Unable to create resource object at {resource_offset:#x}. This resource, " From ed98d453f676072dacac9f3ae343e54bcb17bd2e Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Thu, 30 Nov 2023 11:50:04 +0100 Subject: [PATCH 472/526] Changing requirements + formating --- requirements.txt | 4 +- volatility3/framework/layers/cloudstorage.py | 56 ++++++++++---------- 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7e1c28595..c05546466 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,5 +18,5 @@ pycryptodome leechcorepyc>=2.4.0 # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage -gcsfs>=2023.6.0 -s3fs>=2023.6.0 \ No newline at end of file +gcsfs>=2023.1.0 +s3fs>=2023.1.0 \ No newline at end of file diff --git a/volatility3/framework/layers/cloudstorage.py b/volatility3/framework/layers/cloudstorage.py index ce6cdb327..41afa324d 100644 --- a/volatility3/framework/layers/cloudstorage.py +++ b/volatility3/framework/layers/cloudstorage.py @@ -23,35 +23,33 @@ from volatility3.framework.layers import resources vollog = logging.getLogger(__file__) -class S3FileSystemHandler(resources.VolatilityHandler): - if HAS_S3FS: - @classmethod - def non_cached_schemes(cls) -> List[str]: - return ["s3"] +if HAS_S3FS: - @staticmethod - def default_open(req: urllib.request.Request) -> Optional[Any]: - """Handles the request if it's the s3 scheme.""" - if req.type == "s3": - object_uri = "://".join(req.full_url.split("://")[1:]) - return s3fs.S3FileSystem().open(object_uri) - return None - else: - raise exceptions.LayerException("s3 requirement is missing.") + class S3FileSystemHandler(resources.VolatilityHandler): + + @classmethod + def non_cached_schemes(cls) -> List[str]: + return ["s3"] + @staticmethod + def default_open(req: urllib.request.Request) -> Optional[Any]: + """Handles the request if it's the s3 scheme.""" + if req.type == "s3": + object_uri = "://".join(req.full_url.split("://")[1:]) + return s3fs.S3FileSystem().open(object_uri) + return None -class GSFileSystemHandler(resources.VolatilityHandler): - if HAS_GCSFS: - @classmethod - def non_cached_schemes(cls) -> List[str]: - return ["gs"] - - @staticmethod - def default_open(req: urllib.request.Request) -> Optional[Any]: - """Handles the request if it's the gs scheme.""" - if req.type == "gs": - object_uri = "://".join(req.full_url.split("://")[1:]) - return gcsfs.GCSFileSystem().open(object_uri) - return None - else: - raise exceptions.LayerException("gcsfs requirement is missing.") \ No newline at end of file +if HAS_GCSFS: + + class GSFileSystemHandler(resources.VolatilityHandler): + @classmethod + def non_cached_schemes(cls) -> List[str]: + return ["gs"] + + @staticmethod + def default_open(req: urllib.request.Request) -> Optional[Any]: + """Handles the request if it's the gs scheme.""" + if req.type == "gs": + object_uri = "://".join(req.full_url.split("://")[1:]) + return gcsfs.GCSFileSystem().open(object_uri) + return None \ No newline at end of file From 1f5a18d679424563a419d216e2e3353b20683f2c Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Thu, 30 Nov 2023 11:54:55 +0100 Subject: [PATCH 473/526] patch running black --- volatility3/framework/layers/cloudstorage.py | 48 ++++++++++---------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/volatility3/framework/layers/cloudstorage.py b/volatility3/framework/layers/cloudstorage.py index 41afa324d..3f88ef34f 100644 --- a/volatility3/framework/layers/cloudstorage.py +++ b/volatility3/framework/layers/cloudstorage.py @@ -8,12 +8,14 @@ from typing import Optional, Any, List try: import s3fs + HAS_S3FS = True except ImportError: HAS_S3FS = False try: import gcsfs + HAS_GCSFS = True except ImportError: HAS_GCSFS = False @@ -26,30 +28,30 @@ vollog = logging.getLogger(__file__) if HAS_S3FS: class S3FileSystemHandler(resources.VolatilityHandler): - - @classmethod - def non_cached_schemes(cls) -> List[str]: - return ["s3"] + @classmethod + def non_cached_schemes(cls) -> List[str]: + return ["s3"] + + @staticmethod + def default_open(req: urllib.request.Request) -> Optional[Any]: + """Handles the request if it's the s3 scheme.""" + if req.type == "s3": + object_uri = "://".join(req.full_url.split("://")[1:]) + return s3fs.S3FileSystem().open(object_uri) + return None - @staticmethod - def default_open(req: urllib.request.Request) -> Optional[Any]: - """Handles the request if it's the s3 scheme.""" - if req.type == "s3": - object_uri = "://".join(req.full_url.split("://")[1:]) - return s3fs.S3FileSystem().open(object_uri) - return None if HAS_GCSFS: - + class GSFileSystemHandler(resources.VolatilityHandler): - @classmethod - def non_cached_schemes(cls) -> List[str]: - return ["gs"] - - @staticmethod - def default_open(req: urllib.request.Request) -> Optional[Any]: - """Handles the request if it's the gs scheme.""" - if req.type == "gs": - object_uri = "://".join(req.full_url.split("://")[1:]) - return gcsfs.GCSFileSystem().open(object_uri) - return None \ No newline at end of file + @classmethod + def non_cached_schemes(cls) -> List[str]: + return ["gs"] + + @staticmethod + def default_open(req: urllib.request.Request) -> Optional[Any]: + """Handles the request if it's the gs scheme.""" + if req.type == "gs": + object_uri = "://".join(req.full_url.split("://")[1:]) + return gcsfs.GCSFileSystem().open(object_uri) + return None From 7fe086f64f7c98a2c46990eff2e585a383d4bea2 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 1 Dec 2023 13:37:09 +0000 Subject: [PATCH 474/526] Linux: update maple tree extension to fix issue #1032 correcting the mutable type used as a default parameter. --- .../symbols/linux/extensions/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index c3e50fce4..92c544c30 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -301,11 +301,7 @@ class maple_tree(objects.StructType): self.ma_flags & self.MT_FLAGS_HEIGHT_MASK ) >> self.MT_FLAGS_HEIGHT_OFFSET yield from self._parse_maple_tree_node( - self.ma_root, - maple_tree_offset, - expected_maple_tree_depth, - seen=set(), - current_depth=1, + self.ma_root, maple_tree_offset, expected_maple_tree_depth ) def _parse_maple_tree_node( @@ -313,11 +309,16 @@ class maple_tree(objects.StructType): maple_tree_entry, parent, expected_maple_tree_depth, - seen=set(), + seen=None, current_depth=1, ): """Recursively parse Maple Tree Nodes and yield all non empty slots""" + # create seen set if it does not exist, e.g. on the first call into + # this recursive function. + if seen == None: + seen = set() + # protect against unlikely loop if maple_tree_entry in seen: vollog.warning( @@ -326,6 +327,7 @@ class maple_tree(objects.StructType): return else: seen.add(maple_tree_entry) + # check if we have exceeded the expected depth of this maple tree. # e.g. when current_depth is larger than expected_maple_tree_depth there may be an issue. # it is normal that expected_maple_tree_depth is equal to current_depth. @@ -334,6 +336,7 @@ class maple_tree(objects.StructType): f"The depth for the maple tree at {hex(self.vol.offset)} is {expected_maple_tree_depth}, however when parsing the nodes " f"a depth of {current_depth} was reached. This is unexpected and may lead to incorrect results." ) + # parse the mte to extract the pointer value, node type, and leaf status pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) node_type = ( From 276e695237e0a93cdb4755bcc8c063acda7f3a85 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 1 Dec 2023 13:46:00 +0000 Subject: [PATCH 475/526] Linux: update maple tree extension comment around the seen set. --- .../framework/symbols/linux/extensions/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 92c544c30..dc31a7628 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -314,8 +314,13 @@ class maple_tree(objects.StructType): ): """Recursively parse Maple Tree Nodes and yield all non empty slots""" - # create seen set if it does not exist, e.g. on the first call into - # this recursive function. + # Create seen set if it does not exist, e.g. on the first call into this recursive function. This + # must be None or an existing set of addresses for MTEs that have already been processed or that + # should otherwise be ignored. If parsing from the root node for example this should be None on the + # first call. If you needed to parse all nodes downwards from part of the tree this should still be + # None. If however you wanted to parse from a node, but ignore some parts of the tree below it then + # this could be populated with the addresses of the nodes you wish to ignore. + if seen == None: seen = set() From ed2db939d6b36d18dd44bad13d6a603b760b62a2 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 3 Dec 2023 12:24:11 +0100 Subject: [PATCH 476/526] Better exception handling. Fetching data using objects --- .../framework/plugins/windows/mftscan.py | 93 +++++++++---------- .../symbols/windows/extensions/mft.py | 39 +++++--- 2 files changed, 70 insertions(+), 62 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 623497638..4d58eb1e2 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -31,7 +31,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) ), - ] def _generator(self): @@ -53,6 +52,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" @@ -67,9 +67,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - - attr_header = self.context.object( - header_object, + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -77,17 +76,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr_header.AttrType.is_valid_choice: - vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") - - # Offset past the headers to the attribute data - attr_data_offset = ( - offset - + attr_base_offset - + self.context.symbol_space.get_type( - header_object - ).size - ) + while attr.Attr_Header.AttrType.is_valid_choice: + vollog.debug(f"Attr Type: {attr.Attr_Header.AttrType.lookup()}") # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record @@ -97,19 +87,16 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType.lookup() == "STANDARD_INFORMATION": - attr_data = self.context.object( - si_object, offset=attr_data_offset, layer_name=layer.name - ) - + if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": + attr_data = attr.Attr_Data.cast(si_object) yield 0, ( - format_hints.Hex(attr_data_offset), + format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, renderers.NotApplicableValue(), - attr_header.AttrType.lookup(), + attr.Attr_Header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -118,10 +105,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType.lookup() == "FILE_NAME": - attr_data = self.context.object( - fn_object, offset=attr_data_offset, layer_name=layer.name - ) + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() # If we don't have a valid enum, coerce to hex so we can keep the record @@ -131,13 +116,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): permissions = hex(attr_data.Flags) yield 1, ( - format_hints.Hex(attr_data_offset), + format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, - attr_header.AttrType.lookup(), + attr.Attr_Header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -146,14 +131,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # If there's no advancement the loop will never end, so break it now - if attr_header.Length == 0: + if attr.Attr_Header.Length == 0: break # Update the base offset to point to the next attribute - attr_base_offset += attr_header.Length - # Get the next attribute - attr_header = self.context.object( - header_object, + attr_base_offset += attr.Attr_Header.Length + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -224,7 +208,11 @@ class ADS(interfaces.plugins.PluginInterface): config_path=self.config_path, sub_path="windows", filename="mft", - class_types={"MFT_ENTRY": mft.MFTEntry,"FILE_NAME_ENTRY": mft.MFTFileName, "ATTRIBUTE": mft.MFTAttribute}, + class_types={ + "MFT_ENTRY": mft.MFTEntry, + "FILE_NAME_ENTRY": mft.MFTFileName, + "ATTRIBUTE": mft.MFTAttribute, + }, ) # get each of the individual Field Sets @@ -251,30 +239,39 @@ class ADS(interfaces.plugins.PluginInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr.AttrType - file_name = renderers.NotAvailableValue is_ads = False - + file_name = renderers.NotAvailableValue # The First $DATA Attr is the 'principal' file itself not the ADS while attr.Attr_Header.AttrType.is_valid_choice: - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() - if attr.Attr_Header.AttrType.lookup() == "DATA": if is_ads: if not attr.Attr_Header.NonResidentFlag: # Resident files are the most interesting. if attr.Attr_Header.NameLength > 0: - ads_name = attr.get_resident_filename() - content = attr.get_resident_filecontent() + if not ads_name: + ads_name = renderers.NotAvailableValue - # Preparing for Disassembly - architecture = layer.metadata.get("architecture", None) - disasm = interfaces.renderers.Disassembly( - content, 0, architecture.lower() - ) + content = attr.get_resident_filecontent() + if content: + # Preparing for Disassembly + architecture = layer.metadata.get( + "architecture", None + ) + + disasm = ( + interfaces.renderers.Disassembly( + content, 0, architecture.lower() + ) + if architecture + else interfaces.renderers.BaseAbsentValue + ) + else: + content = renderers.NotAvailableValue + disasm = interfaces.renderers.BaseAbsentValue yield 0, ( format_hints.Hex(attr_data.vol.offset), @@ -288,8 +285,7 @@ class ADS(interfaces.plugins.PluginInterface): ) else: is_ads = True - - + # If there's no advancement the loop will never end, so break it now if attr.Attr_Header.Length == 0: break @@ -302,6 +298,7 @@ class ADS(interfaces.plugins.PluginInterface): offset=offset + attr_base_offset, layer_name=layer.name, ) + def run(self): return renderers.TreeGrid( [ @@ -315,4 +312,4 @@ class ADS(interfaces.plugins.PluginInterface): ("Disasm", interfaces.renderers.Disassembly), ], self._generator(), - ) \ No newline at end of file + ) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 1b5d5fce4..14c1f08d6 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import objects +from volatility3.framework import objects, constants, exceptions class MFTEntry(objects.StructType): @@ -28,18 +28,29 @@ class MFTAttribute(objects.StructType): def get_resident_filename(self) -> str: # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data - layer = self._context.layers[self.vol.layer_name] - attr_name_offset = self.vol.offset + self.Attr_Header.NameOffset - - return self._context.layers[layer.name].read( - attr_name_offset, self.Attr_Header.NameLength*2 , pad=True - ).decode('utf-16') - + try: + name = self._context.object( + self.vol.type_name.split(constants.BANG)[0] + constants.BANG + "string", + layer_name=self.vol.layer_name, + offset=self.vol.offset + self.Attr_Header.NameOffset, + max_length=self.Attr_Header.NameLength * 2, + errors="replace", + encoding="utf16", + ) + return name + except exceptions.InvalidAddressException: + return None + def get_resident_filecontent(self) -> bytes: # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data - layer = self._context.layers[self.vol.layer_name] - attr_content_offset = self.vol.offset + self.Attr_Header.ContentOffset - - return self._context.layers[layer.name].read( - attr_content_offset, self.Attr_Header.ContentLength , pad=True - ) + try: + bytesobj = self._context.object( + self.vol.type_name.split(constants.BANG)[0] + constants.BANG + "bytes", + layer_name=self.vol.layer_name, + offset=self.vol.offset + self.Attr_Header.ContentOffset, + native_layer_name=self.vol.native_layer_name, + length=self.Attr_Header.ContentLength, + ) + return bytesobj + except exceptions.InvalidAddressException: + return None From acb088dbcdb8532e643a72fb4571ae1cba24786c Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 3 Dec 2023 13:05:45 +0100 Subject: [PATCH 477/526] Better code reading --- volatility3/framework/plugins/windows/mftscan.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 4d58eb1e2..7e4e1ca18 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -258,17 +258,14 @@ class ADS(interfaces.plugins.PluginInterface): content = attr.get_resident_filecontent() if content: # Preparing for Disassembly + disasm = interfaces.renderers.BaseAbsentValue architecture = layer.metadata.get( "architecture", None ) - - disasm = ( - interfaces.renderers.Disassembly( + if architecture: + disasm = interfaces.renderers.Disassembly( content, 0, architecture.lower() ) - if architecture - else interfaces.renderers.BaseAbsentValue - ) else: content = renderers.NotAvailableValue disasm = interfaces.renderers.BaseAbsentValue From 831e6cf3412f71dffb929a8f42db383e5792214b Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 7 Dec 2023 09:06:13 +0000 Subject: [PATCH 478/526] Linux: update kmsg KmsgFiveTen class to handle symbol shift and create objects with absolute addresses --- volatility3/framework/plugins/linux/kmsg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 3f7345bdc..f3ef12cfe 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -67,7 +67,7 @@ class ABCKmsg(ABC): vmlinux = context.modules[self._config["kernel"]] self.layer_name = vmlinux.layer_name # type: ignore symbol_table_name = vmlinux.symbol_table_name # type: ignore - self.vmlinux = contexts.Module.create(context, symbol_table_name, self.layer_name, 0) # type: ignore + self.vmlinux = contexts.Module.create(context, symbol_table_name, self.layer_name, vmlinux.offset) # type: ignore self.long_unsigned_int_size = self.vmlinux.get_type("long unsigned int").size @classmethod @@ -365,12 +365,14 @@ class KmsgFiveTen(ABCKmsg): offset=desc_ring.descs, subtype=self.vmlinux.get_type("prb_desc"), count=desc_count, + absolute=True, ) info_arr = self.vmlinux.object( object_type="array", offset=desc_ring.infos, subtype=self.vmlinux.get_type("printk_info"), count=desc_count, + absolute=True, ) # See kernel/printk/printk_ringbuffer.h From c08dda88ded4be77a0f1f2aaf0b3bf3f25181334 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 8 Dec 2023 22:56:17 +0000 Subject: [PATCH 479/526] Linux: Update pslist plugin so that it can be used from pstree, and update pstree to support dumping of processes --- volatility3/framework/plugins/linux/pslist.py | 113 +++++++++++------- volatility3/framework/plugins/linux/pstree.py | 12 +- 2 files changed, 83 insertions(+), 42 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 16e370b6e..7e38625ac 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -1,7 +1,7 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, Callable, Iterable, List +from typing import Any, Callable, Iterable, List, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -17,7 +17,7 @@ class PsList(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 1, 0) + _version = (2, 1, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -78,6 +78,71 @@ class PsList(interfaces.plugins.PluginInterface): else: return lambda _: False + def _get_task_fields( + self, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False + ) -> Tuple[int, int, int, str]: + """Extract the fields needed for the final output + Args: + task: A task object from where to get the fields. + decorate_comm: If True, it decorates the comm string of + - User threads: in curly brackets, + - Kernel threads: in square brackets + Defaults to False. + Returns: + A tuple with the fields to show in the plugin output. + """ + pid = task.tgid + tid = task.pid + ppid = task.parent.tgid if task.parent else 0 + name = utility.array_to_string(task.comm) + if decorate_comm: + if task.is_kernel_thread: + name = f"[{name}]" + elif task.is_user_thread: + name = f"{{{name}}}" + + task_fields = (format_hints.Hex(task.vol.offset), pid, tid, ppid, name) + return task_fields + + def _get_file_output(self, task: interfaces.objects.ObjectInterface) -> str: + """Extract the elf for the process if requested + Args: + task: A task object to extract from. + Returns: + A string showing the results of the extraction, either + the filename used or an error. + """ + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "linux", + "elf", + class_types=elf.class_types, + ) + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + # if we can't build a proc layer we can't + # extract the elf + return renderers.NotApplicableValue() + else: + # Find the vma that belongs to the main ELF of the process + file_output = "Error outputting file" + for v in task.mm.get_mmap_iter(): + if v.vm_start == task.mm.start_code: + file_handle = elfs.Elfs.elf_dump( + self.context, + proc_layer_name, + elf_table_name, + v, + task, + self.open, + ) + if file_handle: + file_output = str(file_handle.preferred_filename) + file_handle.close() + break + return file_output + def _generator( self, pid_filter: Callable[[Any], bool], @@ -104,49 +169,15 @@ class PsList(interfaces.plugins.PluginInterface): for task in self.list_tasks( self.context, self.config["kernel"], pid_filter, include_threads ): - elf_table_name = intermed.IntermediateSymbolTable.create( - self.context, - self.config_path, - "linux", - "elf", - class_types=elf.class_types, - ) - file_output = "Disabled" if dump: - proc_layer_name = task.add_process_layer() - if not proc_layer_name: - continue + file_output = self._get_file_output(task) + else: + file_output = "Disabled" - # Find the vma that belongs to the main ELF of the process - file_output = "Error outputting file" - - for v in task.mm.get_mmap_iter(): - if v.vm_start == task.mm.start_code: - file_handle = elfs.Elfs.elf_dump( - self.context, - proc_layer_name, - elf_table_name, - v, - task, - self.open, - ) - if file_handle: - file_output = str(file_handle.preferred_filename) - file_handle.close() - break - - pid = task.tgid - tid = task.pid - ppid = task.parent.tgid if task.parent else 0 - name = utility.array_to_string(task.comm) - if decorate_comm: - if task.is_kernel_thread: - name = f"[{name}]" - elif task.is_user_thread: - name = f"{{{name}}}" + offset, pid, tid, ppid, name = self._get_task_fields(task, decorate_comm) yield 0, ( - format_hints.Hex(task.vol.offset), + offset, pid, tid, ppid, diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index e07a8aced..f42986f56 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -39,7 +39,11 @@ class PsTree(pslist.PsList): self._levels[pid] = level def _generator( - self, pid_filter, include_threads: bool = False, decorate_com: bool = False + self, + pid_filter, + include_threads: bool = False, + decorate_com: bool = False, + dump: bool = False, ): """Generates the tasks hierarchy tree. @@ -72,6 +76,12 @@ class PsTree(pslist.PsList): task = self._tasks[pid] row = self._get_task_fields(task, decorate_com) + if dump: + file_output = self._get_file_output(task) + else: + file_output = "Disabled" + row = self._get_task_fields(task, decorate_com) + row += (file_output,) # also add the file output column tid = task.pid yield (self._levels[tid] - 1, row) From 19499d511315b718186dff05d133fa86f3310947 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 11 Dec 2023 06:37:17 +0000 Subject: [PATCH 480/526] Linux: update pslist with classmethod for get_task_fields --- volatility3/framework/plugins/linux/pslist.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 7e38625ac..9afd13e5a 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -17,7 +17,7 @@ class PsList(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 1, 1) + _version = (2, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -78,8 +78,9 @@ class PsList(interfaces.plugins.PluginInterface): else: return lambda _: False - def _get_task_fields( - self, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False + @classmethod + def get_task_fields( + cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False ) -> Tuple[int, int, int, str]: """Extract the fields needed for the final output Args: @@ -101,7 +102,7 @@ class PsList(interfaces.plugins.PluginInterface): elif task.is_user_thread: name = f"{{{name}}}" - task_fields = (format_hints.Hex(task.vol.offset), pid, tid, ppid, name) + task_fields = (task.vol.offset, pid, tid, ppid, name) return task_fields def _get_file_output(self, task: interfaces.objects.ObjectInterface) -> str: @@ -174,10 +175,10 @@ class PsList(interfaces.plugins.PluginInterface): else: file_output = "Disabled" - offset, pid, tid, ppid, name = self._get_task_fields(task, decorate_comm) + offset, pid, tid, ppid, name = self.get_task_fields(task, decorate_comm) yield 0, ( - offset, + format_hints.Hex(offset), pid, tid, ppid, From 3d9b0208cb3eb0f939140538054c3ac707af6123 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 12 Dec 2023 06:56:14 +0000 Subject: [PATCH 481/526] Linux: change pstree to a basic plugin rather than inheriting from pslist --- volatility3/framework/plugins/linux/pstree.py | 102 +++++++++++++----- 1 file changed, 74 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index f42986f56..efe5223df 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -2,18 +2,49 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints from volatility3.plugins.linux import pslist -class PsTree(pslist.PsList): +class PsTree(interfaces.plugins.PluginInterface): """Plugin for listing processes in a tree based on their parent process ID.""" - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._tasks = {} - self._levels = {} - self._children = {} + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.BooleanRequirement( + name="threads", + description="Include user threads", + optional=True, + default=False, + ), + requirements.BooleanRequirement( + name="decorate_comm", + description="Show `user threads` comm in curly brackets, and `kernel threads` comm in square brackets", + optional=True, + default=False, + ), + ] def find_level(self, pid: int) -> None: """Finds how deep the PID is in the tasks hierarchy. @@ -40,18 +71,13 @@ class PsTree(pslist.PsList): def _generator( self, - pid_filter, - include_threads: bool = False, - decorate_com: bool = False, - dump: bool = False, + tasks: list, + decorate_comm: bool = False, ): """Generates the tasks hierarchy tree. Args: - pid_filter: A function which takes a process object and returns True if the process should be ignored/filtered - include_threads: If True, the output will also show the user threads - If False, only the thread group leaders will be shown - Defaults to False. + tasks: A list of task objects to be displayed decorate_comm: If True, it decorates the comm string of - User threads: in curly brackets, - Kernel threads: in square brackets @@ -59,13 +85,12 @@ class PsTree(pslist.PsList): Yields: Each rows """ - vmlinux = self.context.modules[self.config["kernel"]] - for proc in self.list_tasks( - self.context, - vmlinux.name, - filter_func=pid_filter, - include_threads=include_threads, - ): + + self._tasks = {} + self._levels = {} + self._children = {} + + for proc in tasks: self._tasks[proc.pid] = proc # Build the child/level maps @@ -75,13 +100,10 @@ class PsTree(pslist.PsList): def yield_processes(pid): task = self._tasks[pid] - row = self._get_task_fields(task, decorate_com) - if dump: - file_output = self._get_file_output(task) - else: - file_output = "Disabled" - row = self._get_task_fields(task, decorate_com) - row += (file_output,) # also add the file output column + row = pslist.PsList.get_task_fields(task, decorate_comm) + # update the first element, the offset, in the row tuple to use format_hints.Hex + # as a simple int is returned from get_task_fields. + row = (format_hints.Hex(row[0]),) + row[1:] tid = task.pid yield (self._levels[tid] - 1, row) @@ -92,3 +114,27 @@ class PsTree(pslist.PsList): for pid, level in self._levels.items(): if level == 1: yield from yield_processes(pid) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + include_threads = self.config.get("threads") + decorate_comm = self.config.get("decorate_comm") + + return renderers.TreeGrid( + [ + ("OFFSET (V)", format_hints.Hex), + ("PID", int), + ("TID", int), + ("PPID", int), + ("COMM", str), + ], + self._generator( + pslist.PsList.list_tasks( + self.context, + self.config["kernel"], + filter_func=filter_func, + include_threads=include_threads, + ), + decorate_comm=decorate_comm, + ), + ) From fda3cedabaae0f98600201ec800ed415521e402b Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 14 Dec 2023 09:18:01 +0000 Subject: [PATCH 482/526] Linux: update kmsg to create objects via context rather than module --- volatility3/framework/plugins/linux/kmsg.py | 23 +++++++++++---------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index f3ef12cfe..17dcc61aa 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -64,10 +64,8 @@ class ABCKmsg(ABC): ): self._context = context self._config = config - vmlinux = context.modules[self._config["kernel"]] - self.layer_name = vmlinux.layer_name # type: ignore - symbol_table_name = vmlinux.symbol_table_name # type: ignore - self.vmlinux = contexts.Module.create(context, symbol_table_name, self.layer_name, vmlinux.offset) # type: ignore + self.vmlinux = context.modules[self._config["kernel"]] + self.layer_name = self.vmlinux.layer_name # type: ignore self.long_unsigned_int_size = self.vmlinux.get_type("long unsigned int").size @classmethod @@ -358,21 +356,24 @@ class KmsgFiveTen(ABCKmsg): desc_ring = ringbuffers.desc_ring text_data_ring = ringbuffers.text_data_ring - desc_count = 1 << desc_ring.count_bits - desc_arr = self.vmlinux.object( - object_type="array", + + array_type = self.vmlinux.symbol_table_name + constants.BANG + "array" + + desc_arr = self._context.object( + array_type, offset=desc_ring.descs, subtype=self.vmlinux.get_type("prb_desc"), count=desc_count, - absolute=True, + layer_name=self.layer_name, ) - info_arr = self.vmlinux.object( - object_type="array", + + info_arr = self._context.object( + array_type, offset=desc_ring.infos, subtype=self.vmlinux.get_type("printk_info"), count=desc_count, - absolute=True, + layer_name=self.layer_name, ) # See kernel/printk/printk_ringbuffer.h From 9b8534267389988e18ba77b8d8a80cdc56a1e4dc Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 14 Dec 2023 09:24:51 +0000 Subject: [PATCH 483/526] Linux: bump version for linux kmsg --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 17dcc61aa..5136a00f6 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -412,7 +412,7 @@ class Kmsg(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 604c23bcbc88679a48d1348ef3c8d2163726ab44 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Thu, 14 Dec 2023 14:50:17 +0100 Subject: [PATCH 484/526] Import Address Table Plugin --- volatility3/framework/plugins/windows/iat.py | 132 +++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 volatility3/framework/plugins/windows/iat.py diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py new file mode 100644 index 000000000..1cf51cfa2 --- /dev/null +++ b/volatility3/framework/plugins/windows/iat.py @@ -0,0 +1,132 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + +import logging +import io +from typing import Callable, List +from volatility3.framework.symbols import intermed +from volatility3.framework import renderers, interfaces, exceptions, constants +from volatility3.framework.configuration import requirements +from volatility3.plugins.windows import pslist +from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.symbols.windows.extensions import pe +import pefile + +vollog = logging.getLogger(__name__) + + +class IAT(interfaces.plugins.PluginInterface): + """Extract Import Address Table to list API (functions) used by a program contained in external libraries""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process ID to include (all other processes are excluded)", + optional=True, + ), + ] + + def _generator(self, procs): + kernel = self.context.modules[self.config["kernel"]] + + for proc in procs: + try: + proc_id = proc.UniqueProcessId + proc_layer_name = proc.add_process_layer() + peb = self.context.object( + kernel.symbol_table_name + constants.BANG + "_PEB", + layer_name=proc_layer_name, + offset=proc.Peb, + ) + + if proc_layer_name is None: + raise TypeError("Layer must be a string not None") + + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "windows", + "pe", + class_types=pe.class_types, + ) + pe_data = io.BytesIO() + + dos_header = self.context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=peb.ImageBaseAddress, + layer_name=proc_layer_name, + ) + + for offset, data in dos_header.reconstruct(): + pe_data.seek(offset) + pe_data.write(data) + + pe_obj = pefile.PE(data=pe_data.getvalue(), fast_load=True) + pe_obj.parse_data_directories( + [pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_IMPORT"]] + ) + if hasattr(pe_obj, "DIRECTORY_ENTRY_IMPORT"): + for entry in pe_obj.DIRECTORY_ENTRY_IMPORT: + dll_entry = entry.dll + if dll_entry: + dll_entry = dll_entry.decode() + else: + dll_entry = renderers.NotAvailableValue + + # Iterate over imported functions + for imp in entry.imports: + import_name = imp.name + if import_name: + import_name = imp.name.decode() + else: + import_name = renderers.NotAvailableValue() + yield ( + 0, + ( + proc_id, + proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ), + dll_entry, + import_name, + ), + ) + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) + continue + + def run(self): + kernel = self.context.modules[self.config["kernel"]] + + return renderers.TreeGrid( + [("PID", int), ("Process", str), ("Library", str), ("Function", str)], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=pslist.PsList.create_pid_filter( + self.config.get("pid", None) + ), + ) + ), + ) From 059f2d012896d96fa0e2d20469d7771ec27b0b6d Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 16 Dec 2023 21:19:35 +0100 Subject: [PATCH 485/526] Adding function addr + bound info. Formatting code. --- volatility3/framework/plugins/windows/iat.py | 32 +++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 1cf51cfa2..11c273859 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,16 +1,13 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -import logging -import io -from typing import Callable, List +import logging, io, pefile from volatility3.framework.symbols import intermed from volatility3.framework import renderers, interfaces, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.plugins.windows import pslist -from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.windows.extensions import pe -import pefile vollog = logging.getLogger(__name__) @@ -86,6 +83,12 @@ class IAT(interfaces.plugins.PluginInterface): else: dll_entry = renderers.NotAvailableValue + bound = True + # Initially set to 0 if not bound + time_date_stamp = entry.struct.TimeDateStamp + if not time_date_stamp: + bound = False + # Iterate over imported functions for imp in entry.imports: import_name = imp.name @@ -93,6 +96,12 @@ class IAT(interfaces.plugins.PluginInterface): import_name = imp.name.decode() else: import_name = renderers.NotAvailableValue() + function_address = ( + pe_obj.OPTIONAL_HEADER.ImageBase + imp.address + ) + if not function_address: + function_address = renderers.NotAvailableValue + yield ( 0, ( @@ -103,7 +112,9 @@ class IAT(interfaces.plugins.PluginInterface): errors="replace", ), dll_entry, + bound, import_name, + format_hints.Hex(function_address), ), ) except exceptions.InvalidAddressException as excp: @@ -118,7 +129,14 @@ class IAT(interfaces.plugins.PluginInterface): kernel = self.context.modules[self.config["kernel"]] return renderers.TreeGrid( - [("PID", int), ("Process", str), ("Library", str), ("Function", str)], + [ + ("PID", int), + ("Name", str), + ("Library", str), + ("Bound", bool), + ("Function", str), + ("Address", format_hints.Hex), + ], self._generator( pslist.PsList.list_processes( context=self.context, From bd96357e33a796b82773df1927f03b1e80617d65 Mon Sep 17 00:00:00 2001 From: Valentin Obst Date: Thu, 14 Dec 2023 22:21:14 +0100 Subject: [PATCH 486/526] tolerate duplicate enum values, keep first name that was seen --- volatility3/framework/objects/__init__.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 3b1745718..831856e3d 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -593,17 +593,19 @@ class Enumeration(interfaces.objects.ObjectInterface, int): inverse_choices: Dict[int, str] = {} for k, v in choices.items(): if v in inverse_choices: - # Technically this shouldn't be a problem, but since we inverse cache - # and can't map one value to two possibilities we throw an exception during build - # We can remove/work around this if it proves a common issue - raise ValueError( - f"Enumeration value {v} duplicated as {k} and {inverse_choices[v]}" + vollog.log( + constants.LOGLEVEL_VVV, + f"Enumeration value {v} duplicated as {k}. Keeping name {inverse_choices[v]}", ) + continue inverse_choices[v] = k return inverse_choices def lookup(self, value: int = None) -> str: - """Looks up an individual value and returns the associated name.""" + """Looks up an individual value and returns the associated name. + + If multiple identifiers map to the same value, the first matching identifier will be returned + """ if value is None: return self.lookup(self) if value in self._inverse_choices: @@ -640,7 +642,10 @@ class Enumeration(interfaces.objects.ObjectInterface, int): @classmethod def lookup(cls, template: interfaces.objects.Template, value: int) -> str: - """Looks up an individual value and returns the associated name.""" + """Looks up an individual value and returns the associated name. + + If multiple identifiers map to the same value, the first matching identifier will be returned + """ _inverse_choices = Enumeration._generate_inverse_choices( template.vol["choices"] ) From aa624b0339d301def19893b42693ead63d7c440c Mon Sep 17 00:00:00 2001 From: Valentin Obst Date: Tue, 19 Dec 2023 10:25:26 +0100 Subject: [PATCH 487/526] remove kernel_cap_struct requirement --- volatility3/framework/symbols/linux/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 894be0dc6..c4e2587f4 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -29,7 +29,6 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) - self.set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) From 409ce95980b0e4c5a29222f4acb90b681886d16f Mon Sep 17 00:00:00 2001 From: hsarkey Date: Thu, 7 Dec 2023 14:04:44 -0500 Subject: [PATCH 488/526] Windows: Added '--refined' option to windows malfind plugin Also updated malfind to include "\x55\x48" and "\x55\x89" as part of the refined_criteria list. --- volatility3/framework/plugins/windows/malfind.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 6ed078996..1c73fdf1c 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -46,6 +46,12 @@ class Malfind(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) ), + requirements.BooleanRequirement( + name="refined", + description="Refine the output. Only show regions with an MZ header or that start with well known opcode combinations (i.e. PUSH EBP). WARNING: This can cause you to overlook regions with wiped headers or shell code blocks starting with NOP sleds, etc.. However, it will in general result in less noisy output.", + default=False, + optional=True, + ), ] @classmethod @@ -138,6 +144,9 @@ class Malfind(interfaces.plugins.PluginInterface): yield vad, data def _generator(self, procs): + # set refined criteria + refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] + # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] @@ -151,6 +160,10 @@ class Malfind(interfaces.plugins.PluginInterface): for vad, data in self.list_injections( self.context, kernel.layer_name, kernel.symbol_table_name, proc ): + # check if refined option was passed + if self.config["refined"] and data[0:2] not in refined_criteria: + continue + # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): architecture = "intel" From 3848fc69a6d93602b4836a87d1e69f493f7dbd49 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 25 Dec 2023 21:32:14 -0300 Subject: [PATCH 489/526] Add support for kernels earlier than version 3.11. * Add support for older ring buffer kernel implementation: - 3.5 < kernels - 3.5 <= kernels < 3.11 * Enabled support for wrapped-around ring buffers in all four kernel implementations. * Bugfix: wrapped around buffer issue with 3.11 <= kernel < 5.10. --- volatility3/framework/plugins/linux/kmsg.py | 187 ++++++++++++++------ 1 file changed, 132 insertions(+), 55 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 5136a00f6..e55d5f14e 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -1,6 +1,7 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import re import logging from abc import ABC, abstractmethod from enum import Enum @@ -9,7 +10,6 @@ from typing import Generator, Iterator, List, Tuple from volatility3.framework import ( class_subclasses, constants, - contexts, interfaces, renderers, ) @@ -143,7 +143,7 @@ class ABCKmsg(ABC): return "%lu.%06lu" % (nsec / 1000000000, (nsec % 1000000000) / 1000) def get_timestamp_in_sec_str(self, obj) -> str: - # obj could be printk_log or printk_info + # obj could be log, printk_log or printk_info return self.nsec_to_sec_str(obj.ts_nsec) def get_caller(self, obj): @@ -153,7 +153,7 @@ class ABCKmsg(ABC): if obj.has_member("caller_id"): return self.get_caller_text(obj.caller_id) else: - return "" + return renderers.NotAvailableValue() def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" @@ -161,7 +161,7 @@ class ABCKmsg(ABC): return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: - # obj could be printk_log or printk_info + # obj could be log, printk_log or printk_info return ( obj.facility, obj.level, @@ -186,39 +186,90 @@ class ABCKmsg(ABC): return str(facility) -class KmsgLegacy(ABCKmsg): - """Linux kernels prior to v5.10, the ringbuffer is initially kept in - __log_buf, and log_buf is a pointer to the former. __log_buf is declared as - a char array but it actually contains an array of printk_log structs. - The length of this array is defined in the kernel KConfig configuration via - the CONFIG_LOG_BUF_SHIFT value as a power of 2. - This can also be modified by the log_buf_len kernel boot parameter. - In SMP systems with more than 64 CPUs this ringbuffer size is dynamically - allocated according the number of CPUs based on the value of - CONFIG_LOG_CPU_MAX_BUF_SHIFT, and the log_buf pointer is updated - consequently to the new buffer. - In that case, the original static buffer in __log_buf is unused. +class Kmsg_pre_3_5(ABCKmsg): + """The kernel ring buffer (log_buf) is a char array that sequentially stores + log lines, each separated by newline (LF) characters. i.e: + <6>[ 9565.250411] line1!\n<6>[ 9565.250412] line2\n... """ @classmethod def symtab_checks(cls, vmlinux) -> bool: - return vmlinux.has_type("printk_log") + return ( + vmlinux.has_symbol("log_end") + and not vmlinux.has_symbol("log_first_idx") + and not ( + vmlinux.has_type("log") + and vmlinux.get_type("log").has_member("ts_nsec") + ) + ) - def get_text_from_printk_log(self, msg) -> str: - msg_offset = msg.vol.offset + self.vmlinux.get_type("printk_log").size + def run(self) -> Iterator[Tuple[str, str, str, str, str]]: + log_buf_ptr = self.vmlinux.object_from_symbol(symbol_name="log_buf") + log_buf_len = self.vmlinux.object_from_symbol(symbol_name="log_buf_len") + log_buf = utility.pointer_to_string(log_buf_ptr, count=log_buf_len) + log_end = self.vmlinux.object_from_symbol(symbol_name="log_end") + + if log_end > log_buf_len: + start = log_end - log_buf_len + first_half = log_buf[start:] + second_half = log_buf[:start] + log_buf = first_half + second_half + + log_buf_lines = log_buf.splitlines() + + for log_buf_line in log_buf_lines: + m = re.match(r"<(\d+)>\[\s*(\d+\.\d+)\]\s(.*?)$", log_buf_line) + if not m: + # If there was a wrap-around in the ring buffer, it will find + # remnants at the top. As those remnants do not conform to the + # expected line format, they are discarded + continue + + level_facility_str, timestamp_str, line = m.groups() + level_facility = int(level_facility_str) + # The lower 3 bit are the log level, the rest are the log facility + level = level_facility & 7 + facility = level_facility >> 3 + level_txt = self.get_level_text(level) + facility_txt = self.get_facility_text(facility) + caller = renderers.NotAvailableValue() + yield facility_txt, level_txt, timestamp_str, caller, line + + +class Kmsg_3_5_to_3_11(ABCKmsg): + """While 'log_buf' is declared as a pointer and '__log_buf' as a char array, + it essentially holds an array of 'log' structs. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return ( + vmlinux.has_type("log") + and vmlinux.get_type("log").has_member("ts_nsec") + and vmlinux.has_symbol("log_first_idx") + ) + + def _get_log_struct_name(self): + return "log" + + def get_text_from_log(self, msg) -> str: + log_struct_name = self._get_log_struct_name() + log_struct_size = self.vmlinux.get_type(log_struct_name).size + msg_offset = msg.vol.offset + log_struct_size return self.get_string(msg_offset, msg.text_len) def get_log_lines(self, msg) -> Generator[str, None, None]: if msg.text_len > 0: - text = self.get_text_from_printk_log(msg) + text = self.get_text_from_log(msg) yield from text.splitlines() def get_dict_lines(self, msg) -> Generator[str, None, None]: if msg.dict_len == 0: return None - dict_offset = ( - msg.vol.offset + self.vmlinux.get_type("printk_log").size + msg.text_len - ) + + log_struct_name = self._get_log_struct_name() + log_struct_size = self.vmlinux.get_type(log_struct_name).size + dict_offset = msg.vol.offset + log_struct_size + msg.text_len dict_data = self._context.layers[self.layer_name].read( dict_offset, msg.dict_len ) @@ -226,29 +277,41 @@ class KmsgLegacy(ABCKmsg): yield " " + chunk.decode() def run(self) -> Iterator[Tuple[str, str, str, str, str]]: - log_buf_ptr = self.vmlinux.object_from_symbol(symbol_name="log_buf") - if log_buf_ptr == 0: - # This is weird, let's fallback to check the static ringbuffer. - log_buf_ptr = self.vmlinux.object_from_symbol( - symbol_name="__log_buf" - ).vol.offset - if log_buf_ptr == 0: - raise ValueError("Log buffer is not available") + # First, the ring buffer size is determined in the kernel configuration + # by CONFIG_LOG_BUF_SHIFT. This static buffer is held in the '__log_buf' + # global variable, with 'log_buf' serving as a pointer to it. + # The user can also update this size using 'log_buf_len' in the + # kernel boot parameters. Additionally, in SMP systems with over 64 CPUs, + # the ring buffer size dynamically allocates based on the number of CPUs, + # following CONFIG_LOG_CPU_MAX_BUF_SHIFT. + # In the last two cases mentioned above, the 'log_buf' pointer is + # updated to this new buffer. The original static buffer in '__log_buf' + # remains unused. Therefore, it is crucial to read from 'log_buf' rather + # than '__log_buf'. + + log_buf_ptr = self.vmlinux.object_from_symbol("log_buf") + log_buf_len = self.vmlinux.object_from_symbol("log_buf_len") + + log_first_idx = int(self.vmlinux.object_from_symbol("log_first_idx")) + log_next_idx = int(self.vmlinux.object_from_symbol("log_next_idx")) + + log_struct_name = self._get_log_struct_name() - log_first_idx = int( - self.vmlinux.object_from_symbol(symbol_name="log_first_idx") - ) cur_idx = log_first_idx - end_idx = None # We don't need log_next_idx here. See below msg.len == 0 - while cur_idx != end_idx: - end_idx = log_first_idx + if log_first_idx < log_next_idx: + end_idx = log_next_idx + else: + end_idx = log_buf_len + + while cur_idx < end_idx: msg_offset = log_buf_ptr + cur_idx # type: ignore - msg = self.vmlinux.object(object_type="printk_log", offset=msg_offset) + msg = self.vmlinux.object(object_type=log_struct_name, offset=msg_offset) if msg.len == 0: - # As per kernel/printk/printk.c: + # As per kernel/printk.c: # A length == 0 for the next message indicates a wrap-around to # the beginning of the buffer. cur_idx = 0 + end_idx = log_next_idx else: facility, level, timestamp, caller = self.get_prefix(msg) level_txt = self.get_level_text(level) @@ -262,39 +325,53 @@ class KmsgLegacy(ABCKmsg): cur_idx += msg.len -class KmsgFiveTen(ABCKmsg): - """In 5.10 the kernel ringbuffer implementation changed. +class Kmsg_3_11_to_5_10(Kmsg_3_5_to_3_11): + """Starting from version 3.11, the struct 'log' was renamed to 'printk_log'. + While 'log_buf' is declared as a pointer and '__log_buf' as a char array, + it essentially holds an array of 'printk_log' structs. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return vmlinux.has_type("printk_log") + + def _get_log_struct_name(self): + return "printk_log" + + +class Kmsg_5_10_to_(ABCKmsg): + """In 5.10 the kernel ring buffer implementation changed. Previously only one process should read /proc/kmsg and it is permanently open and periodically read by the syslog daemon. A high level structure 'printk_ringbuffer' was added to represent the printk - ringbuffer which actually contains two ringbuffers. The descriptor ring + ring buffer which actually contains two ring buffers. The descriptor ring 'desc_ring' contains the records' metadata, text offsets and states. The data block ring 'text_data_ring' contains the records' text strings. A pointer to the high level structure is kept in the prb pointer which is - initialized to a static ringbuffer. + initialized to a static ring buffer. .. code-block:: c static struct printk_ringbuffer *prb = &printk_rb_static; - In SMP systems with more than 64 CPUs this ringbuffer size is dynamically + In SMP systems with more than 64 CPUs this ring buffer size is dynamically allocated according the number of CPUs based on the value of CONFIG_LOG_CPU_MAX_BUF_SHIFT. The prb pointer is updated consequently to - this dynamic ringbuffer in setup_log_buf(). + this dynamic ring buffer in setup_log_buf(). .. code-block:: c prb = &printk_rb_dynamic; - Behind scenes, log_buf is still used as external buffer. - When the static printk_ringbuffer struct is initialized, _DEFINE_PRINTKRB - sets text_data_ring.data pointer to the address in log_buf which points to - the static buffer __log_buff. - If a dynamic ringbuffer takes place, setup_log_buf() sets - text_data_ring.data of printk_rb_dynamic to the new allocated external - buffer via the prb_init function. - In that case, the original external static buffer in __log_buf and - printk_rb_static are unused. + Behind scenes, 'log_buf' is still used as external buffer. + When the static 'printk_ringbuffer' struct is initialized, _DEFINE_PRINTKRB + sets text_data_ring.data pointer to the address in 'log_buf' which points + to the static buffer '__log_buf'. + If a dynamic ring buffer takes place, setup_log_buf() sets + text_data_ring.data of 'printk_rb_dynamic' to the new allocated external + buffer via the 'prb_init' function. + In that case, the original external static buffer in '__log_buf' and + 'printk_rb_static' are unused. .. code-block:: c @@ -352,7 +429,7 @@ class KmsgFiveTen(ABCKmsg): def run(self) -> Iterator[Tuple[str, str, str, str, str]]: # static struct printk_ringbuffer *prb = &printk_rb_static; - ringbuffers = self.vmlinux.object_from_symbol(symbol_name="prb").dereference() + ringbuffers = self.vmlinux.object_from_symbol("prb").dereference() desc_ring = ringbuffers.desc_ring text_data_ring = ringbuffers.text_data_ring From 0d934d4991818cdec0102759e78b324fac509f79 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 26 Dec 2023 16:13:47 -0300 Subject: [PATCH 490/526] Fix error and comment --- volatility3/framework/plugins/linux/kmsg.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index e55d5f14e..2b3d70ccb 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -102,14 +102,13 @@ class ABCKmsg(ABC): subclass.__name__, ) kmsg_inst = subclass(context=context, config=config) - # More than one class could be executed for an specific kernel - # version i.e. Netfilter Ingress hooks - # We expect just one implementation to be executed for an specific kernel yield from kmsg_inst.run() + # So far, it allows only one implementation to be executed for each + # specific kernel. break if kmsg_inst is None: - vollog.error("Unsupported Netfilter kernel implementation") + vollog.error("Unsupported kernel ring buffer implementation") @abstractmethod def run(self) -> Iterator[Tuple[str, str, str, str, str]]: From e4c44698f801967bb7dd41dc87cfb9a501ad2924 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 2 Jan 2024 11:14:43 -0300 Subject: [PATCH 491/526] Reduce imports --- volatility3/framework/plugins/linux/kmsg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 2b3d70ccb..70d80835b 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -14,7 +14,6 @@ from volatility3.framework import ( renderers, ) from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility vollog = logging.getLogger(__name__) @@ -483,7 +482,7 @@ class Kmsg_5_10_to_(ABCKmsg): cur_id &= desc_id_mask -class Kmsg(plugins.PluginInterface): +class Kmsg(interfaces.plugins.PluginInterface): """Kernel log buffer reader""" _required_framework_version = (2, 0, 0) From 02a389a61b460c6ffa3b2cae177cad1f48f23234 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 2 Jan 2024 11:17:18 -0300 Subject: [PATCH 492/526] Improve docstrings --- volatility3/framework/plugins/linux/kmsg.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 70d80835b..d17e0d92d 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -81,7 +81,7 @@ class ABCKmsg(ABC): config: Core configuration Yields: - kmsg records + The kmsg records. Same as run() """ vmlinux = context.modules[config["kernel"]] @@ -102,7 +102,7 @@ class ABCKmsg(ABC): ) kmsg_inst = subclass(context=context, config=config) yield from kmsg_inst.run() - # So far, it allows only one implementation to be executed for each + # So far, it only allows a single implementation to be executed for each # specific kernel. break @@ -111,7 +111,16 @@ class ABCKmsg(ABC): @abstractmethod def run(self) -> Iterator[Tuple[str, str, str, str, str]]: - """Walks through the specific kernel implementation.""" + """Walks through the specific kernel implementation. + + Returns: + tuple: + facility [str]: The log facility: kern, user, etc. see FACILITIES + level [str]: The log level: info, debug, etc. see LEVELS + timestamp [str]: The message timestamp. See nsec_to_sec_str() + caller [str]: The Caller ID: CPU(1) or Task(1234). See get_caller() + line [str]: The log message. + """ @classmethod @abstractmethod @@ -121,7 +130,8 @@ class ABCKmsg(ABC): The first class returning True will be instantiated and called via the run() method. - :return: True is the kernel being analysed fulfill the class requirements. + Returns: + bool: True if the kernel being analysed fulfill the class requirements. """ def get_string(self, addr: int, length: int) -> str: From 8e58815c114e23b28acfe1ec188c703ee555fb1a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 2 Jan 2024 11:25:35 -0300 Subject: [PATCH 493/526] Bump plugin's patch version --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d17e0d92d..919d9145b 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -497,7 +497,7 @@ class Kmsg(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 66b408d6e848a1e566f1b17787298be923b8a783 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 2 Jan 2024 11:37:10 -0300 Subject: [PATCH 494/526] Fix minor docstring typos --- volatility3/framework/plugins/linux/kmsg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 919d9145b..d1f17bf94 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -115,10 +115,10 @@ class ABCKmsg(ABC): Returns: tuple: - facility [str]: The log facility: kern, user, etc. see FACILITIES - level [str]: The log level: info, debug, etc. see LEVELS + facility [str]: The log facility: kern, user, etc. See FACILITIES + level [str]: The log level: info, debug, etc. See LEVELS timestamp [str]: The message timestamp. See nsec_to_sec_str() - caller [str]: The Caller ID: CPU(1) or Task(1234). See get_caller() + caller [str]: The caller ID: CPU(1) or Task(1234). See get_caller() line [str]: The log message. """ @@ -131,7 +131,7 @@ class ABCKmsg(ABC): run() method. Returns: - bool: True if the kernel being analysed fulfill the class requirements. + bool: True if the kernel being analyzed fulfill the class requirements. """ def get_string(self, addr: int, length: int) -> str: From 52dcfb45f45b9442b944f124f66cb0c363b584af Mon Sep 17 00:00:00 2001 From: bbarnacle Date: Tue, 28 Nov 2023 11:20:51 -0500 Subject: [PATCH 495/526] Windows: Add regex filtering to dumpfiles In volatility2, the windows.dumpfiles plugin allows you to filter the dumped files using a regular expression. This PR adds the same functionality to volatility3. The regular expression is passed in using --regex=REGEX and all files matching REGEX will be dumped. The --ignore-case flag can be passed to make the search case-insensitive. The search is case-sensitive by default. The matching volatility2 functionality can be found here: https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee 9338b881c0fa25937/volatility/plugins/dumpfiles.py#L844 Manual testing was performed on windows memory images across different windows versions to verify the expected output. --- .../framework/plugins/windows/dumpfiles.py | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index dd82d897e..4aef660da 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -4,11 +4,12 @@ import logging import ntpath +import re from typing import List, Tuple, Type, Optional, Generator from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints +from volatility3.framework.renderers import format_hints, UnreadableValue from volatility3.plugins.windows import handles from volatility3.plugins.windows import pslist @@ -53,6 +54,15 @@ class DumpFiles(interfaces.plugins.PluginInterface): description="Dump a single _FILE_OBJECT at this physical address", optional=True, ), + requirements.StringRequirement( + name="regex", description="Dump files matching REGEX", optional=True + ), + requirements.BooleanRequirement( + name="ignore-case", + description="Ignore case in pattern match", + default=False, + optional=True, + ), requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), @@ -208,6 +218,11 @@ class DumpFiles(interfaces.plugins.PluginInterface): def _generator(self, procs: List, offsets: List): kernel = self.context.modules[self.config["kernel"]] + if self.config["regex"]: + if self.config["ignore-case"]: + file_re = re.compile(self.config["regex"], re.I) + else: + file_re = re.compile(self.config["regex"]) if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing @@ -243,6 +258,14 @@ class DumpFiles(interfaces.plugins.PluginInterface): obj_type = entry.get_object_type(type_map, cookie) if obj_type == "File": file_obj = entry.Body.cast("_FILE_OBJECT") + + if self.config["regex"]: + name = file_obj.file_name_with_device() + if isinstance(name, UnreadableValue): + continue + if not file_re.search(name): + continue + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): @@ -272,6 +295,13 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue + if self.config["regex"]: + name = file_obj.file_name_with_device() + if isinstance(name, UnreadableValue): + continue + if not file_re.search(name): + continue + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): From 614d4d6f2b9eac50992c04cd08e7389be19ede99 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jan 2024 20:35:54 +0000 Subject: [PATCH 496/526] Layers: Fix cloudstorage unnecessary import --- volatility3/framework/layers/cloudstorage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/layers/cloudstorage.py b/volatility3/framework/layers/cloudstorage.py index 3f88ef34f..97ed54231 100644 --- a/volatility3/framework/layers/cloudstorage.py +++ b/volatility3/framework/layers/cloudstorage.py @@ -20,7 +20,6 @@ try: except ImportError: HAS_GCSFS = False -from volatility3.framework import exceptions from volatility3.framework.layers import resources vollog = logging.getLogger(__file__) From 01ffcd9634af3d6ef90c130f1dfd1a8910554d4d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jan 2024 20:37:26 +0000 Subject: [PATCH 497/526] Core: Fixing None equality test in MapleTree implementation --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d8a2867cc..d73d0cfb9 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -362,7 +362,7 @@ class maple_tree(objects.StructType): # None. If however you wanted to parse from a node, but ignore some parts of the tree below it then # this could be populated with the addresses of the nodes you wish to ignore. - if seen == None: + if seen is None: seen = set() # protect against unlikely loop From 4b86b9ea89b5610c581a574045309b93ea8f3ef5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jan 2024 20:40:27 +0000 Subject: [PATCH 498/526] Plugins: Remove unnecessary variable from windows.mftscan --- volatility3/framework/plugins/windows/mftscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 7e4e1ca18..4298b4e4e 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -53,7 +53,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" - header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" From 4e89040dadfacac01ba0c0a3727969a7b0fb0047 Mon Sep 17 00:00:00 2001 From: hsarkey Date: Sun, 14 Jan 2024 13:20:56 -0500 Subject: [PATCH 499/526] Updated changes to the windows.malfind plugin. Eliminated --refined as a command line option and instead added an additional column called "Notes" to provide info on common headers. --- .../framework/plugins/windows/malfind.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 1c73fdf1c..a14f8889d 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -46,12 +46,6 @@ class Malfind(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) ), - requirements.BooleanRequirement( - name="refined", - description="Refine the output. Only show regions with an MZ header or that start with well known opcode combinations (i.e. PUSH EBP). WARNING: This can cause you to overlook regions with wiped headers or shell code blocks starting with NOP sleds, etc.. However, it will in general result in less noisy output.", - default=False, - optional=True, - ), ] @classmethod @@ -144,9 +138,6 @@ class Malfind(interfaces.plugins.PluginInterface): yield vad, data def _generator(self, procs): - # set refined criteria - refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] - # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] @@ -155,14 +146,21 @@ class Malfind(interfaces.plugins.PluginInterface): ) for proc in procs: + # by default, "Notes" column will be set to none + notes = "None" process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( self.context, kernel.layer_name, kernel.symbol_table_name, proc ): - # check if refined option was passed - if self.config["refined"] and data[0:2] not in refined_criteria: - continue + # Check for unique headers and update "Notes" column if criteria is met + match data[0:2]: + case b"MZ" | b"\x55\x8B": + notes = "MZ header" + case b"\x55\x8B": + notes = "PE header" + case b"\x55\x48" | b"\x55\x89": + notes = "Function prologue" # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): @@ -209,6 +207,7 @@ class Malfind(interfaces.plugins.PluginInterface): vad.get_commit_charge(), vad.get_private_memory(), file_output, + notes, format_hints.HexBytes(data), disasm, ), @@ -229,6 +228,7 @@ class Malfind(interfaces.plugins.PluginInterface): ("CommitCharge", int), ("PrivateMemory", int), ("File output", str), + ("Notes", str), ("Hexdump", format_hints.HexBytes), ("Disasm", interfaces.renderers.Disassembly), ], From e6e138f12d06509da829008867392bc2346f18f9 Mon Sep 17 00:00:00 2001 From: hsarkey Date: Wed, 17 Jan 2024 17:24:26 -0500 Subject: [PATCH 500/526] Updated windows.malfind to have a "Notes" column which will indicate if a process meets a "refined criteria", meaning it has a common header type (MZ, PE, or a function prologue). --- volatility3/framework/plugins/windows/malfind.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index a14f8889d..e5a842611 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -140,6 +140,9 @@ class Malfind(interfaces.plugins.PluginInterface): def _generator(self, procs): # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] + + # set refined criteria to know when to add to "Notes" column + refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] is_32bit_arch = not symbols.symbol_table_is_64bit( self.context, kernel.symbol_table_name @@ -154,12 +157,12 @@ class Malfind(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, proc ): # Check for unique headers and update "Notes" column if criteria is met - match data[0:2]: - case b"MZ" | b"\x55\x8B": + if data[0:2] in refined_criteria: + if data[0:2] == b"MZ": notes = "MZ header" - case b"\x55\x8B": + elif data[0:2] == b"\x55\x8B": notes = "PE header" - case b"\x55\x48" | b"\x55\x89": + else: notes = "Function prologue" # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 From ef36cb5a6c6a30f0bc986cc275c83a734b9d535b Mon Sep 17 00:00:00 2001 From: hsarkey Date: Wed, 17 Jan 2024 17:51:27 -0500 Subject: [PATCH 501/526] Updated windows.malfind to have a "Notes" column to indicate if a process meets "refined" criteria, which includes common headers like MZ,PE or function prologues. Fixed black formatting issue. --- volatility3/framework/plugins/windows/malfind.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index e5a842611..284144077 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -140,7 +140,7 @@ class Malfind(interfaces.plugins.PluginInterface): def _generator(self, procs): # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] - + # set refined criteria to know when to add to "Notes" column refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] @@ -150,7 +150,7 @@ class Malfind(interfaces.plugins.PluginInterface): for proc in procs: # by default, "Notes" column will be set to none - notes = "None" + notes = "None" process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( From a50ebb6014ff11ea165454e0a08b967d01cce9e8 Mon Sep 17 00:00:00 2001 From: Calvin Kusek Date: Tue, 2 Jan 2024 12:00:41 -0500 Subject: [PATCH 502/526] Windows: Display additional process info for windows.pstree --- .../framework/plugins/windows/pstree.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/pstree.py b/volatility3/framework/plugins/windows/pstree.py index a39fe7485..2be96277c 100644 --- a/volatility3/framework/plugins/windows/pstree.py +++ b/volatility3/framework/plugins/windows/pstree.py @@ -5,7 +5,7 @@ import datetime import logging from typing import Callable, Dict, Set, Tuple -from volatility3.framework import objects, interfaces, renderers +from volatility3.framework import objects, interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist @@ -132,6 +132,25 @@ class PsTree(interfaces.plugins.PluginInterface): proc.get_exit_time(), ) + try: + audit = proc.SeAuditProcessCreationInfo.ImageFileName.Name + # If 'audit' is set to the empty string, display NotAvailableValue + row += (audit.get_string() or renderers.NotAvailableValue(),) + except exceptions.InvalidAddressException: + row += (renderers.NotAvailableValue(),) + + try: + process_params = proc.get_peb().ProcessParameters + row += ( + process_params.CommandLine.get_string(), + process_params.ImagePathName.get_string(), + ) + except exceptions.InvalidAddressException: + row += ( + renderers.NotAvailableValue(), + renderers.NotAvailableValue(), + ) + yield (self._levels[pid] - 1, row) for child_pid in self._children.get(pid, []): yield from yield_processes( @@ -161,6 +180,9 @@ class PsTree(interfaces.plugins.PluginInterface): ("Wow64", bool), ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime), + ("Audit", str), + ("Cmd", str), + ("Path", str), ], self._generator( filter_func=pslist.PsList.create_pid_filter( From ae5375a622c82177f6b29adfb8b0f410ecb03059 Mon Sep 17 00:00:00 2001 From: Brandon Barnacle Date: Wed, 17 Jan 2024 09:49:39 -0500 Subject: [PATCH 503/526] PR comment changes Change the --regex flag to --filter. Add a check so that --filter cannot be used with --physaddr or --virtaddr. Change self.config["filter"] check to check if file_re has been set. --- .../framework/plugins/windows/dumpfiles.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 4aef660da..8865007d8 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -55,11 +55,11 @@ class DumpFiles(interfaces.plugins.PluginInterface): optional=True, ), requirements.StringRequirement( - name="regex", description="Dump files matching REGEX", optional=True + name="filter", description="Dump files matching regular expression FILTER", optional=True ), requirements.BooleanRequirement( name="ignore-case", - description="Ignore case in pattern match", + description="Ignore case in filter match", default=False, optional=True, ), @@ -218,11 +218,11 @@ class DumpFiles(interfaces.plugins.PluginInterface): def _generator(self, procs: List, offsets: List): kernel = self.context.modules[self.config["kernel"]] - if self.config["regex"]: - if self.config["ignore-case"]: - file_re = re.compile(self.config["regex"], re.I) - else: - file_re = re.compile(self.config["regex"]) + file_re = None + if self.config["filter"]: + flags = re.I if self.config["ignore-case"] else 0 + file_re = re.compile(self.config["filter"], flags) + if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing @@ -259,7 +259,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if obj_type == "File": file_obj = entry.Body.cast("_FILE_OBJECT") - if self.config["regex"]: + if file_re: name = file_obj.file_name_with_device() if isinstance(name, UnreadableValue): continue @@ -295,7 +295,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue - if self.config["regex"]: + if file_re: name = file_obj.file_name_with_device() if isinstance(name, UnreadableValue): continue @@ -345,6 +345,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): procs = list() kernel = self.context.modules[self.config["kernel"]] + if self.config["filter"] and (self.config["virtaddr"] or self.config["physaddr"]): + raise ValueError("Cannot use filter flag with an address flag") + if self.config.get("virtaddr", None) is not None: offsets.append((self.config["virtaddr"], True)) elif self.config.get("physaddr", None) is not None: From 4dc6f637b055f912bafaa9f6079759539277d60f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jan 2024 23:50:45 +0000 Subject: [PATCH 504/526] Core: Apply black 24.1.0 to the whole codebase --- volatility3/cli/text_renderer.py | 8 +++--- volatility3/framework/__init__.py | 6 ++--- volatility3/framework/automagic/mac.py | 6 ++--- volatility3/framework/automagic/module.py | 6 ++--- .../framework/automagic/symbol_finder.py | 12 ++++----- volatility3/framework/automagic/windows.py | 20 +++++++------- .../framework/configuration/requirements.py | 6 ++--- volatility3/framework/layers/avml.py | 6 ++--- volatility3/framework/layers/elf.py | 6 ++--- volatility3/framework/layers/intel.py | 6 ++--- volatility3/framework/layers/lime.py | 6 ++--- volatility3/framework/layers/qemu.py | 6 ++--- volatility3/framework/layers/xen.py | 6 ++--- volatility3/framework/objects/__init__.py | 6 ++--- volatility3/framework/plugins/mac/pslist.py | 4 +-- .../framework/plugins/windows/crashinfo.py | 6 ++--- .../framework/plugins/windows/mftscan.py | 1 - .../framework/plugins/windows/netscan.py | 10 ++++--- .../plugins/windows/registry/printkey.py | 6 ++--- volatility3/framework/renderers/conversion.py | 6 ++--- volatility3/framework/symbols/__init__.py | 26 +++++++++---------- .../symbols/windows/extensions/__init__.py | 6 ++--- volatility3/plugins/windows/statistics.py | 10 +++---- 23 files changed, 88 insertions(+), 93 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ffb8d516b..6e58ee68d 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -389,9 +389,11 @@ class JsonRenderer(CLIRenderer): interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), - datetime.datetime: lambda x: x.isoformat() - if not isinstance(x, interfaces.renderers.BaseAbsentValue) - else None, + datetime.datetime: lambda x: ( + x.isoformat() + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else None + ), "default": lambda x: x, } diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 9c17846a8..1565b2267 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -206,9 +206,9 @@ def _zipwalk(path: str): if not file.is_dir(): dirlist = zip_results.get(os.path.dirname(file.filename), []) dirlist.append(os.path.basename(file.filename)) - zip_results[ - os.path.join(path, os.path.dirname(file.filename)) - ] = dirlist + zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( + dirlist + ) for value in zip_results: yield value, zip_results[value] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index aa75fbc3d..e51753139 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -138,9 +138,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): config_path = join("automagic", "MacIntelHelper", new_layer_name) context.config[join(config_path, "memory_layer")] = layer_name context.config[join(config_path, "page_map_offset")] = dtb - context.config[ - join(config_path, MacSymbolFinder.banner_config_key) - ] = str(banner, "latin-1") + context.config[join(config_path, MacSymbolFinder.banner_config_key)] = ( + str(banner, "latin-1") + ) new_layer = intel.Intel32e( context, diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index ee56a040c..ff13db905 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -34,9 +34,9 @@ class KernelModule(interfaces.automagic.AutomagicInterface): return None # The requirement is unfulfilled and is a ModuleRequirement - context.config[ - interfaces.configuration.path_join(new_config_path, "class") - ] = "volatility3.framework.contexts.Module" + context.config[interfaces.configuration.path_join(new_config_path, "class")] = ( + "volatility3.framework.contexts.Module" + ) for req in requirement.requirements: if ( diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index bf1c8ff16..21e594549 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -150,12 +150,12 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join - context.config[ - path_join(config_path, requirement.name, "class") - ] = clazz - context.config[ - path_join(config_path, requirement.name, "isf_url") - ] = isf_path + context.config[path_join(config_path, requirement.name, "class")] = ( + clazz + ) + context.config[path_join(config_path, requirement.name, "isf_url")] = ( + isf_path + ) context.config[ path_join(config_path, requirement.name, "symbol_mask") ] = layer.address_mask diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index a8530829b..52296f5ad 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -402,19 +402,19 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): if swap_location: context.config[current_layer_path] = current_layer_name try: - context.config[ - layer_loc_path - ] = requirements.URIRequirement.location_from_file( - swap_location + context.config[layer_loc_path] = ( + requirements.URIRequirement.location_from_file( + swap_location + ) ) except ValueError: vollog.warning( f"Volatility swap_location {swap_location} could not be validated - swap layer disabled" ) continue - context.config[ - layer_class_path - ] = "volatility3.framework.layers.physical.FileLayer" + context.config[layer_class_path] = ( + "volatility3.framework.layers.physical.FileLayer" + ) # Add the requirement new_req = requirements.TranslationLayerRequirement( @@ -424,9 +424,9 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): ) swap_req.add_requirement(new_req) - context.config[ - path_join(swap_sub_config, "number_of_elements") - ] = counter + context.config[path_join(swap_sub_config, "number_of_elements")] = ( + counter + ) context.config[swap_sub_config] = True swap_req.construct(context, swap_config) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index abdffdbe4..1c0622574 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -550,9 +550,9 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} - context.config[ - interfaces.configuration.path_join(config_path, self.name) - ] = True + context.config[interfaces.configuration.path_join(config_path, self.name)] = ( + True + ) return {} @classmethod diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index c825464cc..2e5572192 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -224,7 +224,7 @@ class AVMLStacker(interfaces.automagic.StackerLayerInterface): except exceptions.LayerException: return None new_name = context.layers.free_layer_name("AVMLLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return AVMLLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index a10d36592..b2fd6d4d1 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -115,9 +115,9 @@ class Elf64Stacker(interfaces.automagic.StackerLayerInterface): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("Elf64Layer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) try: return Elf64Layer(context, new_name, new_name) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 7d3b86a12..ae477854d 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -277,9 +277,9 @@ class Intel(linear.LinearlyMappedLayer): This allows translation layers to provide maps of contiguous regions in one layer """ - stashed_offset = ( - stashed_mapped_offset - ) = stashed_size = stashed_mapped_size = stashed_map_layer = None + stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = ( + stashed_map_layer + ) = None for offset, size, mapped_offset, mapped_size, map_layer in self._mapping( offset, length, ignore_errors ): diff --git a/volatility3/framework/layers/lime.py b/volatility3/framework/layers/lime.py index 28d646640..8b93932ab 100644 --- a/volatility3/framework/layers/lime.py +++ b/volatility3/framework/layers/lime.py @@ -104,7 +104,7 @@ class LimeStacker(interfaces.automagic.StackerLayerInterface): except LimeFormatException: return None new_name = context.layers.free_layer_name("LimeLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return LimeLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 501b8655e..ff483291c 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -486,9 +486,9 @@ class QemuStacker(interfaces.automagic.StackerLayerInterface): except exceptions.LayerException: return None new_name = context.layers.free_layer_name("QemuSuspendLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) layer = QemuSuspendLayer(context, new_name, new_name) cls.stacker_slow_warning() return layer diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index f7881a091..927b30430 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -173,8 +173,8 @@ class XenCoreDumpStacker(elf.Elf64Stacker): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("XenCoreDumpLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return XenCoreDumpLayer(context, new_name, new_name) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 831856e3d..316a30bec 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -768,12 +768,10 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): raise IndexError(f"Member not present in array template: {child}") @overload - def __getitem__(self, i: int) -> interfaces.objects.Template: - ... + def __getitem__(self, i: int) -> interfaces.objects.Template: ... @overload - def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: - ... + def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: ... def __getitem__(self, i): """Returns the i-th item from the array.""" diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 88045a277..9b570f3f9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -49,9 +49,7 @@ class PsList(interfaces.plugins.PluginInterface): ] @classmethod - def get_list_tasks( - cls, method: str - ) -> Callable[ + def get_list_tasks(cls, method: str) -> Callable[ [interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface], ]: diff --git a/volatility3/framework/plugins/windows/crashinfo.py b/volatility3/framework/plugins/windows/crashinfo.py index 4ecd85087..862eb6080 100644 --- a/volatility3/framework/plugins/windows/crashinfo.py +++ b/volatility3/framework/plugins/windows/crashinfo.py @@ -46,9 +46,9 @@ class Crashinfo(interfaces.plugins.PluginInterface): bitmap_size = format_hints.Hex(summary_header.BitmapSize) bitmap_pages = format_hints.Hex(summary_header.Pages) else: - bitmap_header_size = ( - bitmap_size - ) = bitmap_pages = renderers.NotApplicableValue() + bitmap_header_size = bitmap_size = bitmap_pages = ( + renderers.NotApplicableValue() + ) yield ( 0, diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 4298b4e4e..91a2e9152 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -175,7 +175,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): class ADS(interfaces.plugins.PluginInterface): - """Scans for Alternate Data Stream""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index d0bbd5cbd..62ead3ab7 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -487,10 +487,12 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if not isinstance(row_data[9], datetime.datetime): continue row_data = [ - "N/A" - if isinstance(i, renderers.UnreadableValue) - or isinstance(i, renderers.UnparsableValue) - else i + ( + "N/A" + if isinstance(i, renderers.UnreadableValue) + or isinstance(i, renderers.UnparsableValue) + else i + ) for i in row_data ] description = ( diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index e248c19bc..180f8f9d9 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -193,9 +193,9 @@ class PrintKey(interfaces.plugins.PluginInterface): vollog.debug( "Couldn't read registry value type, so data is unreadable" ) - value_data: Union[ - interfaces.renderers.BaseAbsentValue, bytes - ] = renderers.UnreadableValue() + value_data: Union[interfaces.renderers.BaseAbsentValue, bytes] = ( + renderers.UnreadableValue() + ) else: try: value_data = node.decode_data() diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index bf7da9ecb..bb18fcc8a 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -28,9 +28,9 @@ def wintime_to_datetime( def unixtime_to_datetime( unixtime: int, ) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: - ret: Union[ - interfaces.renderers.BaseAbsentValue, datetime.datetime - ] = renderers.UnparsableValue() + ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = ( + renderers.UnparsableValue() + ) if unixtime > 0: with contextlib.suppress(ValueError): diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index 10cf39cf1..d1e7a104d 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -35,9 +35,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): def __init__(self) -> None: super().__init__() - self._dict: Dict[ - str, interfaces.symbols.BaseSymbolTableInterface - ] = collections.OrderedDict() + self._dict: Dict[str, interfaces.symbols.BaseSymbolTableInterface] = ( + collections.OrderedDict() + ) # Permanently cache all resolved symbols self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} @@ -73,9 +73,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self, offset: int, size: int = 0, table_name: str = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" - table_list: Iterable[ - interfaces.symbols.BaseSymbolTableInterface - ] = self._dict.values() + table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( + self._dict.values() + ) if table_name is not None: if table_name in self._dict: table_list = [self._dict[table_name]] @@ -179,15 +179,15 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): if child.vol.type_name not in self._resolved: traverse_list.append(child.vol.type_name) try: - self._resolved[ - child.vol.type_name - ] = self._weak_resolve( - SymbolType.TYPE, child.vol.type_name + self._resolved[child.vol.type_name] = ( + self._weak_resolve( + SymbolType.TYPE, child.vol.type_name + ) ) except exceptions.SymbolError: - self._resolved[ - child.vol.type_name - ] = self.UnresolvedTemplate(child.vol.type_name) + self._resolved[child.vol.type_name] = ( + self.UnresolvedTemplate(child.vol.type_name) + ) # Stash the replacement replacements.add((traverser, child)) elif child.children: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d435851d7..846e5bd90 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -452,9 +452,9 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[ - str, interfaces.renderers.BaseAbsentValue - ] = renderers.UnreadableValue() + name: Union[str, interfaces.renderers.BaseAbsentValue] = ( + renderers.UnreadableValue() + ) # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 9915312e3..7f56b75f8 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -31,13 +31,9 @@ class Statistics(plugins.PluginInterface): # Do mass mapping and determine the number of different layers and how many pages go to each one layer = self.context.layers[self.config["primary"]] - page_count = ( - swap_count - ) = ( - invalid_page_count - ) = ( - large_page_count - ) = large_swap_count = large_invalid_count = other_invalid = 0 + page_count = swap_count = invalid_page_count = large_page_count = ( + large_swap_count + ) = large_invalid_count = other_invalid = 0 if isinstance(layer, intel.Intel): page_addr = 0 From 497d291ef4393e2580052a3dfddbef10e4dc2338 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 29 Jan 2024 00:07:40 +0000 Subject: [PATCH 505/526] Windows: Fix black on dumpfiles --- volatility3/framework/plugins/windows/dumpfiles.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 8865007d8..48539c752 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -55,7 +55,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): optional=True, ), requirements.StringRequirement( - name="filter", description="Dump files matching regular expression FILTER", optional=True + name="filter", + description="Dump files matching regular expression FILTER", + optional=True, ), requirements.BooleanRequirement( name="ignore-case", @@ -223,7 +225,6 @@ class DumpFiles(interfaces.plugins.PluginInterface): flags = re.I if self.config["ignore-case"] else 0 file_re = re.compile(self.config["filter"], flags) - if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing # private variables, so we need an instance (for now, anyway). We _could_ call Handles._generator() @@ -345,7 +346,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): procs = list() kernel = self.context.modules[self.config["kernel"]] - if self.config["filter"] and (self.config["virtaddr"] or self.config["physaddr"]): + if self.config["filter"] and ( + self.config["virtaddr"] or self.config["physaddr"] + ): raise ValueError("Cannot use filter flag with an address flag") if self.config.get("virtaddr", None) is not None: From 837d3b7b9fce65d5ef6543b6db9ecbad5092d4b0 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 14 Dec 2023 13:47:00 -0600 Subject: [PATCH 506/526] Windows: Add parsing of service binary/dll Parses the binary or dll associated with each service from the Windows SYSTEM hive and includes it in the output for each service. --- .../framework/plugins/windows/svcscan.py | 139 +++++++++++++++++- 1 file changed, 133 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 60562915e..720f745d8 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -4,20 +4,30 @@ import logging import os -from typing import List +from typing import Dict, List, NamedTuple, Union -from volatility3.framework import interfaces, renderers, constants, symbols, exceptions +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.framework.symbols.windows.extensions import services -from volatility3.plugins.windows import poolscanner, vadyarascan, pslist +from volatility3.plugins.windows import poolscanner, pslist, vadyarascan +from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) +ServiceBinaryInfo = NamedTuple( + "ServiceBinaryInfo", + [ + ("dll", Union[str, interfaces.renderers.BaseAbsentValue]), + ("binary", Union[str, interfaces.renderers.BaseAbsentValue]), + ], +) + + class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" @@ -42,10 +52,16 @@ class SvcScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), ] @staticmethod - def get_record_tuple(service_record: interfaces.objects.ObjectInterface): + def get_record_tuple( + service_record: interfaces.objects.ObjectInterface, + binary_info: ServiceBinaryInfo, + ): return ( format_hints.Hex(service_record.vol.offset), service_record.Order, @@ -56,6 +72,8 @@ class SvcScan(interfaces.plugins.PluginInterface): service_record.get_name(), service_record.get_display(), service_record.get_binary(), + binary_info.binary, + binary_info.dll, ) @staticmethod @@ -150,6 +168,86 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types=native_types, ) + def _get_service_key(self, kernel): + for hive in hivelist.HiveList.list_hives( + context=self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_string="machine\\system", + hive_offsets=None, + ): + # Get ControlSet\Services. + try: + return hive.get_key(r"CurrentControlSet\Services") + except (KeyError, exceptions.InvalidAddressException): + try: + return hive.get_key(r"ControlSet001\Services") + except (KeyError, exceptions.InvalidAddressException): + pass + + return None + + @staticmethod + def _get_service_dll( + service_key, + ) -> Union[str, interfaces.renderers.BaseAbsentValue]: + try: + param_key = next( + key + for key in service_key.get_subkeys() + if key.get_name() == "Parameters" + ) + return ( + next( + val + for val in param_key.get_values() + if val.get_name() == "ServiceDll" + ) + .decode_data() + .decode("utf-16") + .rstrip("\x00") + ) + + except UnicodeDecodeError: + return renderers.UnparsableValue() + except StopIteration: + return renderers.UnreadableValue() + + @staticmethod + def _get_service_binary( + service_key, + ) -> Union[str, interfaces.renderers.BaseAbsentValue]: + try: + return ( + next( + val + for val in service_key.get_values() + if val.get_name() == "ImagePath" + ) + .decode_data() + .decode("utf-16") + .rstrip("\x00") + ) + + except UnicodeDecodeError: + return renderers.UnparsableValue() + except StopIteration: + return renderers.UnreadableValue() + + @staticmethod + def _get_service_binary_map( + services_key: interfaces.objects.ObjectInterface, + ) -> Dict[str, ServiceBinaryInfo]: + services = services_key.get_subkeys() + return { + service_key.get_name(): ServiceBinaryInfo( + SvcScan._get_service_dll(service_key), + SvcScan._get_service_binary(service_key), + ) + for service_key in services + } + def _generator(self): kernel = self.context.modules[self.config["kernel"]] @@ -157,6 +255,15 @@ class SvcScan(interfaces.plugins.PluginInterface): self.context, kernel.symbol_table_name, self.config_path ) + # Building the dictionary ahead of time is much better for performance + # vs looking up each service's DLL individually. + services_key = self._get_service_key(kernel) + service_binary_dll_map = ( + self._get_service_binary_map(services_key) + if services_key is not None + else {} + ) + relative_tag_offset = self.context.symbol_space.get_type( service_table_name + constants.BANG + "_SERVICE_RECORD" ).relative_child_offset("Tag") @@ -209,7 +316,16 @@ class SvcScan(interfaces.plugins.PluginInterface): if not service_record.is_valid(): continue - yield (0, self.get_record_tuple(service_record)) + service_info = service_binary_dll_map.get( + service_record.get_name(), + ServiceBinaryInfo( + renderers.UnreadableValue(), renderers.UnreadableValue() + ), + ) + yield ( + 0, + self.get_record_tuple(service_record, service_info), + ) else: service_header = self.context.object( service_table_name + constants.BANG + "_SERVICE_HEADER", @@ -227,7 +343,16 @@ class SvcScan(interfaces.plugins.PluginInterface): if service_record in seen: break seen.append(service_record) - yield (0, self.get_record_tuple(service_record)) + service_info = service_binary_dll_map.get( + service_record.get_name(), + ServiceBinaryInfo( + renderers.UnreadableValue(), renderers.UnreadableValue() + ), + ) + yield ( + 0, + self.get_record_tuple(service_record, service_info), + ) def run(self): return renderers.TreeGrid( @@ -241,6 +366,8 @@ class SvcScan(interfaces.plugins.PluginInterface): ("Name", str), ("Display", str), ("Binary", str), + ("Binary (Registry)", str), + ("Dll", str), ], self._generator(), ) From 77b01106356ab30ec531799a66d38db3951033c4 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 14 Dec 2023 14:16:40 -0600 Subject: [PATCH 507/526] Windows: Fixes svc symbols and OS version detection There were some changes made to the service structures between certain versions of Windows that caused the PID/binary path to fail to parse across quite a few samples. This adds the appropriate version detection logic and updated symbol files to ensure that services are parsed correctly across Windows versions. --- volatility3/framework/interfaces/context.py | 2 +- .../framework/plugins/windows/svcscan.py | 50 +++- .../services/services-win10-17763-x86.json | 248 +++++++++++++++++ .../services/services-win10-18362-x64.json | 255 ++++++++++++++++++ .../services/services-win10-18362-x86.json | 248 +++++++++++++++++ .../services/services-win10-19041-x64.json | 255 ++++++++++++++++++ .../services/services-win10-19041-x86.json | 248 +++++++++++++++++ .../services/services-win10-25398-x64.json | 255 ++++++++++++++++++ .../framework/symbols/windows/versions.py | 34 +++ 9 files changed, 1584 insertions(+), 11 deletions(-) create mode 100644 volatility3/framework/symbols/windows/services/services-win10-17763-x86.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-18362-x64.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-18362-x86.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-19041-x64.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-19041-x86.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-25398-x64.json diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 29cb41379..a95f2b464 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -87,7 +87,7 @@ class ContextInterface(metaclass=ABCMeta): offset: int, native_layer_name: str = None, **arguments, - ): + ) -> "interfaces.objects.ObjectInterface": """Object factory, takes a context, symbol, offset and optional layer_name. diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 720f745d8..74fe2d802 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -106,26 +106,46 @@ class SvcScan(interfaces.plugins.PluginInterface): and is_64bit ): symbol_filename = "services-xp-2003-x64" + elif ( + versions.is_win10_25398_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win10-25398-x64" + elif ( + versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win10-19041-x64" + elif ( + versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win10-19041-x86" + elif ( + versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win10-18362-x64" + elif ( + versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win10-18362-x86" elif ( versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) and is_64bit ): symbol_filename = "services-win10-16299-x64" + elif ( + versions.is_win10_17763_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win10-17763-x86" elif ( versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) and not is_64bit ): symbol_filename = "services-win10-16299-x86" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win8-x64" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win8-x86" elif ( versions.is_win10_15063(context=context, symbol_table=symbol_table) and is_64bit @@ -136,6 +156,16 @@ class SvcScan(interfaces.plugins.PluginInterface): and not is_64bit ): symbol_filename = "services-win10-15063-x86" + elif ( + versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win8-x64" + elif ( + versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win8-x86" elif ( versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) and is_64bit diff --git a/volatility3/framework/symbols/windows/services/services-win10-17763-x86.json b/volatility3/framework/symbols/windows/services/services-win10-17763-x86.json new file mode 100644 index 000000000..8f2854721 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-17763-x86.json @@ -0,0 +1,248 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 4 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 12 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 20 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 12 + }, + "_SERVICE_RECORD": { + "fields": { + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 48 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 160 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 24 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 56 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 44 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 160 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 52 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 156 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-18362-x64.json b/volatility3/framework/symbols/windows/services/services-win10-18362-x64.json new file mode 100644 index 000000000..a6a80c1d3 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-18362-x64.json @@ -0,0 +1,255 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 16 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 24 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 40 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_RECORD": { + "fields": { + "ServiceList": { + "type": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + }, + "offset": 0 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 32 + }, + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 64 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 240 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 36 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 76 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 56 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 240 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 72 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 248 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-18362-x86.json b/volatility3/framework/symbols/windows/services/services-win10-18362-x86.json new file mode 100644 index 000000000..4684dfe5b --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-18362-x86.json @@ -0,0 +1,248 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 4 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 12 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 20 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 12 + }, + "_SERVICE_RECORD": { + "fields": { + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 48 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 164 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 24 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 56 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 44 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 164 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 52 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 156 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-19041-x64.json b/volatility3/framework/symbols/windows/services/services-win10-19041-x64.json new file mode 100644 index 000000000..e44dbbd37 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-19041-x64.json @@ -0,0 +1,255 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 16 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 24 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 40 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_RECORD": { + "fields": { + "ServiceList": { + "type": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + }, + "offset": 0 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 32 + }, + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 64 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 296 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 36 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 76 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 56 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 296 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 72 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 296 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "David McDonald", + "datetime": "2023-11-16T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-19041-x86.json b/volatility3/framework/symbols/windows/services/services-win10-19041-x86.json new file mode 100644 index 000000000..cc5ed9a73 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-19041-x86.json @@ -0,0 +1,248 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 4 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 12 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 20 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 12 + }, + "_SERVICE_RECORD": { + "fields": { + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 48 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 192 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 24 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 56 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 44 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 192 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 52 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 192 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-25398-x64.json b/volatility3/framework/symbols/windows/services/services-win10-25398-x64.json new file mode 100644 index 000000000..cd29abc43 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-25398-x64.json @@ -0,0 +1,255 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 16 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 24 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 40 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_RECORD": { + "fields": { + "ServiceList": { + "type": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + }, + "offset": 0 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 32 + }, + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 64 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 336 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 36 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 84 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 56 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 296 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 80 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 336 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "David McDonald", + "datetime": "2023-11-16T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index 84ce65432..6e2c2e9bd 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -151,11 +151,45 @@ is_win10_16299_or_later = OsDistinguisher( ], ) +is_win10_17763_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 17763), + fallback_checks=[ + ("_EPROCESS", "TrustletIdentity", False), + ("ParentSecurityDomain", None, False), + ], +) + +is_win10_18362_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 18362), + fallback_checks=[ + ("ObHeaderCookie", None, True), + ("_CM_CACHED_VALUE_INDEX", None, False), + ("_WNF_PROCESS_CONTEXT", None, True), + ], +) + is_win10_18363_or_later = OsDistinguisher( version_check=lambda x: x >= (10, 0, 18363), fallback_checks=[("_KQOS_GROUPING_SETS", None, True)], ) +is_win10_19041_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 19041), + fallback_checks=[ + ("_EPROCESS", "TimerResolutionIgnore", True), + ("_EPROCESS", "VmProcessorHostTransition", True), + ("_KQOS_GROUPING_SETS", None, True), + ], +) + +is_win10_25398_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 25398), + fallback_checks=[ + ("_EPROCESS", "MmSlabIdentity", True), + ("_EPROCESS", "EnableProcessImpersonationLogging", True), + ], +) + is_windows_10 = OsDistinguisher( version_check=lambda x: x >= (10, 0), fallback_checks=[("ObHeaderCookie", None, True)], From 86d6b4f7245a553bb2b7db6a3d844375a57f8cd4 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:08:14 -0600 Subject: [PATCH 508/526] Bump plugin version --- volatility3/framework/plugins/windows/svcscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 74fe2d802..7688594d2 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -32,7 +32,7 @@ class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 4b400b6df612fe0f3a796c06dbfb7fa272e5d0fd Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:09:36 -0600 Subject: [PATCH 509/526] Refactor out code duplication in OS version checks --- .../framework/plugins/windows/svcscan.py | 132 ++++++------------ 1 file changed, 39 insertions(+), 93 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 7688594d2..4f20ed346 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -4,9 +4,16 @@ import logging import os -from typing import Dict, List, NamedTuple, Union +from typing import Dict, List, NamedTuple, Optional, Tuple, Union, cast -from volatility3.framework import constants, exceptions, interfaces, renderers, symbols +from volatility3.framework import ( + constants, + exceptions, + interfaces, + objects, + renderers, + symbols, +) from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints @@ -76,6 +83,28 @@ class SvcScan(interfaces.plugins.PluginInterface): binary_info.dll, ) + # These checks must be completed from newest -> oldest OS version. + _win_version_file_map: List[Tuple[versions.OsDistinguisher, bool, str]] = [ + (versions.is_win10_25398_or_later, True, "services-win10-25398-x64"), + (versions.is_win10_19041_or_later, True, "services-win10-19041-x64"), + (versions.is_win10_19041_or_later, False, "services-win10-19041-x86"), + (versions.is_win10_18362_or_later, True, "services-win10-18362-x64"), + (versions.is_win10_18362_or_later, False, "services-win10-18362-x86"), + (versions.is_win10_17763_or_later, False, "services-win10-17763-x86"), + (versions.is_win10_16299_or_later, True, "services-win10-16299-x64"), + (versions.is_win10_16299_or_later, False, "services-win10-16299-x86"), + (versions.is_win10_15063, True, "services-win10-15063-x64"), + (versions.is_win10_15063, False, "services-win10-15063-x86"), + (versions.is_win10_up_to_15063, True, "services-win8-x64"), + (versions.is_win10_up_to_15063, False, "services-win8-x86"), + (versions.is_windows_8_or_later, True, "services-win8-x64"), + (versions.is_windows_8_or_later, True, "services-win8-x86"), + (versions.is_vista_or_later, True, "services-vista-x64"), + (versions.is_vista_or_later, False, "services-vista-x86"), + (versions.is_windows_xp, False, "services-xp-x86"), + (versions.is_xp_or_2003, True, "services-xp-2003-x64"), + ] + @staticmethod def create_service_table( context: interfaces.context.ContextInterface, @@ -96,97 +125,14 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types = context.symbol_space[symbol_table].natives is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) - if ( - versions.is_windows_xp(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-xp-x86" - elif ( - versions.is_xp_or_2003(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-xp-2003-x64" - elif ( - versions.is_win10_25398_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-25398-x64" - elif ( - versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-19041-x64" - elif ( - versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-19041-x86" - elif ( - versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-18362-x64" - elif ( - versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-18362-x86" - elif ( - versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-16299-x64" - elif ( - versions.is_win10_17763_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-17763-x86" - elif ( - versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-16299-x86" - elif ( - versions.is_win10_15063(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-15063-x64" - elif ( - versions.is_win10_15063(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-15063-x86" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win8-x64" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win8-x86" - elif ( - versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win8-x64" - elif ( - versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win8-x86" - elif ( - versions.is_vista_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-vista-x64" - elif ( - versions.is_vista_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-vista-x86" - else: + try: + symbol_filename = next( + filename + for version_check, for_64bit, filename in SvcScan._win_version_file_map + if is_64bit == for_64bit + and version_check(context=context, symbol_table=symbol_table) + ) + except StopIteration: raise NotImplementedError("This version of Windows is not supported!") return intermed.IntermediateSymbolTable.create( From 5541378afe86d41734c05f6b84bbafcd26d46644 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:09:59 -0600 Subject: [PATCH 510/526] Create unique config path --- volatility3/framework/plugins/windows/svcscan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 4f20ed346..d16bee113 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -147,7 +147,9 @@ class SvcScan(interfaces.plugins.PluginInterface): def _get_service_key(self, kernel): for hive in hivelist.HiveList.list_hives( context=self.context, - base_config_path=self.config_path, + base_config_path=interfaces.configuration.path_join( + self.config_path, "hivelist" + ), layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, filter_string="machine\\system", From f5c1bf0f1f5c177b2dcee9d09d3ab522ba83f41e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:10:20 -0600 Subject: [PATCH 511/526] Remove argument that matches default --- volatility3/framework/plugins/windows/svcscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index d16bee113..b77491a65 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -153,7 +153,6 @@ class SvcScan(interfaces.plugins.PluginInterface): layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, filter_string="machine\\system", - hive_offsets=None, ): # Get ControlSet\Services. try: From c517a44cde3cb87d48da3a72d685630eca7acaa1 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:10:54 -0600 Subject: [PATCH 512/526] Add type hints/casts to _get_service_key method --- volatility3/framework/plugins/windows/svcscan.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index b77491a65..dd2f76890 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -144,7 +144,7 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types=native_types, ) - def _get_service_key(self, kernel): + def _get_service_key(self, kernel) -> Optional[objects.StructType]: for hive in hivelist.HiveList.list_hives( context=self.context, base_config_path=interfaces.configuration.path_join( @@ -156,10 +156,14 @@ class SvcScan(interfaces.plugins.PluginInterface): ): # Get ControlSet\Services. try: - return hive.get_key(r"CurrentControlSet\Services") + return cast( + objects.StructType, hive.get_key(r"CurrentControlSet\Services") + ) except (KeyError, exceptions.InvalidAddressException): try: - return hive.get_key(r"ControlSet001\Services") + return cast( + objects.StructType, hive.get_key(r"ControlSet001\Services") + ) except (KeyError, exceptions.InvalidAddressException): pass From 42c2a86a37ace33cf93d28efd491b00e8c29dd53 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:11:16 -0600 Subject: [PATCH 513/526] Add logging for when a Services key cannot be retrieved --- volatility3/framework/plugins/windows/svcscan.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index dd2f76890..095661880 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -165,6 +165,10 @@ class SvcScan(interfaces.plugins.PluginInterface): objects.StructType, hive.get_key(r"ControlSet001\Services") ) except (KeyError, exceptions.InvalidAddressException): + vollog.log( + constants.LOGLEVEL_VVVV, + "Could not retrieve any control set from SYSTEM hive", + ) pass return None From 4192e8a711aff985352f3347dc8cd289894a812f Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 18 Jan 2024 12:22:23 -0600 Subject: [PATCH 514/526] Fix condition in OsDistinguisher --- volatility3/framework/symbols/windows/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index 6e2c2e9bd..e1e74afc0 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -155,7 +155,7 @@ is_win10_17763_or_later = OsDistinguisher( version_check=lambda x: x >= (10, 0, 17763), fallback_checks=[ ("_EPROCESS", "TrustletIdentity", False), - ("ParentSecurityDomain", None, False), + ("ParentSecurityDomain", None, True), ], ) From 43e8e01daff8dc1ff88b9edfc6d5d6b2fac33417 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 30 Jan 2024 09:26:17 -0600 Subject: [PATCH 515/526] Removes unnecessary pass and deindents return stmt --- volatility3/framework/plugins/windows/svcscan.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 095661880..10de46e2a 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -169,9 +169,8 @@ class SvcScan(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVVV, "Could not retrieve any control set from SYSTEM hive", ) - pass - return None + return None @staticmethod def _get_service_dll( From 470c750d82ee0208f673472bcf2bf6632ae4190f Mon Sep 17 00:00:00 2001 From: hsarkey Date: Wed, 31 Jan 2024 16:00:29 -0500 Subject: [PATCH 516/526] Updated to use dictionary for refined criteria and BaseAbsentValue for when criteria is not met --- .../framework/plugins/windows/malfind.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 284144077..8c0cb68ac 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -142,15 +142,20 @@ class Malfind(interfaces.plugins.PluginInterface): kernel = self.context.modules[self.config["kernel"]] # set refined criteria to know when to add to "Notes" column - refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] + refined_criteria = { + b"MZ": "MZ header", + b"\x55\x8B": "PE header", + b"\x55\x48": "Function prologue", + b"\x55\x89": "Function prologue", + } is_32bit_arch = not symbols.symbol_table_is_64bit( self.context, kernel.symbol_table_name ) for proc in procs: - # by default, "Notes" column will be set to none - notes = "None" + # by default, "Notes" column will be set to N/A + notes = renderers.NotApplicableValue() process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( @@ -158,12 +163,7 @@ class Malfind(interfaces.plugins.PluginInterface): ): # Check for unique headers and update "Notes" column if criteria is met if data[0:2] in refined_criteria: - if data[0:2] == b"MZ": - notes = "MZ header" - elif data[0:2] == b"\x55\x8B": - notes = "PE header" - else: - notes = "Function prologue" + notes = refined_criteria[data[0:2]] # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): From 6d093a4e4338aaee5b69384f1652cda994a3e1d3 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jan 2024 21:04:16 +0000 Subject: [PATCH 517/526] Documentation: Update copyright in README.md for 2024 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 471735af8..f69acb3bb 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 31 Jan 2024 21:04:16 +0000 Subject: [PATCH 518/526] Documentation: Update copyright in README.md for 2024 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 471735af8..f69acb3bb 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 31 Jan 2024 21:09:48 +0000 Subject: [PATCH 519/526] Core: Apply black 24.1.0 across the whole codebase --- volatility3/cli/text_renderer.py | 8 +++--- volatility3/framework/__init__.py | 6 ++--- volatility3/framework/automagic/mac.py | 6 ++--- volatility3/framework/automagic/module.py | 6 ++--- .../framework/automagic/symbol_finder.py | 12 ++++----- volatility3/framework/automagic/windows.py | 20 +++++++------- .../framework/configuration/requirements.py | 6 ++--- volatility3/framework/layers/avml.py | 6 ++--- volatility3/framework/layers/elf.py | 6 ++--- volatility3/framework/layers/intel.py | 6 ++--- volatility3/framework/layers/lime.py | 6 ++--- volatility3/framework/layers/qemu.py | 6 ++--- volatility3/framework/layers/xen.py | 6 ++--- volatility3/framework/objects/__init__.py | 6 ++--- volatility3/framework/plugins/mac/pslist.py | 4 +-- .../framework/plugins/windows/crashinfo.py | 6 ++--- .../framework/plugins/windows/mftscan.py | 1 - .../framework/plugins/windows/netscan.py | 10 ++++--- .../plugins/windows/registry/printkey.py | 6 ++--- volatility3/framework/renderers/conversion.py | 6 ++--- volatility3/framework/symbols/__init__.py | 26 +++++++++---------- .../symbols/windows/extensions/__init__.py | 6 ++--- volatility3/plugins/windows/statistics.py | 10 +++---- 23 files changed, 88 insertions(+), 93 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ffb8d516b..6e58ee68d 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -389,9 +389,11 @@ class JsonRenderer(CLIRenderer): interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), - datetime.datetime: lambda x: x.isoformat() - if not isinstance(x, interfaces.renderers.BaseAbsentValue) - else None, + datetime.datetime: lambda x: ( + x.isoformat() + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else None + ), "default": lambda x: x, } diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 9c17846a8..1565b2267 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -206,9 +206,9 @@ def _zipwalk(path: str): if not file.is_dir(): dirlist = zip_results.get(os.path.dirname(file.filename), []) dirlist.append(os.path.basename(file.filename)) - zip_results[ - os.path.join(path, os.path.dirname(file.filename)) - ] = dirlist + zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( + dirlist + ) for value in zip_results: yield value, zip_results[value] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index aa75fbc3d..e51753139 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -138,9 +138,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): config_path = join("automagic", "MacIntelHelper", new_layer_name) context.config[join(config_path, "memory_layer")] = layer_name context.config[join(config_path, "page_map_offset")] = dtb - context.config[ - join(config_path, MacSymbolFinder.banner_config_key) - ] = str(banner, "latin-1") + context.config[join(config_path, MacSymbolFinder.banner_config_key)] = ( + str(banner, "latin-1") + ) new_layer = intel.Intel32e( context, diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index ee56a040c..ff13db905 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -34,9 +34,9 @@ class KernelModule(interfaces.automagic.AutomagicInterface): return None # The requirement is unfulfilled and is a ModuleRequirement - context.config[ - interfaces.configuration.path_join(new_config_path, "class") - ] = "volatility3.framework.contexts.Module" + context.config[interfaces.configuration.path_join(new_config_path, "class")] = ( + "volatility3.framework.contexts.Module" + ) for req in requirement.requirements: if ( diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index bf1c8ff16..21e594549 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -150,12 +150,12 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join - context.config[ - path_join(config_path, requirement.name, "class") - ] = clazz - context.config[ - path_join(config_path, requirement.name, "isf_url") - ] = isf_path + context.config[path_join(config_path, requirement.name, "class")] = ( + clazz + ) + context.config[path_join(config_path, requirement.name, "isf_url")] = ( + isf_path + ) context.config[ path_join(config_path, requirement.name, "symbol_mask") ] = layer.address_mask diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index a8530829b..52296f5ad 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -402,19 +402,19 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): if swap_location: context.config[current_layer_path] = current_layer_name try: - context.config[ - layer_loc_path - ] = requirements.URIRequirement.location_from_file( - swap_location + context.config[layer_loc_path] = ( + requirements.URIRequirement.location_from_file( + swap_location + ) ) except ValueError: vollog.warning( f"Volatility swap_location {swap_location} could not be validated - swap layer disabled" ) continue - context.config[ - layer_class_path - ] = "volatility3.framework.layers.physical.FileLayer" + context.config[layer_class_path] = ( + "volatility3.framework.layers.physical.FileLayer" + ) # Add the requirement new_req = requirements.TranslationLayerRequirement( @@ -424,9 +424,9 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): ) swap_req.add_requirement(new_req) - context.config[ - path_join(swap_sub_config, "number_of_elements") - ] = counter + context.config[path_join(swap_sub_config, "number_of_elements")] = ( + counter + ) context.config[swap_sub_config] = True swap_req.construct(context, swap_config) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index abdffdbe4..1c0622574 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -550,9 +550,9 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} - context.config[ - interfaces.configuration.path_join(config_path, self.name) - ] = True + context.config[interfaces.configuration.path_join(config_path, self.name)] = ( + True + ) return {} @classmethod diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index c825464cc..2e5572192 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -224,7 +224,7 @@ class AVMLStacker(interfaces.automagic.StackerLayerInterface): except exceptions.LayerException: return None new_name = context.layers.free_layer_name("AVMLLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return AVMLLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index a10d36592..b2fd6d4d1 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -115,9 +115,9 @@ class Elf64Stacker(interfaces.automagic.StackerLayerInterface): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("Elf64Layer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) try: return Elf64Layer(context, new_name, new_name) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 7d3b86a12..ae477854d 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -277,9 +277,9 @@ class Intel(linear.LinearlyMappedLayer): This allows translation layers to provide maps of contiguous regions in one layer """ - stashed_offset = ( - stashed_mapped_offset - ) = stashed_size = stashed_mapped_size = stashed_map_layer = None + stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = ( + stashed_map_layer + ) = None for offset, size, mapped_offset, mapped_size, map_layer in self._mapping( offset, length, ignore_errors ): diff --git a/volatility3/framework/layers/lime.py b/volatility3/framework/layers/lime.py index 28d646640..8b93932ab 100644 --- a/volatility3/framework/layers/lime.py +++ b/volatility3/framework/layers/lime.py @@ -104,7 +104,7 @@ class LimeStacker(interfaces.automagic.StackerLayerInterface): except LimeFormatException: return None new_name = context.layers.free_layer_name("LimeLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return LimeLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 501b8655e..ff483291c 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -486,9 +486,9 @@ class QemuStacker(interfaces.automagic.StackerLayerInterface): except exceptions.LayerException: return None new_name = context.layers.free_layer_name("QemuSuspendLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) layer = QemuSuspendLayer(context, new_name, new_name) cls.stacker_slow_warning() return layer diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index f7881a091..927b30430 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -173,8 +173,8 @@ class XenCoreDumpStacker(elf.Elf64Stacker): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("XenCoreDumpLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return XenCoreDumpLayer(context, new_name, new_name) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 831856e3d..316a30bec 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -768,12 +768,10 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): raise IndexError(f"Member not present in array template: {child}") @overload - def __getitem__(self, i: int) -> interfaces.objects.Template: - ... + def __getitem__(self, i: int) -> interfaces.objects.Template: ... @overload - def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: - ... + def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: ... def __getitem__(self, i): """Returns the i-th item from the array.""" diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 88045a277..9b570f3f9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -49,9 +49,7 @@ class PsList(interfaces.plugins.PluginInterface): ] @classmethod - def get_list_tasks( - cls, method: str - ) -> Callable[ + def get_list_tasks(cls, method: str) -> Callable[ [interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface], ]: diff --git a/volatility3/framework/plugins/windows/crashinfo.py b/volatility3/framework/plugins/windows/crashinfo.py index 4ecd85087..862eb6080 100644 --- a/volatility3/framework/plugins/windows/crashinfo.py +++ b/volatility3/framework/plugins/windows/crashinfo.py @@ -46,9 +46,9 @@ class Crashinfo(interfaces.plugins.PluginInterface): bitmap_size = format_hints.Hex(summary_header.BitmapSize) bitmap_pages = format_hints.Hex(summary_header.Pages) else: - bitmap_header_size = ( - bitmap_size - ) = bitmap_pages = renderers.NotApplicableValue() + bitmap_header_size = bitmap_size = bitmap_pages = ( + renderers.NotApplicableValue() + ) yield ( 0, diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 7e4e1ca18..a266fd864 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -176,7 +176,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): class ADS(interfaces.plugins.PluginInterface): - """Scans for Alternate Data Stream""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index d0bbd5cbd..62ead3ab7 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -487,10 +487,12 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if not isinstance(row_data[9], datetime.datetime): continue row_data = [ - "N/A" - if isinstance(i, renderers.UnreadableValue) - or isinstance(i, renderers.UnparsableValue) - else i + ( + "N/A" + if isinstance(i, renderers.UnreadableValue) + or isinstance(i, renderers.UnparsableValue) + else i + ) for i in row_data ] description = ( diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index e248c19bc..180f8f9d9 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -193,9 +193,9 @@ class PrintKey(interfaces.plugins.PluginInterface): vollog.debug( "Couldn't read registry value type, so data is unreadable" ) - value_data: Union[ - interfaces.renderers.BaseAbsentValue, bytes - ] = renderers.UnreadableValue() + value_data: Union[interfaces.renderers.BaseAbsentValue, bytes] = ( + renderers.UnreadableValue() + ) else: try: value_data = node.decode_data() diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index bf7da9ecb..bb18fcc8a 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -28,9 +28,9 @@ def wintime_to_datetime( def unixtime_to_datetime( unixtime: int, ) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: - ret: Union[ - interfaces.renderers.BaseAbsentValue, datetime.datetime - ] = renderers.UnparsableValue() + ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = ( + renderers.UnparsableValue() + ) if unixtime > 0: with contextlib.suppress(ValueError): diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index 10cf39cf1..d1e7a104d 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -35,9 +35,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): def __init__(self) -> None: super().__init__() - self._dict: Dict[ - str, interfaces.symbols.BaseSymbolTableInterface - ] = collections.OrderedDict() + self._dict: Dict[str, interfaces.symbols.BaseSymbolTableInterface] = ( + collections.OrderedDict() + ) # Permanently cache all resolved symbols self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} @@ -73,9 +73,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self, offset: int, size: int = 0, table_name: str = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" - table_list: Iterable[ - interfaces.symbols.BaseSymbolTableInterface - ] = self._dict.values() + table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( + self._dict.values() + ) if table_name is not None: if table_name in self._dict: table_list = [self._dict[table_name]] @@ -179,15 +179,15 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): if child.vol.type_name not in self._resolved: traverse_list.append(child.vol.type_name) try: - self._resolved[ - child.vol.type_name - ] = self._weak_resolve( - SymbolType.TYPE, child.vol.type_name + self._resolved[child.vol.type_name] = ( + self._weak_resolve( + SymbolType.TYPE, child.vol.type_name + ) ) except exceptions.SymbolError: - self._resolved[ - child.vol.type_name - ] = self.UnresolvedTemplate(child.vol.type_name) + self._resolved[child.vol.type_name] = ( + self.UnresolvedTemplate(child.vol.type_name) + ) # Stash the replacement replacements.add((traverser, child)) elif child.children: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d435851d7..846e5bd90 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -452,9 +452,9 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[ - str, interfaces.renderers.BaseAbsentValue - ] = renderers.UnreadableValue() + name: Union[str, interfaces.renderers.BaseAbsentValue] = ( + renderers.UnreadableValue() + ) # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 9915312e3..7f56b75f8 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -31,13 +31,9 @@ class Statistics(plugins.PluginInterface): # Do mass mapping and determine the number of different layers and how many pages go to each one layer = self.context.layers[self.config["primary"]] - page_count = ( - swap_count - ) = ( - invalid_page_count - ) = ( - large_page_count - ) = large_swap_count = large_invalid_count = other_invalid = 0 + page_count = swap_count = invalid_page_count = large_page_count = ( + large_swap_count + ) = large_invalid_count = other_invalid = 0 if isinstance(layer, intel.Intel): page_addr = 0 From ba488c78cd02478a60ba94a72d1c6a712f02c365 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jan 2024 21:13:00 +0000 Subject: [PATCH 520/526] Documentation: Bump the copyright here to 2024 --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index d601c1eee..cabfdc327 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -169,7 +169,7 @@ master_doc = "index" # General information about the project. project = "Volatility 3" -copyright = "2012-2022, Volatility Foundation" +copyright = "2012-2024, Volatility Foundation" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From f95be04305d7fe1bedf599849d40996f82d2032b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jan 2024 21:13:00 +0000 Subject: [PATCH 521/526] Documentation: Bump the copyright here to 2024 --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index d601c1eee..cabfdc327 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -169,7 +169,7 @@ master_doc = "index" # General information about the project. project = "Volatility 3" -copyright = "2012-2022, Volatility Foundation" +copyright = "2012-2024, Volatility Foundation" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 130621a2e7655015f220ac561de974e99f571fca Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Feb 2024 20:04:36 +0100 Subject: [PATCH 522/526] Changing comment when Type error is occuring --- volatility3/framework/plugins/windows/iat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 11c273859..73976fb86 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -50,7 +50,7 @@ class IAT(interfaces.plugins.PluginInterface): ) if proc_layer_name is None: - raise TypeError("Layer must be a string not None") + raise TypeError("add_process_layer failed") pe_table_name = intermed.IntermediateSymbolTable.create( self.context, From ab8ed049a4c4bddbc4c2a2129cb425b9113b5d37 Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sat, 3 Feb 2024 11:32:00 +0300 Subject: [PATCH 523/526] Windows: add TrueCrypt plugin --- .../framework/plugins/windows/truecrypt.py | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 volatility3/framework/plugins/windows/truecrypt.py diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py new file mode 100644 index 000000000..988c900ea --- /dev/null +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -0,0 +1,141 @@ +from typing import Iterable, Generator, List, Tuple + +from volatility3.framework import constants, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces.configuration import RequirementInterface +from volatility3.framework.interfaces.objects import ObjectInterface +from volatility3.framework.objects import Bytes, DataFormatInfo, Integer, StructType +from volatility3.framework.objects.templates import ObjectTemplate +from volatility3.framework.objects.utility import array_to_string +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe + +from volatility3.plugins.windows import modules + +class Passphrase(interfaces.plugins.PluginInterface): + """TrueCrypt Cached Passphrase Finder""" + + _version = (0, 1, 0) + _required_framework_version = (2, 5, 2) + + @classmethod + def get_requirements(cls) -> List[RequirementInterface]: + return [ + requirements.ModuleRequirement( + 'kernel', + description='Windows kernel', + architectures=['Intel32', 'Intel64'] + ), + requirements.VersionRequirement( + name='modules', + component=modules.Modules, + version=(1, 1, 0) + ), + requirements.IntRequirement( + name="min-length", + description="Minimum length of passphrases to identify", + default=5, + optional=True + ), + ] + + def scan_module(self, module_base: int, layer_name: str) -> Generator[Tuple[int, str], None, None]: + """Scans the TrueCrypt kernel module for cached passphrases. + + Args: + module_base: the module's DLL base + layer_name: the name of the layer in which the module resides + + Generates: + A tuple of the offset at which a password is found, and the password + """ + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "windows", + "pe", + class_types=pe.class_types + ) + dos_header: pe.IMAGE_DOS_HEADER = self.context.object( + pe_table_name + constants.BANG + '_IMAGE_DOS_HEADER', + layer_name, + module_base, + ) + data_section: StructType = next( + sec for sec in dos_header.get_nt_header().get_sections() + if array_to_string(sec.Name) == '.data' + ) + base: int = data_section.VirtualAddress + module_base + size: int = data_section.Misc.VirtualSize + # Looking at `Length` in TrueCrypt/Common/Password.h::Password struct + DWORD_SIZE_BYTES: int = 4 + format = DataFormatInfo(length=DWORD_SIZE_BYTES, byteorder="little", signed=True) + int32 = ObjectTemplate( + Integer, + pe_table_name + constants.BANG + 'int', + data_format=format + ) + count, not_aligned = divmod(size, DWORD_SIZE_BYTES) + if not_aligned: + raise ValueError("PE data section not DWORD-aligned!") + lengths = self.context.object( + pe_table_name + constants.BANG + 'array', + layer_name, + base, + count=count, + subtype=int32, + ) + min_length = self.config.get('min-length') + for length in lengths: + # TrueCrypt maximum password length is 64 + # (see TrueCrypt/Common/Password.h) + if not min_length <= length <= 64: + continue + offset = length.vol['offset'] + DWORD_SIZE_BYTES + passphrase: Bytes = self.context.object( + pe_table_name + constants.BANG + 'bytes', + layer_name, + offset, + length=length, + ) + # TrueCrypt/Common/Password.c permits chars in the range + # [0x20, 0x7F). + if not all(0x20 <= c < 0x7F for c in passphrase): + continue + # TrueCrypt/Common/Password.h::Password struct is padded with + # 3 zero bytes to keep 64-byte alignment. + buf: Bytes = self.context.object( + pe_table_name + constants.BANG + 'bytes', + layer_name, + offset + length + 1, # +1 for '\0'-terminated password string + length=3 + ) + if any(buf): + continue + # Password found. + yield offset, passphrase.decode(encoding='ascii') + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + mods: Iterable[ObjectInterface] = modules.Modules.list_modules( + self.context, + kernel.layer_name, + kernel.symbol_table_name + ) + truecrypt_module_base = next( + mod.DllBase for mod in mods + if mod.BaseDllName.get_string().lower() == 'truecrypt.sys' + ) + for offset, password in self.scan_module(truecrypt_module_base, kernel.layer_name): + yield (0, (format_hints.Hex(offset), len(password), password)) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Length", int), + ("Password", str), + ], + self._generator() + ) From 41cbfbfa340ef8949d20e1717aa5baaee02ef984 Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sat, 3 Feb 2024 11:58:25 +0300 Subject: [PATCH 524/526] add license --- volatility3/framework/plugins/windows/truecrypt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index 988c900ea..feba56965 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -1,3 +1,7 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from typing import Iterable, Generator, List, Tuple from volatility3.framework import constants, interfaces, renderers From cf9029fd85ae090034a253e3dcd819a0381b6442 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 4 Feb 2024 00:43:33 +0100 Subject: [PATCH 525/526] Fixing the year in the Copyright --- volatility3/framework/plugins/windows/iat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 73976fb86..d2fdc0ad8 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,4 +1,4 @@ -# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 import logging, io, pefile From d44c0fef36dfe1f7a83cc25211073489a56820be Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sun, 4 Feb 2024 11:21:14 +0300 Subject: [PATCH 526/526] run Black formatter --- .../framework/plugins/windows/truecrypt.py | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index feba56965..81250a749 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -17,6 +17,7 @@ from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import modules + class Passphrase(interfaces.plugins.PluginInterface): """TrueCrypt Cached Passphrase Finder""" @@ -27,78 +28,75 @@ class Passphrase(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[RequirementInterface]: return [ requirements.ModuleRequirement( - 'kernel', - description='Windows kernel', - architectures=['Intel32', 'Intel64'] + "kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name='modules', - component=modules.Modules, - version=(1, 1, 0) + name="modules", component=modules.Modules, version=(1, 1, 0) ), requirements.IntRequirement( name="min-length", description="Minimum length of passphrases to identify", default=5, - optional=True + optional=True, ), ] - - def scan_module(self, module_base: int, layer_name: str) -> Generator[Tuple[int, str], None, None]: + + def scan_module( + self, module_base: int, layer_name: str + ) -> Generator[Tuple[int, str], None, None]: """Scans the TrueCrypt kernel module for cached passphrases. - + Args: module_base: the module's DLL base layer_name: the name of the layer in which the module resides - + Generates: A tuple of the offset at which a password is found, and the password """ pe_table_name = intermed.IntermediateSymbolTable.create( - self.context, - self.config_path, - "windows", - "pe", - class_types=pe.class_types + self.context, self.config_path, "windows", "pe", class_types=pe.class_types ) dos_header: pe.IMAGE_DOS_HEADER = self.context.object( - pe_table_name + constants.BANG + '_IMAGE_DOS_HEADER', + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", layer_name, module_base, ) data_section: StructType = next( - sec for sec in dos_header.get_nt_header().get_sections() - if array_to_string(sec.Name) == '.data' + sec + for sec in dos_header.get_nt_header().get_sections() + if array_to_string(sec.Name) == ".data" ) base: int = data_section.VirtualAddress + module_base size: int = data_section.Misc.VirtualSize # Looking at `Length` in TrueCrypt/Common/Password.h::Password struct DWORD_SIZE_BYTES: int = 4 - format = DataFormatInfo(length=DWORD_SIZE_BYTES, byteorder="little", signed=True) + format = DataFormatInfo( + length=DWORD_SIZE_BYTES, byteorder="little", signed=True + ) int32 = ObjectTemplate( - Integer, - pe_table_name + constants.BANG + 'int', - data_format=format + Integer, pe_table_name + constants.BANG + "int", data_format=format ) count, not_aligned = divmod(size, DWORD_SIZE_BYTES) if not_aligned: raise ValueError("PE data section not DWORD-aligned!") lengths = self.context.object( - pe_table_name + constants.BANG + 'array', + pe_table_name + constants.BANG + "array", layer_name, base, count=count, subtype=int32, ) - min_length = self.config.get('min-length') + min_length = self.config.get("min-length") for length in lengths: # TrueCrypt maximum password length is 64 # (see TrueCrypt/Common/Password.h) if not min_length <= length <= 64: continue - offset = length.vol['offset'] + DWORD_SIZE_BYTES + offset = length.vol["offset"] + DWORD_SIZE_BYTES passphrase: Bytes = self.context.object( - pe_table_name + constants.BANG + 'bytes', + pe_table_name + constants.BANG + "bytes", layer_name, offset, length=length, @@ -110,30 +108,31 @@ class Passphrase(interfaces.plugins.PluginInterface): # TrueCrypt/Common/Password.h::Password struct is padded with # 3 zero bytes to keep 64-byte alignment. buf: Bytes = self.context.object( - pe_table_name + constants.BANG + 'bytes', + pe_table_name + constants.BANG + "bytes", layer_name, - offset + length + 1, # +1 for '\0'-terminated password string - length=3 + offset + length + 1, # +1 for '\0'-terminated password string + length=3, ) if any(buf): continue # Password found. - yield offset, passphrase.decode(encoding='ascii') - + yield offset, passphrase.decode(encoding="ascii") + def _generator(self): kernel = self.context.modules[self.config["kernel"]] mods: Iterable[ObjectInterface] = modules.Modules.list_modules( - self.context, - kernel.layer_name, - kernel.symbol_table_name + self.context, kernel.layer_name, kernel.symbol_table_name ) truecrypt_module_base = next( - mod.DllBase for mod in mods - if mod.BaseDllName.get_string().lower() == 'truecrypt.sys' + mod.DllBase + for mod in mods + if mod.BaseDllName.get_string().lower() == "truecrypt.sys" ) - for offset, password in self.scan_module(truecrypt_module_base, kernel.layer_name): + for offset, password in self.scan_module( + truecrypt_module_base, kernel.layer_name + ): yield (0, (format_hints.Hex(offset), len(password), password)) - + def run(self) -> renderers.TreeGrid: return renderers.TreeGrid( [ @@ -141,5 +140,5 @@ class Passphrase(interfaces.plugins.PluginInterface): ("Length", int), ("Password", str), ], - self._generator() + self._generator(), )