From ccd4c1bee0d09a8d464dd603330f68de049cc24e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 2 Dec 2021 17:38:31 +1100 Subject: [PATCH 001/181] Fixed exceptions.elf reference --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fbc02399f..0edd60608 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -108,7 +108,7 @@ class module(generic.GenericIntelProcess): "linux", "elf", native_types = None, - class_types = extensions.elf.class_types) + class_types = elf.class_types) syms = self._context.object( self.get_symbol_table().name + constants.BANG + "array", From e8fa6e1e7faac7ac1b70284a0d14e4a5ef30ceba Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 2 Dec 2021 18:36:04 +1100 Subject: [PATCH 002/181] Add the Linux mountinfo module. --- .../framework/plugins/linux/mountinfo.py | 227 ++++++++++++++++++ .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 166 ++++++++++++- 3 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 volatility3/framework/plugins/linux/mountinfo.py diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py new file mode 100644 index 000000000..be423a2f4 --- /dev/null +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -0,0 +1,227 @@ +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +# Author: Gustavo Moreira + +import logging +from collections import namedtuple +from typing import Tuple, List, Iterable, Union + +from volatility3.framework import renderers, interfaces, constants +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + +MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "mnt_root_path", "path_root", + "mnt_opts", "fields", "mnt_type", "devname", "sb_opts")) + +class MountInfo(plugins.PluginInterface): + """Lists mount points in processes mount namespaces""" + + _required_framework_version = (2, 0, 0) + + _version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name="kernel", description="Linux kernel", + architectures=["Intel32", "Intel64"]), + requirements.PluginRequirement(name="pslist", + plugin=pslist.PsList, version=(2, 0, 0)), + requirements.ListRequirement(name="pids", + description="Filter on specific process IDs.", + element_type=int, + optional=True), + requirements.BooleanRequirement(name="all-processes", + description="Shows information about mount points for each process mount " + "namespace. It could take a while depending on the number of processes " + "running. Note that if this argument is not specified it uses the root " + "mount namespace based on pid 1.", + optional=True, + default=False), + requirements.BooleanRequirement(name="mount-format", + description="Shows a brief summary of a process mount points information " + "with similar output format to the older /proc/[pid]/mounts or the " + "user-land command 'mount -l'.", + optional=True, + default=False), + ] + + def _get_symbol_fullname(self, symbol_basename: str) -> str: + """Given a short symbol or type name, it returns its full name""" + return self._vmlinux.symbol_table_name + constants.BANG + symbol_basename + + @classmethod + def _do_get_path(cls, mnt, fs_root) -> Union[None, str]: + """It mimics the Linux kernel prepend_path function.""" + vfsmnt = mnt.mnt + dentry = vfsmnt.get_mnt_root() + + path_reversed = [] + while dentry != fs_root.dentry or vfsmnt.vol.offset != fs_root.mnt: + if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): + parent = mnt.get_mnt_parent().dereference() + # Escaped? + if dentry != vfsmnt.get_mnt_root(): + return None + + # Global root? + if mnt.vol.offset != parent.vol.offset: + dentry = mnt.get_mnt_mountpoint() + mnt = parent + vfsmnt = mnt.mnt + continue + + return None + + parent = dentry.d_parent + dname = dentry.d_name.name_as_str() + path_reversed.append(dname.strip("/")) + dentry = parent + + path = "/" + "/".join(reversed(path_reversed)) + return path + + @classmethod + def get_mountinfo(cls, mnt, task) -> Union[None, Tuple[int, int, str, str, str, List[str], + List[str], str, str, List[str]]]: + """Extract various information about a mount point. + It mimics the Linux kernel show_mountinfo function. + """ + mnt_root = mnt.get_mnt_root() + if not mnt_root: + return None + + mnt_root_path = mnt_root.path() + superblock = mnt.get_mnt_sb() + + mnt_id: int = mnt.mnt_id + parent_id: int = mnt.mnt_parent.mnt_id + + st_dev = f"{superblock.major}:{superblock.minor}" + + path_root = cls._do_get_path(mnt, task.fs.root) + if path_root is None: + return None + + mnt_opts: List[str] = [] + mnt_opts.append(mnt.get_flags_access()) + mnt_opts.extend(mnt.get_flags_opts()) + + # Tagged fields + fields: List[str] = [] + if mnt.is_shared(): + fields.append(f"shared:{mnt.mnt_group_id}") + + if mnt.is_slave(): + master = mnt.mnt_master.mnt_group_id + fields.append(f"master:{master}") + dominating_id = mnt.get_dominating_id(task.fs.root) + if dominating_id and dominating_id != master: + fields.append(f"propagate_from:{dominating_id}") + + if mnt.is_unbindable(): + fields.append("unbindable") + + mnt_type = superblock.get_type() + + devname = mnt.get_devname() + if not devname: + devname = "none" + + sb_opts: List[str] = [] + sb_opts.append(superblock.get_flags_access()) + sb_opts.extend(superblock.get_flags_opts()) + + return MountInfoData(mnt_id, parent_id, st_dev, mnt_root_path, path_root, mnt_opts, fields, + mnt_type, devname, sb_opts) + + def _get_mnt_namespace_mountpoints(self, mnt_namespace): + mnt_type = self._get_symbol_fullname("mount") + if not self.context.symbol_space.has_type(mnt_type): + # Old kernels ~ 2.6 + mnt_type = self._get_symbol_fullname("vfsmount") + + for mount in mnt_namespace.list.to_list(mnt_type, "mnt_list"): + yield mount + + def _get_tasks_mountpoints(self, pids: Iterable[int]): + self._vmlinux = self.context.modules[self.config['kernel']] + + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks(self.context, self.config['kernel'], filter_func=pid_filter) + + seen_namespaces = set() + for task in tasks: + if not (task and task.fs and task.fs.root and task.nsproxy and task.nsproxy.mnt_ns): + # This task doesn't have all the information required + continue + + mnt_namespace = task.nsproxy.mnt_ns + mount_ns_id = mnt_namespace.get_inode() + + if self._show_mountpoints_per_namespace: + if mount_ns_id in seen_namespaces: + continue + else: + seen_namespaces.add(mount_ns_id) + + for mount in self._get_mnt_namespace_mountpoints(mnt_namespace): + yield task, mount, mount_ns_id + + def _generator(self): + pids = self.config.get('pids') + + for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(pids=pids): + mnt_info = self.get_mountinfo(mnt, task) + if mnt_info is None: + continue + + if self.config.get('mount-format'): + all_opts = set() + all_opts.update(mnt_info.mnt_opts) + all_opts.update(mnt_info.sb_opts) + all_opts_str = ",".join(all_opts) + + extra_fields_values = [mnt_info.devname, mnt_info.path_root, mnt_info.mnt_type, all_opts_str] + else: + mnt_opts_str = ",".join(mnt_info.mnt_opts) + fields_str = " ".join(mnt_info.fields) + sb_opts_str = ",".join(mnt_info.sb_opts) + + extra_fields_values = [mnt_info.mnt_id, mnt_info.parent_id, mnt_info.st_dev, mnt_info.mnt_root_path, + mnt_info.path_root, mnt_opts_str, fields_str, mnt_info.mnt_type, + mnt_info.devname, sb_opts_str] + + fields_values = [mnt_ns_id] + if not self._show_mountpoints_per_namespace: + fields_values.append(task.pid) + fields_values.extend(extra_fields_values) + + yield (0, fields_values) + + def run(self): + if self.config.get('all-processes') and self.config.get('pids'): + raise ValueError("Unable to use --all-processes and specified a pid") + + # When no arguments are specified, it displays the mountpoints per namespace + self._show_mountpoints_per_namespace = not any([self.config.get('pids'), self.config.get('all-processes')]) + + columns = [("MNT_NS_ID", int)] + if not self._show_mountpoints_per_namespace: + columns.append(("PID", int)) + + if self.config.get('mount-format'): + extra_columns = [("DEVNAME", str), ("PATH", str), ("FSTYPE", str), ("MNT_OPTS", str)] + else: + # /proc/[pid]/mountinfo output format + extra_columns = [("MOUNT ID", int), ("PARENT_ID", int), ("MAJOR:MINOR", str), ("ROOT", str), + ("MOUNT_POINT", str), ("MOUNT_OPTIONS", str), ("FIELDS", str), ("FSTYPE", str), + ("MOUNT_SRC", str), ("SB_OPTIONS", str)] + + columns.extend(extra_columns) + + return renderers.TreeGrid(columns, self._generator()) \ No newline at end of file diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 36e23a35d..5b0bfcf40 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -29,6 +29,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('files_struct', extensions.files_struct) self.set_type_class('vfsmount', extensions.vfsmount) self.set_type_class('kobject', extensions.kobject) + self.set_type_class('mnt_namespace', extensions.mnt_namespace) if 'module' in self.types: self.set_type_class('module', extensions.module) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0edd60608..3a237d280 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -246,6 +246,29 @@ class super_block(objects.StructType): # include/linux/kdev_t.h MINORBITS = 20 + # Superblock flags + SB_RDONLY = 1 # Mount read-only + SB_NOSUID = 2 # Ignore suid and sgid bits + SB_NODEV = 4 # Disallow access to device special files + SB_NOEXEC = 8 # Disallow program execution + SB_SYNCHRONOUS = 16 # Writes are synced at once + SB_MANDLOCK = 64 # Allow mandatory locks on an FS + SB_DIRSYNC = 128 # Directory modifications are synchronous + SB_NOATIME = 1024 # Do not update access times + SB_NODIRATIME = 2048 # Do not update directory access times + SB_SILENT = 32768 + SB_POSIXACL = (1 << 16) # VFS does not apply the umask + SB_KERNMOUNT = (1 << 22) # this is a kern_mount call + SB_I_VERSION = (1 << 23) # Update inode I_version field + SB_LAZYTIME = (1 << 25) # Update the on-disk [acm]times lazily + + SB_OPTS = { + SB_SYNCHRONOUS: "sync", + SB_DIRSYNC: "dirsync", + SB_MANDLOCK: "mand", + SB_LAZYTIME: "lazytime" + } + @property def major(self) -> int: return self.s_dev >> self.MINORBITS @@ -254,6 +277,20 @@ class super_block(objects.StructType): def minor(self) -> int: return self.s_dev & ((1 << self.MINORBITS) - 1) + def get_flags_access(self) -> str: + return 'ro' if self.s_flags & self.SB_RDONLY else 'rw' + + def get_flags_opts(self) -> Iterable[str]: + sb_opts = [self.SB_OPTS[sb_opt] for sb_opt in self.SB_OPTS if sb_opt & self.s_flags] + return sb_opts + + def get_type(self): + mnt_sb_type = utility.pointer_to_string(self.s_type.name, count=255) + if self.s_subtype: + mnt_sb_subtype = utility.pointer_to_string(self.s_subtype, count=255) + mnt_sb_type += "." + mnt_sb_subtype + return mnt_sb_type + class vm_area_struct(objects.StructType): perm_flags = { @@ -373,7 +410,44 @@ class qstr(objects.StructType): class dentry(objects.StructType): def path(self) -> str: - return self.d_name.name_as_str() + """ Based on __dentry_path Linux kernel function""" + reversed_path = [] + current_dentry = self + while not current_dentry.is_root(): + parent = current_dentry.d_parent + reversed_path.append(current_dentry.d_name.name_as_str()) + current_dentry = parent + return "/" + "/".join(reversed(reversed_path)) + + def is_root(self) -> bool: + return self.vol.offset == self.d_parent + + def is_subdir(self, old_dentry): + """Is this dentry a subdirectory of old_dentry? + + Returns true if this dentry is a subdirectory of the parent (at any depth). + Otherwise, it returns false. + """ + if self.vol.offset == old_dentry: + return True + + return self.d_ancestor(old_dentry) + + def d_ancestor(self, ancestor_dentry): + """Search for an ancestor + + Returns the ancestor dentry which is a child of "ancestor_dentry", + if "ancestor_dentry" is an ancestor of "child_dentry", else None. + """ + + current_dentry = self + while not current_dentry.is_root(): + if current_dentry.d_parent == ancestor_dentry.vol.offset: + return current_dentry + + current_dentry = current_dentry.d_parent + + return None class struct_file(objects.StructType): @@ -468,6 +542,27 @@ class files_struct(objects.StructType): class mount(objects.StructType): + MNT_NOSUID = 0x01 + MNT_NODEV = 0x02 + MNT_NOEXEC = 0x04 + MNT_NOATIME = 0x08 + MNT_NODIRATIME = 0x10 + MNT_RELATIME = 0x20 + MNT_READONLY = 0x40 + MNT_SHRINKABLE = 0x100 + MNT_WRITE_HOLD = 0x200 + MNT_SHARED = 0x1000 + MNT_UNBINDABLE = 0x2000 + + MNT_FLAGS = { + MNT_NOSUID: "nosuid", + MNT_NODEV: "nodev", + MNT_NOEXEC: "noexec", + MNT_NOATIME: "noatime", + MNT_NODIRATIME: "nodiratime", + MNT_RELATIME: "relatime", + } + def get_mnt_sb(self): if self.has_member("mnt"): return self.mnt.mnt_sb @@ -498,6 +593,66 @@ class mount(objects.StructType): def get_mnt_mountpoint(self): return self.mnt_mountpoint + def get_flags_access(self) -> str: + return "ro" if self.get_mnt_flags() & self.MNT_READONLY else "rw" + + def get_flags_opts(self) -> Iterable[str]: + flags = [self.MNT_FLAGS[mntflag] for mntflag in self.MNT_FLAGS if mntflag & self.get_mnt_flags()] + return flags + + def is_shared(self) -> bool: + return self.get_mnt_flags() & self.MNT_SHARED + + def is_unbindable(self) -> bool: + return self.get_mnt_flags() & self.MNT_UNBINDABLE + + def is_slave(self) -> bool: + return self.mnt_master and self.mnt_master.vol.offset != 0 + + def get_devname(self) -> str: + return utility.pointer_to_string(self.mnt_devname, count=255) + + def has_parent(self) -> bool: + return self.vol.offset != self.mnt_parent + + def get_dominating_id(self, root) -> int: + """Get ID of closest dominating peer group having a representative under the given root.""" + current_mnt = self.mnt_master + while current_mnt and current_mnt.vol.offset != 0: + peer = current_mnt.get_peer_under_root(self.mnt_ns, root) + if peer and peer.vol.offset != 0: + return peer.mnt_group_id + + current_mnt = current_mnt.mnt_master + return 0 + + def get_peer_under_root(self, ns, root): + current = self + while True: + if current.mnt_ns == ns and current.is_path_reachable(current.mnt.mnt_root, root): + return current + current = current.next_peer() + if current.vol.offset == self.vol.offset: + break + + return None + + def is_path_reachable(self, current_dentry, root): + """Return true if path is reachable + """ + current_mnt = self + while current_mnt.mnt.vol.offset != root.mnt and current_mnt.has_parent(): + current_dentry = current_mnt.mnt_mountpoint + current_mnt = current_mnt.mnt_parent + + return current_mnt.mnt.vol.offset == root.mnt and current_dentry.is_subdir(root.dentry) + + def next_peer(self): + table_name = self.vol.type_name.split(constants.BANG)[0] + mount_struct = "{0}{1}mount".format(table_name, constants.BANG) + offset = self._context.symbol_space.get_type(mount_struct).relative_child_offset("mnt_share") + + return self._context.object(mount_struct, self.vol.layer_name, offset=self.mnt_share.next.vol.offset - offset) class vfsmount(objects.StructType): @@ -539,3 +694,12 @@ class kobject(objects.StructType): ret = refcnt.refs.counter return ret + +class mnt_namespace(objects.StructType): + def get_inode(self): + if self.has_member("proc_inum"): + return self.proc_inum + elif self.ns.has_member("inum"): + return self.ns.inum + else: + raise AttributeError("Unable to find mnt_namespace inode") From a1d145c34f60db7f68c3949ba642466502403cd2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 22:10:52 +1100 Subject: [PATCH 003/181] Cleaning space issues --- .../framework/plugins/linux/mountinfo.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index be423a2f4..10869a45b 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -14,14 +14,14 @@ from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) -MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "mnt_root_path", "path_root", +MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "mnt_root_path", "path_root", "mnt_opts", "fields", "mnt_type", "devname", "sb_opts")) class MountInfo(plugins.PluginInterface): """Lists mount points in processes mount namespaces""" _required_framework_version = (2, 0, 0) - + _version = (2, 0, 0) @classmethod @@ -86,7 +86,7 @@ class MountInfo(plugins.PluginInterface): return path @classmethod - def get_mountinfo(cls, mnt, task) -> Union[None, Tuple[int, int, str, str, str, List[str], + def get_mountinfo(cls, mnt, task) -> Union[None, Tuple[int, int, str, str, str, List[str], List[str], str, str, List[str]]]: """Extract various information about a mount point. It mimics the Linux kernel show_mountinfo function. @@ -136,7 +136,7 @@ class MountInfo(plugins.PluginInterface): sb_opts.append(superblock.get_flags_access()) sb_opts.extend(superblock.get_flags_opts()) - return MountInfoData(mnt_id, parent_id, st_dev, mnt_root_path, path_root, mnt_opts, fields, + return MountInfoData(mnt_id, parent_id, st_dev, mnt_root_path, path_root, mnt_opts, fields, mnt_type, devname, sb_opts) def _get_mnt_namespace_mountpoints(self, mnt_namespace): @@ -147,13 +147,13 @@ class MountInfo(plugins.PluginInterface): for mount in mnt_namespace.list.to_list(mnt_type, "mnt_list"): yield mount - + def _get_tasks_mountpoints(self, pids: Iterable[int]): self._vmlinux = self.context.modules[self.config['kernel']] - + pid_filter = pslist.PsList.create_pid_filter(pids) tasks = pslist.PsList.list_tasks(self.context, self.config['kernel'], filter_func=pid_filter) - + seen_namespaces = set() for task in tasks: if not (task and task.fs and task.fs.root and task.nsproxy and task.nsproxy.mnt_ns): @@ -162,13 +162,13 @@ class MountInfo(plugins.PluginInterface): mnt_namespace = task.nsproxy.mnt_ns mount_ns_id = mnt_namespace.get_inode() - + if self._show_mountpoints_per_namespace: if mount_ns_id in seen_namespaces: continue else: seen_namespaces.add(mount_ns_id) - + for mount in self._get_mnt_namespace_mountpoints(mnt_namespace): yield task, mount, mount_ns_id @@ -186,34 +186,34 @@ class MountInfo(plugins.PluginInterface): all_opts.update(mnt_info.sb_opts) all_opts_str = ",".join(all_opts) - extra_fields_values = [mnt_info.devname, mnt_info.path_root, mnt_info.mnt_type, all_opts_str] + extra_fields_values = [mnt_info.devname, mnt_info.path_root, mnt_info.mnt_type, all_opts_str] else: mnt_opts_str = ",".join(mnt_info.mnt_opts) fields_str = " ".join(mnt_info.fields) sb_opts_str = ",".join(mnt_info.sb_opts) - extra_fields_values = [mnt_info.mnt_id, mnt_info.parent_id, mnt_info.st_dev, mnt_info.mnt_root_path, - mnt_info.path_root, mnt_opts_str, fields_str, mnt_info.mnt_type, + extra_fields_values = [mnt_info.mnt_id, mnt_info.parent_id, mnt_info.st_dev, mnt_info.mnt_root_path, + mnt_info.path_root, mnt_opts_str, fields_str, mnt_info.mnt_type, mnt_info.devname, sb_opts_str] - + fields_values = [mnt_ns_id] if not self._show_mountpoints_per_namespace: fields_values.append(task.pid) fields_values.extend(extra_fields_values) - + yield (0, fields_values) def run(self): if self.config.get('all-processes') and self.config.get('pids'): raise ValueError("Unable to use --all-processes and specified a pid") - + # When no arguments are specified, it displays the mountpoints per namespace self._show_mountpoints_per_namespace = not any([self.config.get('pids'), self.config.get('all-processes')]) columns = [("MNT_NS_ID", int)] if not self._show_mountpoints_per_namespace: columns.append(("PID", int)) - + if self.config.get('mount-format'): extra_columns = [("DEVNAME", str), ("PATH", str), ("FSTYPE", str), ("MNT_OPTS", str)] else: @@ -221,7 +221,7 @@ class MountInfo(plugins.PluginInterface): extra_columns = [("MOUNT ID", int), ("PARENT_ID", int), ("MAJOR:MINOR", str), ("ROOT", str), ("MOUNT_POINT", str), ("MOUNT_OPTIONS", str), ("FIELDS", str), ("FSTYPE", str), ("MOUNT_SRC", str), ("SB_OPTIONS", str)] - + columns.extend(extra_columns) - return renderers.TreeGrid(columns, self._generator()) \ No newline at end of file + return renderers.TreeGrid(columns, self._generator()) From fcf9983d5127370b67b350e72fb7e8363ff3b283 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Dec 2021 13:48:46 +1100 Subject: [PATCH 004/181] Removed --all-process and added --mntns. --- .../framework/plugins/linux/mountinfo.py | 87 +++++++++---------- .../symbols/linux/extensions/__init__.py | 20 +++-- 2 files changed, 55 insertions(+), 52 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 10869a45b..a3006d47e 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -7,7 +7,7 @@ import logging from collections import namedtuple from typing import Tuple, List, Iterable, Union -from volatility3.framework import renderers, interfaces, constants +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.plugins.linux import pslist @@ -18,11 +18,11 @@ MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "m "mnt_opts", "fields", "mnt_type", "devname", "sb_opts")) class MountInfo(plugins.PluginInterface): - """Lists mount points in processes mount namespaces""" + """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -35,25 +35,19 @@ class MountInfo(plugins.PluginInterface): description="Filter on specific process IDs.", element_type=int, optional=True), - requirements.BooleanRequirement(name="all-processes", - description="Shows information about mount points for each process mount " - "namespace. It could take a while depending on the number of processes " - "running. Note that if this argument is not specified it uses the root " - "mount namespace based on pid 1.", - optional=True, - default=False), + requirements.ListRequirement(name="mntns", + description="Filter results by mount namespace. " + "Otherwise, all of them are shown.", + element_type=int, + optional=True), requirements.BooleanRequirement(name="mount-format", - description="Shows a brief summary of a process mount points information " + description="Shows a brief summary of the mount points information " "with similar output format to the older /proc/[pid]/mounts or the " "user-land command 'mount -l'.", optional=True, default=False), ] - def _get_symbol_fullname(self, symbol_basename: str) -> str: - """Given a short symbol or type name, it returns its full name""" - return self._vmlinux.symbol_table_name + constants.BANG + symbol_basename - @classmethod def _do_get_path(cls, mnt, fs_root) -> Union[None, str]: """It mimics the Linux kernel prepend_path function.""" @@ -139,21 +133,7 @@ class MountInfo(plugins.PluginInterface): return MountInfoData(mnt_id, parent_id, st_dev, mnt_root_path, path_root, mnt_opts, fields, mnt_type, devname, sb_opts) - def _get_mnt_namespace_mountpoints(self, mnt_namespace): - mnt_type = self._get_symbol_fullname("mount") - if not self.context.symbol_space.has_type(mnt_type): - # Old kernels ~ 2.6 - mnt_type = self._get_symbol_fullname("vfsmount") - - for mount in mnt_namespace.list.to_list(mnt_type, "mnt_list"): - yield mount - - def _get_tasks_mountpoints(self, pids: Iterable[int]): - self._vmlinux = self.context.modules[self.config['kernel']] - - pid_filter = pslist.PsList.create_pid_filter(pids) - tasks = pslist.PsList.list_tasks(self.context, self.config['kernel'], filter_func=pid_filter) - + def _get_tasks_mountpoints(self, tasks: Iterable[interfaces.objects.ObjectInterface], per_namespace: bool): seen_namespaces = set() for task in tasks: if not (task and task.fs and task.fs.root and task.nsproxy and task.nsproxy.mnt_ns): @@ -161,26 +141,33 @@ class MountInfo(plugins.PluginInterface): continue mnt_namespace = task.nsproxy.mnt_ns - mount_ns_id = mnt_namespace.get_inode() + mnt_ns_id = mnt_namespace.get_inode() - if self._show_mountpoints_per_namespace: - if mount_ns_id in seen_namespaces: + if per_namespace: + if mnt_ns_id in seen_namespaces: continue else: - seen_namespaces.add(mount_ns_id) + seen_namespaces.add(mnt_ns_id) - for mount in self._get_mnt_namespace_mountpoints(mnt_namespace): - yield task, mount, mount_ns_id + for mount in mnt_namespace.get_mount_points(): + yield task, mount, mnt_ns_id - def _generator(self): - pids = self.config.get('pids') + def _generator( + self, + tasks: Iterable[interfaces.objects.ObjectInterface], + mnt_ns_ids: List[int], + mount_format: bool, + per_namespace: bool) -> Iterable[Tuple[int, Tuple]]: + + for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, per_namespace): + if mnt_ns_ids and mnt_ns_id not in mnt_ns_ids: + continue - for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(pids=pids): mnt_info = self.get_mountinfo(mnt, task) if mnt_info is None: continue - if self.config.get('mount-format'): + if mount_format: all_opts = set() all_opts.update(mnt_info.mnt_opts) all_opts.update(mnt_info.sb_opts) @@ -197,22 +184,28 @@ class MountInfo(plugins.PluginInterface): mnt_info.devname, sb_opts_str] fields_values = [mnt_ns_id] - if not self._show_mountpoints_per_namespace: + if not per_namespace: fields_values.append(task.pid) fields_values.extend(extra_fields_values) yield (0, fields_values) def run(self): - if self.config.get('all-processes') and self.config.get('pids'): - raise ValueError("Unable to use --all-processes and specified a pid") + pids = self.config.get('pids') + mount_ns_ids = self.config.get('mntns') + mount_format = self.config.get('mount-format') - # When no arguments are specified, it displays the mountpoints per namespace - self._show_mountpoints_per_namespace = not any([self.config.get('pids'), self.config.get('all-processes')]) + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks(self.context, self.config['kernel'], filter_func=pid_filter) columns = [("MNT_NS_ID", int)] - if not self._show_mountpoints_per_namespace: + # The PID column does not make sense when a PID filter is not specified. In that case, the default behavior is + # to displays the mountpoints per namespace. + if pids: columns.append(("PID", int)) + per_namespace = False + else: + per_namespace = True if self.config.get('mount-format'): extra_columns = [("DEVNAME", str), ("PATH", str), ("FSTYPE", str), ("MNT_OPTS", str)] @@ -224,4 +217,4 @@ class MountInfo(plugins.PluginInterface): columns.extend(extra_columns) - return renderers.TreeGrid(columns, self._generator()) + return renderers.TreeGrid(columns, self._generator(tasks, mount_ns_ids, mount_format, per_namespace)) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3a237d280..23967dd25 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -555,12 +555,12 @@ class mount(objects.StructType): MNT_UNBINDABLE = 0x2000 MNT_FLAGS = { - MNT_NOSUID: "nosuid", - MNT_NODEV: "nodev", - MNT_NOEXEC: "noexec", - MNT_NOATIME: "noatime", + MNT_NOSUID: "nosuid", + MNT_NODEV: "nodev", + MNT_NOEXEC: "noexec", + MNT_NOATIME: "noatime", MNT_NODIRATIME: "nodiratime", - MNT_RELATIME: "relatime", + MNT_RELATIME: "relatime", } def get_mnt_sb(self): @@ -703,3 +703,13 @@ class mnt_namespace(objects.StructType): return self.ns.inum else: raise AttributeError("Unable to find mnt_namespace inode") + + def get_mount_points(self): + table_name = self.vol.type_name.split(constants.BANG)[0] + mnt_type = table_name + constants.BANG + "mount" + if not self._context.symbol_space.has_type(mnt_type): + # Old kernels ~ 2.6 + mnt_type = table_name + constants.BANG + "vfsmount" + + for mount in self.list.to_list(mnt_type, "mnt_list"): + yield mount From f23b288ffcc15234eb25e88ce82df86f7285e0ea Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Dec 2021 17:52:07 +1100 Subject: [PATCH 005/181] Bump framework and plugin required minor version --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/linux/mountinfo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 23598837b..09a2e4820 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 0 # Number of changes that only add to the interface +VERSION_MINOR = 1 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index a3006d47e..6c0f8bcc3 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -20,7 +20,7 @@ MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "m class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 1, 0) _version = (1, 0, 0) From 338fcdd0ab7f59491017ed63d8e780e744c0fe88 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Wed, 2 Feb 2022 15:48:41 +0000 Subject: [PATCH 006/181] Add the psaux plugin for Linux command line argument listing --- volatility3/framework/plugins/linux/psaux.py | 90 ++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 volatility3/framework/plugins/linux/psaux.py diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py new file mode 100644 index 000000000..31777f458 --- /dev/null +++ b/volatility3/framework/plugins/linux/psaux.py @@ -0,0 +1,90 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from typing import Optional + +from volatility3.framework import symbols, exceptions, renderers, interfaces +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist + +class PsAux(pslist.PsList): + """ Lists processes with their command line arguments """ + + def _get_command_line_args(self, task: interfaces.objects.ObjectInterface, + name: str) -> Optional[str]: + """ + Reads the command line arguments of a process + These are stored on the userland stack + Kernel threads re-use the process data structure, but do not have a valid 'mm' pointer + + Parameters: + task: task_struct object of the process + name: string name of the process (from task.comm) + """ + + # kernel theads never have an mm as they do not have userland mappings + try: + mm = task.mm + except exceptions.InvalidAddressException: + mm = None + + if mm: + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + return renderers.UnreadableValue() + + proc_layer = self.context.layers[proc_layer_name] + + # read argv from userland + start = task.mm.arg_start + + # get the size of the arguments with sanity checking + size_to_read = task.mm.arg_end - task.mm.arg_start + if size_to_read < 1 or size_to_read > 4096: + return renderers.UnreadableValue() + + # attempt to read it all as partial values are invalid and misleading + try: + argv = proc_layer.read(start, size_to_read) + except exceptions.InvalidAddressException: + return renderers.UnreadableValue() + + # the arguments are null byte terminated, replace the nulls with spaces + s = argv.decode().split('\x00') + args = " ".join(s) + else: + # kernel thread + # [ ] mimics ps on a live system + # also helps identify malware masquerading as a kernel thread, which is fairly common + args = "[" + name + "]" + + # remove trailing space, if present + if len(args) > 1 and args[-1] == " ": + args = args[:-1] + + return args + + def _generator(self): + """ Generates a listing of processes along with command line arguments """ + + vmlinux = self.context.modules[self.config['kernel']] + + # walk the process list and report the arguments + for task in self.list_tasks(self.context, vmlinux.name): + pid = task.pid + + try: + ppid = task.parent.pid + except exceptions.InvalidAddressException: + ppid = 0 + + name = utility.array_to_string(task.comm) + + args = self._get_command_line_args(task, name) + + yield (0, (pid, ppid, name, args)) + + def run(self): + return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], self._generator()) + From 72ebf11fd36ff6d0d942181713529c25a02f55f5 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 24 Apr 2022 15:13:55 +0300 Subject: [PATCH 007/181] support CallbackListHead when CmpCallBackVector not present --- .../framework/plugins/windows/callbacks.py | 75 ++++++++++++++----- .../symbols/windows/callbacks-x64.json | 43 +++++++++++ .../symbols/windows/callbacks-x86.json | 43 +++++++++++ 3 files changed, 143 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 352dba448..d8e5aea86 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -111,30 +111,19 @@ class Callbacks(interfaces.plugins.PluginInterface): yield symbol_name, callback.Callback, None @classmethod - def list_registry_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - callback_table_name: str) -> Iterable[Tuple[str, int, None]]: - """Lists all registry callbacks. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - symbol_table: The name of the table containing the kernel symbols - callback_table_name: The nae of the table containing the callback symbols - - Yields: - A name, location and optional detail string + def _list_registry_callbacks_legacy(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, + callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + """ + Lists all registry callbacks from the old format via the CmpCallBackVector. """ kvo = context.layers[layer_name].config['kernel_virtual_offset'] ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) full_type_name = callback_table_name + constants.BANG + "_EX_CALLBACK_ROUTINE_BLOCK" - try: - symbol_offset = ntkrnlmp.get_symbol("CmpCallBackVector").address - symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address - except exceptions.SymbolError: - vollog.debug("Cannot find CmpCallBackVector or CmpCallBackCount") - return + symbol_offset = ntkrnlmp.get_symbol("CmpCallBackVector").address + symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address + callback_count = ntkrnlmp.object(object_type = "unsigned int", offset = symbol_count_offset) @@ -155,6 +144,56 @@ class Callbacks(interfaces.plugins.PluginInterface): if callback.Function != 0: yield "CmRegisterCallback", callback.Function, None + @classmethod + def _list_registry_callbacks_new(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, + callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + """ + Lists all registry callbacks via the CallbackListHead. + """ + + kvo = context.layers[layer_name].config['kernel_virtual_offset'] + ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + full_type_name = callback_table_name + constants.BANG + "_CM_CALLBACK_ENTRY" + + symbol_offset = ntkrnlmp.get_symbol("CallbackListHead").address + symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address + + callback_count = ntkrnlmp.object(object_type = "unsigned int", offset = symbol_count_offset) + + if callback_count == 0: + return + + callback_list = ntkrnlmp.object(object_type = "_LIST_ENTRY", offset = symbol_offset) + for callback in callback_list.to_list(full_type_name, "Link"): + yield "CmRegisterCallbackEx", callback.Function, f"Alltitude: {callback.Alltitude.String}" + + @classmethod + def list_registry_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, + callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + """Lists all registry callbacks. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + callback_table_name: The nae of the table containing the callback symbols + + Yields: + A name, location and optional detail string + """ + + kvo = context.layers[layer_name].config['kernel_virtual_offset'] + ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + full_type_name = callback_table_name + constants.BANG + "_EX_CALLBACK_ROUTINE_BLOCK" + + if ntkrnlmp.has_symbol("CmpCallBackVector") and ntkrnlmp.has_symbol("CmpCallBackCount"): + yield from cls._list_registry_callbacks_legacy(context, layer_name, symbol_table, callback_table_name) + elif ntkrnlmp.has_symbol("CallbackListHead") and ntkrnlmp.has_symbol("CmpCallBackCount"): + yield from cls._list_registry_callbacks_new(context, layer_name, symbol_table, callback_table_name) + else: + vollog.debug("Cannot find CmpCallBackVector or CmpCallBackCount or CallbackListHead") + return + @classmethod def list_bugcheck_reason_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, callback_table_name: str) -> Iterable[Tuple[str, int, str]]: diff --git a/volatility3/framework/symbols/windows/callbacks-x64.json b/volatility3/framework/symbols/windows/callbacks-x64.json index dbb6086df..5300d28f9 100644 --- a/volatility3/framework/symbols/windows/callbacks-x64.json +++ b/volatility3/framework/symbols/windows/callbacks-x64.json @@ -8,6 +8,12 @@ "signed": false, "endian": "little" }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, "unsigned char": { "kind": "char", "size": 1, @@ -137,6 +143,43 @@ }, "kind": "struct", "size": 64 + }, + "_CM_CALLBACK_ENTRY": { + "fields": { + "Link": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Cookie": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 24 + }, + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "Alltitude": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 64 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/callbacks-x86.json b/volatility3/framework/symbols/windows/callbacks-x86.json index cf0cb8b65..52baeb18f 100644 --- a/volatility3/framework/symbols/windows/callbacks-x86.json +++ b/volatility3/framework/symbols/windows/callbacks-x86.json @@ -8,6 +8,12 @@ "signed": false, "endian": "little" }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, "unsigned char": { "kind": "char", "size": 1, @@ -137,6 +143,43 @@ }, "kind": "struct", "size": 28 + }, + "_CM_CALLBACK_ENTRY": { + "fields": { + "Link": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Cookie": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 16 + }, + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 28 + }, + "Alltitude": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 40 } }, "metadata": { From bc04d22a1b7e2969f1ddd5ec6598439724c17bc7 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 24 Apr 2022 16:42:25 +0300 Subject: [PATCH 008/181] remove unused line --- volatility3/framework/plugins/windows/callbacks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index d8e5aea86..c10d97405 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -184,7 +184,6 @@ class Callbacks(interfaces.plugins.PluginInterface): kvo = context.layers[layer_name].config['kernel_virtual_offset'] ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) - full_type_name = callback_table_name + constants.BANG + "_EX_CALLBACK_ROUTINE_BLOCK" if ntkrnlmp.has_symbol("CmpCallBackVector") and ntkrnlmp.has_symbol("CmpCallBackCount"): yield from cls._list_registry_callbacks_legacy(context, layer_name, symbol_table, callback_table_name) From 0057f81269b382fdaa9f15922ec41f0cd1f31faa Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 25 Apr 2022 09:27:54 +0300 Subject: [PATCH 009/181] log which symbol does not exist --- volatility3/framework/plugins/windows/callbacks.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index c10d97405..5ee11f164 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -190,7 +190,14 @@ class Callbacks(interfaces.plugins.PluginInterface): elif ntkrnlmp.has_symbol("CallbackListHead") and ntkrnlmp.has_symbol("CmpCallBackCount"): yield from cls._list_registry_callbacks_new(context, layer_name, symbol_table, callback_table_name) else: - vollog.debug("Cannot find CmpCallBackVector or CmpCallBackCount or CallbackListHead") + symbols_to_check = ["CmpCallBackVector", "CmpCallBackCount", "CallbackListHead"] + vollog.debug("Failed to get registry callbacks!") + for symbol_name in symbols_to_check: + symbol_status = "does not exist" + if ntkrnlmp.has_symbol(symbol_name): + symbol_status = "exists" + vollog.debug(f"symbol {symbol_name} {symbol_status}.") + return @classmethod From a8d70c065738b3c3691da3ec2a82a6d4a7ca24a6 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Tue, 26 Apr 2022 10:11:12 +0300 Subject: [PATCH 010/181] fix typo --- volatility3/framework/plugins/windows/callbacks.py | 2 +- volatility3/framework/symbols/windows/callbacks-x64.json | 2 +- volatility3/framework/symbols/windows/callbacks-x86.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 5ee11f164..dca17aff7 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -165,7 +165,7 @@ class Callbacks(interfaces.plugins.PluginInterface): callback_list = ntkrnlmp.object(object_type = "_LIST_ENTRY", offset = symbol_offset) for callback in callback_list.to_list(full_type_name, "Link"): - yield "CmRegisterCallbackEx", callback.Function, f"Alltitude: {callback.Alltitude.String}" + yield "CmRegisterCallbackEx", callback.Function, f"Altitude: {callback.Altitude.String}" @classmethod def list_registry_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, diff --git a/volatility3/framework/symbols/windows/callbacks-x64.json b/volatility3/framework/symbols/windows/callbacks-x64.json index 5300d28f9..87682cb92 100644 --- a/volatility3/framework/symbols/windows/callbacks-x64.json +++ b/volatility3/framework/symbols/windows/callbacks-x64.json @@ -170,7 +170,7 @@ }, "offset": 40 }, - "Alltitude": { + "Altitude": { "type": { "kind": "struct", "name": "nt_symbols!_UNICODE_STRING" diff --git a/volatility3/framework/symbols/windows/callbacks-x86.json b/volatility3/framework/symbols/windows/callbacks-x86.json index 52baeb18f..702b68a65 100644 --- a/volatility3/framework/symbols/windows/callbacks-x86.json +++ b/volatility3/framework/symbols/windows/callbacks-x86.json @@ -170,7 +170,7 @@ }, "offset": 28 }, - "Alltitude": { + "Altitude": { "type": { "kind": "struct", "name": "nt_symbols!_UNICODE_STRING" From 5679135f1aeb5ed82421eee37f3a57f6c0f97c53 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 08:12:06 +1000 Subject: [PATCH 011/181] The exception message is not accurate In the hypothetical case that the framework has a lower version than the module requirement, the message would be, for instance: 'Framework interface version 2 is an older revision than the required version 2.2' instead of: 'Framework interface version 2.1 is an older revision than the required version 2.2' See https://github.com/volatilityfoundation/volatility3/pull/593#discussion_r769238045 --- volatility3/framework/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 2c9444130..176eb2242 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -51,7 +51,7 @@ def require_interface_version(*args) -> None: if args[1] > interface_version()[1]: raise RuntimeError( "Framework interface version {} is an older revision than the required version {}".format( - ".".join([str(x) for x in interface_version()[0:1]]), ".".join([str(x) for x in args[0:2]]))) + ".".join([str(x) for x in interface_version()[0:2]]), ".".join([str(x) for x in args[0:2]]))) class NonInheritable(object): From a59de1b918ec4bdbc4d7e5a3ead8f842ae2e84d5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 08:13:24 +1000 Subject: [PATCH 012/181] Updated to resolve merge conflict --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09a2e4820..2afaf84cc 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 1 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 4 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From e5494ce8c9fb370841a0ffe9c1098b84fef11318 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 08:52:25 +1000 Subject: [PATCH 013/181] Temporarely setting interface minor version to zero to allow to merge the upstream latest changes --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 2afaf84cc..abf537811 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 1 # Number of changes that only add to the interface +VERSION_MINOR = 0 # Number of changes that only add to the interface VERSION_PATCH = 4 # Number of changes that do not change the interface VERSION_SUFFIX = "" From c6fbb9ce14a68c82c07d38eddb70d61f6efd00c2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 08:58:05 +1000 Subject: [PATCH 014/181] Restoring framework minor version to meet the mountinfo plugin see f23b288ffcc15234eb25e88ce82df86f7285e0ea --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index abf537811..2afaf84cc 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 0 # Number of changes that only add to the interface +VERSION_MINOR = 1 # Number of changes that only add to the interface VERSION_PATCH = 4 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 73a42577d7cdbe42d8e501b25e2cb3682005be39 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 09:24:19 +1000 Subject: [PATCH 015/181] VERSION_PATCH has to be reset after VERSION_MINOR was increased. --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 21b3ac17b..5060906d5 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 1 # Number of changes that only add to the interface -VERSION_PATCH = 4 # Number of changes that do not change the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From b43d61ca036926047a13343eb401ad920cd5e62b Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Thu, 28 Apr 2022 15:42:10 +0000 Subject: [PATCH 016/181] Address feedback from ikelos --- volatility3/framework/plugins/linux/psaux.py | 30 ++++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 31777f458..089bb61c7 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -4,6 +4,7 @@ from typing import Optional +from volatility3.framework.configuration import requirements from volatility3.framework import symbols, exceptions, renderers, interfaces from volatility3.framework.objects import utility from volatility3.plugins.linux import pslist @@ -11,6 +12,19 @@ from volatility3.plugins.linux import pslist class PsAux(pslist.PsList): """ Lists processes with their command line arguments """ + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + description = 'Filter on specific process IDs', + element_type = int, + optional = True) + ] + def _get_command_line_args(self, task: interfaces.objects.ObjectInterface, name: str) -> Optional[str]: """ @@ -41,7 +55,7 @@ class PsAux(pslist.PsList): # get the size of the arguments with sanity checking size_to_read = task.mm.arg_end - task.mm.arg_start - if size_to_read < 1 or size_to_read > 4096: + if not (0 < size_to_read <= 4096): return renderers.UnreadableValue() # attempt to read it all as partial values are invalid and misleading @@ -65,13 +79,11 @@ class PsAux(pslist.PsList): return args - def _generator(self): + def _generator(self, tasks): """ Generates a listing of processes along with command line arguments """ - vmlinux = self.context.modules[self.config['kernel']] - # walk the process list and report the arguments - for task in self.list_tasks(self.context, vmlinux.name): + for task in tasks: pid = task.pid try: @@ -86,5 +98,11 @@ class PsAux(pslist.PsList): yield (0, (pid, ppid, name, args)) def run(self): - return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], self._generator()) + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + + return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], + self._generator( + pslist.PsList.list_tasks(self.context, + self.config['kernel'], + filter_func = filter_func))) From 3175e25420095f237fcc987c1efd970b8cfc3305 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Thu, 28 Apr 2022 16:11:51 +0000 Subject: [PATCH 017/181] Remove the inheritance from pslist --- volatility3/framework/plugins/linux/psaux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 089bb61c7..c62712907 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -8,10 +8,13 @@ from volatility3.framework.configuration import requirements from volatility3.framework import symbols, exceptions, renderers, interfaces from volatility3.framework.objects import utility from volatility3.plugins.linux import pslist +from volatility3.framework.interfaces import plugins -class PsAux(pslist.PsList): +class PsAux(plugins.PluginInterface): """ Lists processes with their command line arguments """ + _required_framework_version = (2, 0, 0) + @classmethod def get_requirements(cls): # Since we're calling the plugin, make sure we have the plugin's requirements From 78eb31014b0a97145582d8ae4beef598c2659b1f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 2 May 2022 00:00:33 +0900 Subject: [PATCH 018/181] Fix: get owning process method from _ETHREAD --- volatility3/framework/symbols/windows/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 7d083fbba..fd5e075da 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -448,9 +448,9 @@ class KMUTANT(objects.StructType, pool.ExecutiveObject): class ETHREAD(objects.StructType): """A class for executive thread objects.""" - def owning_process(self, kernel_layer: str = None) -> interfaces.objects.ObjectInterface: + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" - return self.ThreadsProcess.dereference(kernel_layer) + return self.Tcb.Process.dereference().cast("_EPROCESS") def get_cross_thread_flags(self) -> str: dictCrossThreadFlags = { From 51b901960169b5681f4a1cbebf03c19997685232 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 2 May 2022 10:34:04 +0900 Subject: [PATCH 019/181] Bump: patch version 2.1.1 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 5060906d5..e08bc42bc 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 1 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 6b7d5b63fe6d3fbef6fdcf339dc5c6b9426129e8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 6 May 2022 00:49:03 +0900 Subject: [PATCH 020/181] Add: venv environments, memory dump for .gitignore --- .gitignore | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitignore b/.gitignore index c6da33754..5986612ea 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,16 @@ config*.json # Pyinstaller files build dist + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Memory dump files +*.dmp +*.vmem From 16313c593989a9d5c42afa430194bfe248766c80 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 6 May 2022 01:18:31 +0900 Subject: [PATCH 021/181] Fix: typo for dlllist --- volatility3/framework/plugins/windows/dlllist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index b24f2c0ca..2fd7deeaf 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -65,7 +65,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): try: name = dll_entry.FullDllName.get_string() except exceptions.InvalidAddressException: - name = 'UnreadbleDLLName' + name = 'UnreadableDLLName' if layer_name is None: layer_name = dll_entry.vol.layer_name From 8b6194f8f4231ce8304366f2e985ca7e47f71e4e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 7 May 2022 18:16:43 +0900 Subject: [PATCH 022/181] Remove: environment .bak on .gitignore --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index 5986612ea..d26e17d91 100644 --- a/.gitignore +++ b/.gitignore @@ -34,8 +34,6 @@ dist env/ venv/ ENV/ -env.bak/ -venv.bak/ # Memory dump files *.dmp From 8f369180e4c1dcb8fafcc618b78450bcf36b7bb2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 8 May 2022 18:33:38 +0900 Subject: [PATCH 023/181] Fix: typo for documents --- doc/source/symbol-tables.rst | 2 +- doc/source/volshell.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 245dd9c67..4dea6077d 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -63,7 +63,7 @@ To determine the string for a particular memory image, use the `banners` plugin. try to locate that exact kernel debugging package for the operating system. Unfortunately each distribution provides its debugging packages under different package names and there are so many that the distribution may not keep all old versions of the debugging symbols, and therefore **it may not be possible to find the right symbols to analyze a linux -memory image with volatlity**. With Macs there are far fewer kernels and only one distribution, making it easier to +memory image with volatility**. With Macs there are far fewer kernels and only one distribution, making it easier to ensure that the right symbols can be found. Once a kernel with debugging symbols/appropriate DWARF file has been located, `dwarf2json `_ will convert it into an diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index de3c4398a..5a4b21ade 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -110,7 +110,7 @@ This means that pointers do not need to be explicitly dereferenced to access und Running plugins --------------- -It's possible to run any plugin by importing it appropriately and passing it to the `display_plugin_ouptut` or `dpo` +It's possible to run any plugin by importing it appropriately and passing it to the `display_plugin_output` or `dpo` method. In the following example we'll provide no additional parameters. Volatility will show us which parameters were required: From fde05cd1f2d7a35dd1949988f667d7f97d8e9459 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 9 May 2022 11:23:13 +0900 Subject: [PATCH 024/181] Fix: typo for cli exception message --- volatility3/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 488892d37..e3fb726a1 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -423,7 +423,7 @@ class CommandLine: detail = f"{excp}" caused_by = ["A required python module is not installed (install the module and re-run)"] else: - general = "Volatilty encountered an unexpected situation." + general = "Volatility encountered an unexpected situation." detail = "" caused_by = [ "Please re-run using with -vvv and file a bug with the output", f"at {constants.BUG_URL}" From f54edee36203d8537bf6716a577799bd9184bb1c Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 9 May 2022 10:56:40 +0300 Subject: [PATCH 025/181] removed svcscan import --- volatility3/framework/plugins/windows/callbacks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index dca17aff7..2195671df 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -11,7 +11,6 @@ from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.plugins.windows import ssdt -from volatility3.plugins.windows import svcscan vollog = logging.getLogger(__name__) @@ -28,7 +27,6 @@ class Callbacks(interfaces.plugins.PluginInterface): requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'ssdt', plugin = ssdt.SSDT, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'svcscan', plugin = svcscan.SvcScan, version = (1, 0, 0)) ] @staticmethod From 2c181fb9befd81079e575a554b3d4a625378a876 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 9 May 2022 19:38:25 +1000 Subject: [PATCH 026/181] Move path_root check up so path() and get_mnt_sb() can be avoided in the cases _do_get_path() fails. --- volatility3/framework/plugins/linux/mountinfo.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 6c0f8bcc3..3179eadd8 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -89,6 +89,10 @@ class MountInfo(plugins.PluginInterface): if not mnt_root: return None + path_root = cls._do_get_path(mnt, task.fs.root) + if path_root is None: + return None + mnt_root_path = mnt_root.path() superblock = mnt.get_mnt_sb() @@ -97,10 +101,6 @@ class MountInfo(plugins.PluginInterface): st_dev = f"{superblock.major}:{superblock.minor}" - path_root = cls._do_get_path(mnt, task.fs.root) - if path_root is None: - return None - mnt_opts: List[str] = [] mnt_opts.append(mnt.get_flags_access()) mnt_opts.extend(mnt.get_flags_opts()) From 62e00c087990c5094dcdd9ab23aeb559eee7f640 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 9 May 2022 20:18:11 +1000 Subject: [PATCH 027/181] Added smear protection in loops. Added/improved some docstrings. --- .../symbols/linux/extensions/__init__.py | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 43a6e0d1d..89c20fc0c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -202,14 +202,14 @@ class task_struct(generic.GenericIntelProcess): yield (start, end - start) def get_threads(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns a list of the task_struct based on the list_head + """Returns a list of the task_struct based on the list_head thread_node structure.""" task_symbol_table_name = self.get_symbol_table_name() # iterating through the thread_list from thread_group - # this allows iterating through pointers to grab the - # threads and using the thread_group offset to get the + # this allows iterating through pointers to grab the + # threads and using the thread_group offset to get the # corresponding task_struct for task in self.thread_group.to_list( f"{task_symbol_table_name}{constants.BANG}task_struct", @@ -425,12 +425,15 @@ class qstr(objects.StructType): class dentry(objects.StructType): def path(self) -> str: - """ Based on __dentry_path Linux kernel function""" + """Based on __dentry_path Linux kernel function""" reversed_path = [] + dentry_seen = set() current_dentry = self - while not current_dentry.is_root(): + while (not current_dentry.is_root() and + current_dentry.vol.offset not in dentry_seen): parent = current_dentry.d_parent reversed_path.append(current_dentry.d_name.name_as_str()) + dentry_seen.add(current_dentry.vol.offset) current_dentry = parent return "/" + "/".join(reversed(reversed_path)) @@ -455,11 +458,14 @@ class dentry(objects.StructType): if "ancestor_dentry" is an ancestor of "child_dentry", else None. """ + dentry_seen = set() current_dentry = self - while not current_dentry.is_root(): + while (not current_dentry.is_root() and + current_dentry.vol.offset not in dentry_seen): if current_dentry.d_parent == ancestor_dentry.vol.offset: return current_dentry + dentry_seen.add(current_dentry.vol.offset) current_dentry = current_dentry.d_parent return None @@ -632,32 +638,48 @@ class mount(objects.StructType): def get_dominating_id(self, root) -> int: """Get ID of closest dominating peer group having a representative under the given root.""" + mnt_seen = set() current_mnt = self.mnt_master - while current_mnt and current_mnt.vol.offset != 0: + while (current_mnt and + current_mnt.vol.offset != 0 and + current_mnt.vol.offset not in mnt_seen): peer = current_mnt.get_peer_under_root(self.mnt_ns, root) if peer and peer.vol.offset != 0: return peer.mnt_group_id + mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.mnt_master return 0 def get_peer_under_root(self, ns, root): - current = self - while True: - if current.mnt_ns == ns and current.is_path_reachable(current.mnt.mnt_root, root): - return current - current = current.next_peer() - if current.vol.offset == self.vol.offset: + """Return true if path is reachable from root. + It mimics the kernel function is_path_reachable(), ref: fs/namespace.c + """ + mnt_seen = set() + current_mnt = self + while current_mnt.vol.offset not in mnt_seen: + if current_mnt.mnt_ns == ns and current_mnt.is_path_reachable(current_mnt.mnt.mnt_root, root): + return current_mnt + + mnt_seen.add(current_mnt.vol.offset) + current_mnt = current_mnt.next_peer() + if current_mnt.vol.offset == self.vol.offset: break return None def is_path_reachable(self, current_dentry, root): - """Return true if path is reachable + """Return true if path is reachable. + It mimics the kernel function with same name, ref fs/namespace.c: """ + mnt_seen = set() current_mnt = self - while current_mnt.mnt.vol.offset != root.mnt and current_mnt.has_parent(): + while (current_mnt.mnt.vol.offset != root.mnt and + current_mnt.has_parent() and + current_mnt.vol.offset not in mnt_seen): + current_dentry = current_mnt.mnt_mountpoint + mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.mnt_parent return current_mnt.mnt.vol.offset == root.mnt and current_dentry.is_subdir(root.dentry) From f4dd582f158e8024e3fc5b4fba21a727f0913bfb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 12:29:36 +0900 Subject: [PATCH 028/181] Fix: ThreadsProcess for windows older version --- .../framework/symbols/windows/extensions/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b317f7693..ccfcb4290 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -450,7 +450,12 @@ class ETHREAD(objects.StructType): def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" - return self.Tcb.Process.dereference().cast("_EPROCESS") + if(self.has_member("ThreadsProcess")): + return self.ThreadsProcess.dereference().cast("_EPROCESS") + elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): + return self.Tcb.Process.dereference().cast("_EPROCESS") + else: + raise AttributeError("Unable to find the owning process of ethread") def get_cross_thread_flags(self) -> str: dictCrossThreadFlags = { From 1125be122e8d330cad156ba67a279bf83f22c2f0 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 12:31:56 +0900 Subject: [PATCH 029/181] Add: code comment for windows version --- volatility3/framework/symbols/windows/extensions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ccfcb4290..a1c347ed2 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -450,6 +450,7 @@ class ETHREAD(objects.StructType): def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" + if(self.has_member("ThreadsProcess")): return self.ThreadsProcess.dereference().cast("_EPROCESS") elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): From 3956f0ecc0406f32482123c0c1866755b8fd7cf7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 12:36:56 +0900 Subject: [PATCH 030/181] Add: code comment for windows version --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index a1c347ed2..805f8c26b 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -451,8 +451,10 @@ class ETHREAD(objects.StructType): def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" + # For Windows XPs if(self.has_member("ThreadsProcess")): return self.ThreadsProcess.dereference().cast("_EPROCESS") + # For Windows Vista and later versions elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): return self.Tcb.Process.dereference().cast("_EPROCESS") else: From 9fc6e5725c739e9339a5d4cb97dbb0b3ae3e08a4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 10 May 2022 17:24:32 +1000 Subject: [PATCH 031/181] Bump framework version to 2.2.0 --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/linux/mountinfo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 5060906d5..472a743e6 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 1 # Number of changes that only add to the interface +VERSION_MINOR = 2 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 3179eadd8..6f3cb712d 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -20,7 +20,7 @@ MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "m class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" - _required_framework_version = (2, 1, 0) + _required_framework_version = (2, 2, 0) _version = (1, 0, 0) From 690d8e3efe8ec08d4500b75ba60f14a281a52673 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 19:07:04 +0900 Subject: [PATCH 032/181] Fix: sync bump version --- volatility3/framework/constants/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index e08bc42bc..472a743e6 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,8 +39,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 1 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 2 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From a5bf5548b8d7e83ffd3b9065e968e321f6fcc964 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 19:08:29 +0900 Subject: [PATCH 033/181] Bump: patch version 2.2.1 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 472a743e6..44b98b95f 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 2 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From e5dfc47cc419d4d1ac929782008bc24765f46428 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 19:08:56 +0900 Subject: [PATCH 034/181] Fix: required framework version of psscan by bump --- volatility3/framework/plugins/windows/psscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index a0601aef1..cc030b4bf 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -22,7 +22,7 @@ vollog = logging.getLogger(__name__) class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for processes present in a particular windows memory image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 2, 1) _version = (1, 1, 0) @classmethod From 64cabc154679f629c113347798b3f1ea9ae6d44a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 20:53:03 +0900 Subject: [PATCH 035/181] Fix: typo, unification for API_CHANGES.md --- API_CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index 4e1820eff..e4b229dcf 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -10,11 +10,11 @@ Add in the linux `task.get_threads` method added to the API. 2.0.3 ===== -`DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` added to the API. +Add in the windows `DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` method added to the API. 2.0.2 ===== -Fix the behaviour of the offsets returned by the PDB scanner. +Fix the behavior of the offsets returned by the PDB scanner. 2.0.0 ===== From b9abb3f06c114ebac3191c785b041defe9c78cb8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 20:53:18 +0900 Subject: [PATCH 036/181] Fix: typo for windows code comment --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index e7da0316d..69e8ba94e 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -719,7 +719,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): env = envar[:split_index] var = envar[split_index + 1:] - # Exlude parse problem with some types of env + # Exclude parse problem with some types of env if env and var: yield env, var From b81eb04fb26e1a84677a00e1992eff071bc85386 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 11 May 2022 07:39:45 +0900 Subject: [PATCH 037/181] Revert: british english by code review --- API_CHANGES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index e4b229dcf..274d1d8bb 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -6,15 +6,15 @@ When an API feature or function is removed or changed, the major version is bump 2.1.0 ===== -Add in the linux `task.get_threads` method added to the API. +Add in the linux `task.get_threads` method to the API. 2.0.3 ===== -Add in the windows `DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` method added to the API. +Add in the windows `DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` methods to the API. 2.0.2 ===== -Fix the behavior of the offsets returned by the PDB scanner. +Fix the behaviour of the offsets returned by the PDB scanner. 2.0.0 ===== From 51bce3e62049155b88feb6459b3a75bd1dcdffd7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 00:25:13 +0900 Subject: [PATCH 038/181] Fix: compare logic for key_path and top_key --- volatility3/plugins/windows/registry/certificates.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 96a7e3977..5bc2b18e3 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -6,7 +6,6 @@ from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey - class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" @@ -52,7 +51,7 @@ class Certificates(interfaces.plugins.PluginInterface): node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) - unique_key_offset = key_path.index(top_key) + len(top_key) + 1 + unique_key_offset = key_path.casefold().index(top_key.casefold()) + len(top_key) + 1 reg_section = key_path[unique_key_offset:key_path.index("\\", unique_key_offset)] key_hash = key_path[key_path.rindex("\\") + 1:] From fa6465dcfb05c3d3f63f3903d5d062eda3ddc131 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 00:26:39 +0900 Subject: [PATCH 039/181] Revert: blank line --- volatility3/plugins/windows/registry/certificates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 5bc2b18e3..91f17fb2d 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -6,6 +6,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey + class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" From baaedf21e51b33c9c3eee772296ebfd3b9dd9869 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 19:37:30 +0900 Subject: [PATCH 040/181] Add: plugin version, logger, dump options --- .../plugins/windows/registry/certificates.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 91f17fb2d..81b7d766f 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,16 +1,19 @@ +import logging import struct from typing import List, Iterator, Tuple -from volatility3.framework import interfaces, renderers +from volatility3.framework import constants, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey +vollog = logging.getLogger(__name__) class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -20,7 +23,11 @@ class Certificates(interfaces.plugins.PluginInterface): architectures = ["Intel32", "Intel64"]), requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)) + requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)), + requirements.BooleanRequirement(name = 'dump', + description = "Extract listed certificates", + default = False, + optional = True) ] def parse_data(self, data: bytes) -> Tuple[str, bytes]: @@ -48,21 +55,22 @@ class Certificates(interfaces.plugins.PluginInterface): try: # Walk it node_path = hive.get_key(top_key, return_list = True) - for (depth, is_key, last_write_time, key_path, volatility, - node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): + for (_, is_key, _, key_path, _, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = key_path.casefold().index(top_key.casefold()) + len(top_key) + 1 reg_section = key_path[unique_key_offset:key_path.index("\\", unique_key_offset)] key_hash = key_path[key_path.rindex("\\") + 1:] - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - with self.open("{} - {} - {}.crt".format(hex(hive.hive_offset), reg_section, - key_hash)) as file_data: - file_data.write(certificate_data) + if self.config['dump']: + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + with self.open("{} - {} - {}.crt".format(hex(hive.hive_offset), reg_section, + key_hash)) as file_data: + file_data.write(certificate_data) yield (0, (top_key, reg_section, key_hash, name)) except KeyError: # Key wasn't found in this hive, carry on + vollog.log(constants.LOGLEVEL_VVVV, "Key wasn't found in this hive") pass def run(self) -> renderers.TreeGrid: From dafd62d6cee5f8366e3071adfc80e1c910d301fa Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 19:52:57 +0900 Subject: [PATCH 041/181] Fix: dump options for depreated step --- .../plugins/windows/registry/certificates.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 81b7d766f..f261b5e36 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -41,6 +41,12 @@ class Certificates(interfaces.plugins.PluginInterface): elif ctype == 0x100000020: certificate_data = cvalue return (name, certificate_data) + + def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str) -> str: + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) + with self.open(dump_name) as file_data: + file_data.write(certificate_data) def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: for hive in hivelist.HiveList.list_hives(self.context, @@ -63,10 +69,11 @@ class Certificates(interfaces.plugins.PluginInterface): key_hash = key_path[key_path.rindex("\\") + 1:] if self.config['dump']: - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - with self.open("{} - {} - {}.crt".format(hex(hive.hive_offset), reg_section, - key_hash)) as file_data: - file_data.write(certificate_data) + self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) + else: + vollog.warning("Certificates plugin is no longer support automatically dumped, please use the dump option.") + self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) + yield (0, (top_key, reg_section, key_hash, name)) except KeyError: # Key wasn't found in this hive, carry on From f63f869506186d0396625d6232f9657ca1dad717 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 20:23:42 +0900 Subject: [PATCH 042/181] Remove: return type of dump method --- volatility3/plugins/windows/registry/certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index f261b5e36..ee9cba6d3 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -42,7 +42,7 @@ class Certificates(interfaces.plugins.PluginInterface): certificate_data = cvalue return (name, certificate_data) - def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str) -> str: + def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str): if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) with self.open(dump_name) as file_data: From ba4a7d9262103ab3c650db47dba4f0d93c7fa6e6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 15 May 2022 00:17:46 +0900 Subject: [PATCH 043/181] Fix: JSON renderer for EOF format --- volatility3/cli/text_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 08608a3d0..ecb5179e0 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -345,7 +345,7 @@ class JsonRenderer(CLIRenderer): def output_result(self, outfd, result): """Outputs the JSON data to a file in a particular format""" - outfd.write(json.dumps(result, indent = 2, sort_keys = True)) + outfd.write("{}\n".format(json.dumps(result, indent = 2, sort_keys = True))) def render(self, grid: interfaces.renderers.TreeGrid): outfd = sys.stdout From 0e8958b8416350ac9c22961674532e62b72050ec Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 16 May 2022 15:11:07 +0900 Subject: [PATCH 044/181] Fix: classmethod, variable name, exceptions, etc --- .../plugins/windows/registry/certificates.py | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index ee9cba6d3..a27b3545a 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,8 +1,8 @@ import logging import struct -from typing import List, Iterator, Tuple +from typing import List, Iterator, Tuple, Type -from volatility3.framework import constants, interfaces, renderers +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey @@ -13,7 +13,6 @@ class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -42,11 +41,20 @@ class Certificates(interfaces.plugins.PluginInterface): certificate_data = cvalue return (name, certificate_data) - def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str): - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) - with self.open(dump_name) as file_data: - file_data.write(certificate_data) + @classmethod + def dump_certificate(cls, certificate_data: bytes, hive_offset: int, + reg_section: str, key_hash: str, + open_method: Type[interfaces.plugins.FileHandlerInterface]) -> \ + interfaces.plugins.FileHandlerInterface: + try: + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) + with open_method(dump_name) as file_data: + file_data.write(certificate_data) + except exceptions.InvalidAddressException: + vollog.debug(f"Unable to certificate file at {hive_offset:#x}") + return None + def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: for hive in hivelist.HiveList.list_hives(self.context, @@ -61,7 +69,7 @@ class Certificates(interfaces.plugins.PluginInterface): try: # Walk it node_path = hive.get_key(top_key, return_list = True) - for (_, is_key, _, key_path, _, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): + for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = key_path.casefold().index(top_key.casefold()) + len(top_key) + 1 @@ -69,10 +77,9 @@ class Certificates(interfaces.plugins.PluginInterface): key_hash = key_path[key_path.rindex("\\") + 1:] if self.config['dump']: - self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) - else: - vollog.warning("Certificates plugin is no longer support automatically dumped, please use the dump option.") - self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) + file_handle = self.dump_certificate(certificate_data, hive.hive_offset, reg_section, key_hash, self.open) + if file_handle: + file_handle.close() yield (0, (top_key, reg_section, key_hash, name)) except KeyError: From 5770d35a4afd718760ddeeb1c21606e6b5bd1e2e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 17 May 2022 00:18:06 +0900 Subject: [PATCH 045/181] Add: return file handle --- volatility3/plugins/windows/registry/certificates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index a27b3545a..b079844e7 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -49,8 +49,9 @@ class Certificates(interfaces.plugins.PluginInterface): try: if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) - with open_method(dump_name) as file_data: - file_data.write(certificate_data) + file_handle = open_method(dump_name) + file_handle.write(certificate_data) + return file_handle except exceptions.InvalidAddressException: vollog.debug(f"Unable to certificate file at {hive_offset:#x}") return None From 3846268bf7a8b3731a80a61959ca1aee227a112d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 17 May 2022 00:19:39 +0900 Subject: [PATCH 046/181] Add: optional return type --- volatility3/plugins/windows/registry/certificates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index b079844e7..d2fb61f02 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,6 +1,6 @@ import logging import struct -from typing import List, Iterator, Tuple, Type +from typing import List, Iterator, Optional, Tuple, Type from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -45,7 +45,7 @@ class Certificates(interfaces.plugins.PluginInterface): def dump_certificate(cls, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str, open_method: Type[interfaces.plugins.FileHandlerInterface]) -> \ - interfaces.plugins.FileHandlerInterface: + Optional[interfaces.plugins.FileHandlerInterface]: try: if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) From 05ae20c78bd982e95ab3dd99d92a2efaf68be3b3 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 22 May 2022 11:49:18 +0300 Subject: [PATCH 047/181] fix off by in filelayer --- volatility3/framework/layers/physical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 73d46b211..5d5fd17a9 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -118,7 +118,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): with self._lock: orig = self._file.tell() self._file.seek(0, 2) - self._size = self._file.tell() + self._size = self._file.tell() - 1 self._file.seek(orig) return self._size From 98001e7dd72ac6e39f440191da019ec33a9e64c5 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 22 May 2022 23:58:38 +0900 Subject: [PATCH 048/181] Fix: typo for code comments --- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/symbols.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index ab81beb5e..85a7d32b7 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -321,7 +321,7 @@ class SizedModule(Module): The mapping should be sorted and should be quicker than reading the data We turn it into JSON to make a common string and use a - quick hash, because collissions are unlikely + quick hash, because collisions are unlikely """ layer = self._context.layers[self.layer_name] if not isinstance(layer, interfaces.layers.TranslationLayerInterface): diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index 9f2cb9fc9..1ad30cfdf 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -169,7 +169,7 @@ class BaseSymbolTableInterface: def optional_set_type_class(self, name: str, clazz: Type[objects.ObjectInterface]) -> bool: """Calls the set_type_class function but does not throw an exception. - Returns whether setting the type class was successfull. + Returns whether setting the type class was successful. Args: name: The name of the type to override the class for clazz: The actual class to override for the provided type name From 2a011487a91c9f8e71b86a80e0d186e03b84a5b4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 22 May 2022 23:23:02 +0100 Subject: [PATCH 049/181] Core: Old linux systems may not have mnt_namespace structures --- volatility3/framework/symbols/linux/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0c5ce395c..d59a95db5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,10 +1,10 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import List, Tuple, Iterator +from typing import Iterator, List, Tuple from volatility3 import framework -from volatility3.framework import exceptions, constants, interfaces, objects +from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -29,7 +29,9 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('files_struct', extensions.files_struct) self.set_type_class('vfsmount', extensions.vfsmount) self.set_type_class('kobject', extensions.kobject) - self.set_type_class('mnt_namespace', extensions.mnt_namespace) + + if 'mnt_namespace' in self.types: + self.set_type_class('mnt_namespace', extensions.mnt_namespace) if 'module' in self.types: self.set_type_class('module', extensions.module) @@ -267,4 +269,4 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): while list_start: list_struct = vmlinux.object(object_type = struct_name, offset = list_start.vol.offset) yield list_struct - list_start = getattr(list_struct, list_member) \ No newline at end of file + list_start = getattr(list_struct, list_member) From 786fd61fc9b6b2978e0de7595bdb22aca9e91843 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 9 Apr 2022 20:57:34 +0100 Subject: [PATCH 050/181] Layers: Add architecture to qemu layer --- volatility3/framework/layers/qemu.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index f1ba1e468..65f8eed9a 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -3,9 +3,9 @@ # import functools import json -from typing import Optional, Dict, Any, Tuple, List, Set +from typing import Any, Dict, List, Optional, Set, Tuple -from volatility3.framework import interfaces, exceptions, constants +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.layers import segmented from volatility3.framework.symbols import intermed @@ -39,6 +39,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): metadata: Optional[Dict[str, Any]] = None) -> None: self._qemu_table_name = intermed.IntermediateSymbolTable.create(context, config_path, 'generic', 'qemu') self._configuration = None + self._architecture = None self._compressed: Set[int] = set() self._current_segment_name = b'' super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) @@ -139,6 +140,9 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', offset = index, layer_name = self._base_layer) + self._architecture = self.context.object(self._qemu_table_name + constants.BANG + 'string', + offset = index + 4, layer_name = self._base_layer, + max_length = section_len) index += 4 + section_len elif section_byte == self.QEVM_SECTION_START or section_byte == self.QEVM_SECTION_FULL: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', From 302bb63645af3b9b20b2a361a73c06a2c27e3513 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 9 Apr 2022 21:46:47 +0100 Subject: [PATCH 051/181] Layers: Detect and compensate for QEVM pci-hole --- volatility3/framework/layers/qemu.py | 30 +++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 65f8eed9a..4e6252b2f 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -3,12 +3,17 @@ # import functools import json +import logging +import re +import struct from typing import Any, Dict, List, Optional, Set, Tuple from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.layers import segmented from volatility3.framework.symbols import intermed +vollog = logging.getLogger(__name__) + class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): """A Qemu suspend-to-disk translation layer.""" @@ -32,6 +37,13 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): SEGMENT_FLAG_XBZRLE = 0x40 SEGMENT_FLAG_HOOK = 0x80 + pci_hole_table = {re.compile(r"^pc-i440fx-\d\.\d$"): (0xc0000000, 0x100000000), + re.compile(r"^pc-1440fx-eoan$"): (0xe0000000, 0x100000000), + re.compile(r"^pc-q35$"): (0x80000000, 0x100000000), + re.compile(r"^microvm$"): (0xc0000000, 0x100000000), + re.compile(r"^xen$"): (0xf0000000, 0x100000000) + } + def __init__(self, context: interfaces.context.ContextInterface, config_path: str, @@ -42,6 +54,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._architecture = None self._compressed: Set[int] = set() self._current_segment_name = b'' + self._pci_hole_start = 0 + self._pci_hole_end = 0 super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) @classmethod @@ -77,9 +91,9 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): base_layer = self.context.layers[self._base_layer] while not done: - addr = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', - offset = index, - layer_name = self._base_layer) + # Use struct.unpack here for performance improvements + addr = struct.unpack('>Q', base_layer.read(index, 8))[0] + # Flags are stored in the n least significant bits, where n equals the bit-length of pagesize flags = addr & (page_size - 1) # addr equals the highest multiple of pagesize <= offset @@ -87,6 +101,9 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): addr = addr ^ (addr & (page_size - 1)) index += 8 + if addr > self._pci_hole_start: + addr += self._pci_hole_end - self._pci_hole_start + if flags & self.SEGMENT_FLAG_MEM_SIZE: namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, @@ -143,6 +160,13 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._architecture = self.context.object(self._qemu_table_name + constants.BANG + 'string', offset = index + 4, layer_name = self._base_layer, max_length = section_len) + for regex in self.pci_hole_table: + if regex.match(self._architecture): + self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM archicture detected as: {self._architecture}") + break + else: + vollog.debug(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") index += 4 + section_len elif section_byte == self.QEVM_SECTION_START or section_byte == self.QEVM_SECTION_FULL: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', From dea8e1dac9090b40823a941fdcfc1a828f80de9e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 18 Apr 2022 01:45:12 +0100 Subject: [PATCH 052/181] Layers: Add QEVM architecture fallback detection --- volatility3/framework/layers/qemu.py | 97 ++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 4e6252b2f..7835cad17 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -9,7 +9,7 @@ import struct from typing import Any, Dict, List, Optional, Set, Tuple from volatility3.framework import constants, exceptions, interfaces -from volatility3.framework.layers import segmented +from volatility3.framework.layers import scanners, segmented from volatility3.framework.symbols import intermed vollog = logging.getLogger(__name__) @@ -37,11 +37,32 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): SEGMENT_FLAG_XBZRLE = 0x40 SEGMENT_FLAG_HOOK = 0x80 - pci_hole_table = {re.compile(r"^pc-i440fx-\d\.\d$"): (0xc0000000, 0x100000000), - re.compile(r"^pc-1440fx-eoan$"): (0xe0000000, 0x100000000), - re.compile(r"^pc-q35$"): (0x80000000, 0x100000000), - re.compile(r"^microvm$"): (0xc0000000, 0x100000000), - re.compile(r"^xen$"): (0xf0000000, 0x100000000) + # See https://qemu.readthedocs.io/en/latest/devel/memory.html for more info + # + # At least the following values could occur for devices using > 3-4 GB RAM: + # +--------------------------------+--------------------------------+------------+-------------+ + # | Architecture | Reference Code | Hole Start | Hole End | + # +--------------------------------+--------------------------------+------------+-------------+ + # | PC i440FX + PIIX "New Default" | qemu/hw/i386/pc_piix.c:98 | 0xc0000000 | 0x100000000 | + # | PC i440FX + PIIX "Old Default" | qemu/hw/i386/pc_piix.c:98 | 0xe0000000 | 0x100000000 | + # | PC Q35 + ICH9 | qemu/hw/i386/pc_q35.c:141 | 0x80000000 | 0x100000000 | + # | MicroVM | qemu/hw/i386/microvm.c:291 | 0xc0000000 | 0x100000000 | + # | Xen | qemu/hw/i386/xen/xen-hvm.c:248 | 0xf0000000 | 0x100000000 | + # +--------------------------------+--------------------------------+------------+-------------+ + # + # For now, we assume that the parameter max-ram-below-4g is not set, since this parameter influences the size + # and location of the memory gap. Deviating hole sizes could eventually be detected for Linux by e.g. scanning + # for dmesg entries with a regex like rb'\[mem (0x[0-9a-f]{4,10})-0x[0-9a-f]{4,10}\] available for PCI devices' + + debian_re = r"artful|eoan" + + pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), + re.compile(r"^pc-i440fx-[01].\d$"): (0xe0000000, 0xe0000000, 0x100000000), + re.compile(r"^pc-q35-\d.\d$"): (0xe0000000, 0x80000000, 0x100000000), + re.compile(r"^microvm$"): (0xe0000000, 0xc0000000, 0x100000000), + re.compile(r"^xen$"): (0xe0000000, 0xf0000000, 0x100000000), + re.compile(r"^pc-i440fx-" + debian_re + r"$"): (0xe0000000, 0xc0000000, 0x100000000), + re.compile(r"^pc-q35-" + debian_re + r"$"): (0xe0000000, 0x80000000, 0x100000000), } def __init__(self, @@ -65,6 +86,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): raise exceptions.LayerException(name, 'No QEMU magic bytes') if header[4:] != b'\x00\x00\x00\x03': raise exceptions.LayerException(name, 'Unsupported QEMU version found') + vollog.debug("QEVM header found") def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, name: str) -> Any: """Reads the JSON configuration from the end of the file""" @@ -160,13 +182,6 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._architecture = self.context.object(self._qemu_table_name + constants.BANG + 'string', offset = index + 4, layer_name = self._base_layer, max_length = section_len) - for regex in self.pci_hole_table: - if regex.match(self._architecture): - self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] - vollog.log(constants.LOGLEVEL_VVVV, f"QEVM archicture detected as: {self._architecture}") - break - else: - vollog.debug(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") index += 4 + section_len elif section_byte == self.QEVM_SECTION_START or section_byte == self.QEVM_SECTION_FULL: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', @@ -217,6 +232,62 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): else: raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}') + # If no architecture has been set, try to determine it using fallback mechanisms + if not self._architecture: + self._architecture = self._fallback_determine_architecture() + if self._architecture is None: + vollog.log(constants.LOGLEVEL_VV, f"QEVM architecture could not be determined") + + # Once all segments have been read, determine the PCI hole if any + for regex in self.pci_hole_table: + if regex.match(self._architecture): + self._pci_hole_minimum, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] + if self.maximum_address < self._pci_hole_minimum: + # The PCI hole isn't present because we're below the minimum value + self._pci_hole_start, self._pci_hole_end = 0, 0 + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") + break + else: + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") + + def _fallback_determine_architecture(self) -> str: + architecture_pattern = rb'pc-(i440fx|q35)-([0-9]{1,2}.[0-9]{1,2}(?:.[0-9]{1,2})?)' + base_layer = self.context.layers[self._base_layer] + + vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") + + res = scanners.RegExScanner(architecture_pattern) + for offset in base_layer.scan(context = self.context, scanner = res): + line = base_layer.read(offset, 64) + regex_results = re.search(architecture_pattern, line) + architecture = "pc-" + regex_results.groups()[0].decode() + return architecture + + # If that does not work, look in configuration JSON for devices specific to a certain architecture + architecture = None + for device in self._configuration.get('devices', []): + device_name = device.get('vmsd_name', '').lower() + if 'i440fx' in device_name or 'piix' in device_name: + architecture = 'pc-i440fx-2.0' + break + elif 'ich9' in device_name: + architecture = 'pc-q35-1.0' + break + if architecture: + return architecture + + # Still haven't found architecture, switch to fallback-method + architecture_pattern = rb'Standard PC \((i440FX|Q35)' + res = scanners.RegExScanner(architecture_pattern) + for offset in base_layer.scan(context = self.context, scanner = res): + line = base_layer.read(offset, 64) + regex_results = re.search(architecture_pattern, line) + architecture = "pc-" + regex_results.groups()[0].decode().lower() + return architecture + + vollog.warning("Could not determine QEMU target architecture!") + return None + def extract_data(self, index, name, version_id): if name == 'ram': if version_id != 4: From 1fca57ffc3603ab383332076f05e54f973c21452 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 20 Apr 2022 21:43:30 +0100 Subject: [PATCH 053/181] Layers: QEVM more fixes for minimum addresses --- volatility3/framework/layers/qemu.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 7835cad17..23c29dc39 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -54,15 +54,15 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): # and location of the memory gap. Deviating hole sizes could eventually be detected for Linux by e.g. scanning # for dmesg entries with a regex like rb'\[mem (0x[0-9a-f]{4,10})-0x[0-9a-f]{4,10}\] available for PCI devices' - debian_re = r"artful|eoan" + distro_re = r"(artful|eoan|rhel[\d\.]+)" pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), - re.compile(r"^pc-i440fx-[01].\d$"): (0xe0000000, 0xe0000000, 0x100000000), - re.compile(r"^pc-q35-\d.\d$"): (0xe0000000, 0x80000000, 0x100000000), - re.compile(r"^microvm$"): (0xe0000000, 0xc0000000, 0x100000000), - re.compile(r"^xen$"): (0xe0000000, 0xf0000000, 0x100000000), - re.compile(r"^pc-i440fx-" + debian_re + r"$"): (0xe0000000, 0xc0000000, 0x100000000), - re.compile(r"^pc-q35-" + debian_re + r"$"): (0xe0000000, 0x80000000, 0x100000000), + re.compile(r"^pc-i440fx-[01]\.\d$"): (0xe0000000, 0xe0000000, 0x100000000), + re.compile(r"^pc-q35-\d\.\d$"): (0xb0000000, 0x80000000, 0x100000000), + re.compile(r"^microvm$"): (0xc0000000, 0xc0000000, 0x100000000), + re.compile(r"^xen$"): (0xf0000000, 0xf0000000, 0x100000000), + re.compile(r"^pc-i440fx-" + distro_re + r"$"): (0xe0000000, 0xe0000000, 0x100000000), + re.compile(r"^pc-q35-" + distro_re + r"$"): (0xb0000000, 0x80000000, 0x100000000), } def __init__(self, @@ -252,6 +252,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): def _fallback_determine_architecture(self) -> str: architecture_pattern = rb'pc-(i440fx|q35)-([0-9]{1,2}.[0-9]{1,2}(?:.[0-9]{1,2})?)' + old_suffix = "-1.0" base_layer = self.context.layers[self._base_layer] vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") @@ -260,7 +261,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for offset in base_layer.scan(context = self.context, scanner = res): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) - architecture = "pc-" + regex_results.groups()[0].decode() + architecture = "pc-" + regex_results.groups()[0].decode() + old_suffix return architecture # If that does not work, look in configuration JSON for devices specific to a certain architecture @@ -268,10 +269,10 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for device in self._configuration.get('devices', []): device_name = device.get('vmsd_name', '').lower() if 'i440fx' in device_name or 'piix' in device_name: - architecture = 'pc-i440fx-2.0' + architecture = 'pc-i440fx' + old_suffix break elif 'ich9' in device_name: - architecture = 'pc-q35-1.0' + architecture = 'pc-q35' + old_suffix break if architecture: return architecture @@ -282,7 +283,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for offset in base_layer.scan(context = self.context, scanner = res): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) - architecture = "pc-" + regex_results.groups()[0].decode().lower() + architecture = "pc-" + regex_results.groups()[0].decode().lower() + old_suffix return architecture vollog.warning("Could not determine QEMU target architecture!") From afec05106127a8de73c22e7cc7f87ab3dd67ef9d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 21 Apr 2022 10:29:26 +0100 Subject: [PATCH 054/181] Layers: Make QEVM changes based on @cstation 's feedback --- volatility3/framework/layers/qemu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 23c29dc39..16f18cca1 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -123,7 +123,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): addr = addr ^ (addr & (page_size - 1)) index += 8 - if addr > self._pci_hole_start: + if addr >= self._pci_hole_start: addr += self._pci_hole_end - self._pci_hole_start if flags & self.SEGMENT_FLAG_MEM_SIZE: @@ -252,7 +252,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): def _fallback_determine_architecture(self) -> str: architecture_pattern = rb'pc-(i440fx|q35)-([0-9]{1,2}.[0-9]{1,2}(?:.[0-9]{1,2})?)' - old_suffix = "-1.0" + old_suffix = "-2.0" base_layer = self.context.layers[self._base_layer] vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") @@ -261,7 +261,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for offset in base_layer.scan(context = self.context, scanner = res): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) - architecture = "pc-" + regex_results.groups()[0].decode() + old_suffix + architecture = regex_results.group().decode() return architecture # If that does not work, look in configuration JSON for devices specific to a certain architecture From fb861eb6dfc528ec8f4c2a3f71a0995bdada0cba Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 21 Apr 2022 10:33:29 +0100 Subject: [PATCH 055/181] Layers: Simplification of QEVM fallback regex --- volatility3/framework/layers/qemu.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 16f18cca1..11bec2dd3 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -61,7 +61,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): re.compile(r"^pc-q35-\d\.\d$"): (0xb0000000, 0x80000000, 0x100000000), re.compile(r"^microvm$"): (0xc0000000, 0xc0000000, 0x100000000), re.compile(r"^xen$"): (0xf0000000, 0xf0000000, 0x100000000), - re.compile(r"^pc-i440fx-" + distro_re + r"$"): (0xe0000000, 0xe0000000, 0x100000000), + re.compile(r"^pc-i440fx-" + distro_re + r"$"): (0xe0000000, 0xc0000000, 0x100000000), re.compile(r"^pc-q35-" + distro_re + r"$"): (0xb0000000, 0x80000000, 0x100000000), } @@ -251,7 +251,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") def _fallback_determine_architecture(self) -> str: - architecture_pattern = rb'pc-(i440fx|q35)-([0-9]{1,2}.[0-9]{1,2}(?:.[0-9]{1,2})?)' + architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|[\w\d\.]+)' old_suffix = "-2.0" base_layer = self.context.layers[self._base_layer] From 21b0cb56a5746410f8ff9c96fba9d0e0ee86730f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 21 Apr 2022 23:50:13 +0100 Subject: [PATCH 056/181] Layers: Shift when we calculate the QEVM PCI hole --- volatility3/framework/layers/qemu.py | 39 +++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 11bec2dd3..270405645 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -170,7 +170,28 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): index = 8 section_info = dict() current_section_id = -1 + version_id = -1 + name = None + arch_detected = False while section_byte != self.QEVM_EOF and index <= base_layer.maximum_address: + if index > 20 and not arch_detected: + # We're past where the QEVM_CONFIGURATION might be, so set the values + # If no architecture has been set, try to determine it using fallback mechanisms + if not self._architecture: + self._architecture = self._fallback_determine_architecture() + if self._architecture is None: + vollog.log(constants.LOGLEVEL_VV, f"QEVM architecture could not be determined") + + # Once all segments have been read, determine the PCI hole if any + for regex in self.pci_hole_table: + if regex.match(self._architecture): + _, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") + break + else: + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") + arch_detected = True + section_byte = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) @@ -232,24 +253,6 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): else: raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}') - # If no architecture has been set, try to determine it using fallback mechanisms - if not self._architecture: - self._architecture = self._fallback_determine_architecture() - if self._architecture is None: - vollog.log(constants.LOGLEVEL_VV, f"QEVM architecture could not be determined") - - # Once all segments have been read, determine the PCI hole if any - for regex in self.pci_hole_table: - if regex.match(self._architecture): - self._pci_hole_minimum, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] - if self.maximum_address < self._pci_hole_minimum: - # The PCI hole isn't present because we're below the minimum value - self._pci_hole_start, self._pci_hole_end = 0, 0 - vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") - break - else: - vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") - def _fallback_determine_architecture(self) -> str: architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|[\w\d\.]+)' old_suffix = "-2.0" From 4eacdd9bea6be1be5506273abba5a4cc7715e7ff Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 25 Apr 2022 00:09:07 +0100 Subject: [PATCH 057/181] Layers: use QEVM size to turn off pci hole if needed --- volatility3/framework/layers/qemu.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 270405645..c755b8a9b 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -77,6 +77,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._current_segment_name = b'' self._pci_hole_start = 0 self._pci_hole_end = 0 + self._pci_hole_minimum = 0 super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) @classmethod @@ -110,6 +111,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): done = None segments = [] + size_array = {} base_layer = self.context.layers[self._base_layer] while not done: @@ -131,14 +133,21 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): offset = index, layer_name = self._base_layer) while namelen != 0: - # if base_layer.read(index + 1, namelen) == b'pc.ram': - # total_size = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', - # offset = index + 1 + namelen, - # layer_name = self._base_layer) + total_size = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', + offset = index + 1 + namelen, + layer_name = self._base_layer) + size_array[base_layer.read(index + 1, namelen)] = total_size index += 1 + namelen + 8 namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) + if size_array.get(b'pc.ram', + max([x[0] for x in self.pci_hole_table.values()]) + 1) <= self._pci_hole_minimum: + # Turns off the pci_hole if it's not supposed to be there + vollog.debug( + f"QEVM tunrning off PCI hole due to small image size: {size_array.get(b'pc.ram'):x} < {self._pci_hole_minimum:x}") + self._pci_hole_start, self._pci_hole_end = 0, 0 + if flags & (self.SEGMENT_FLAG_COMPRESS | self.SEGMENT_FLAG_PAGE): if not (flags & self.SEGMENT_FLAG_CONTINUE): namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', @@ -185,7 +194,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): # Once all segments have been read, determine the PCI hole if any for regex in self.pci_hole_table: if regex.match(self._architecture): - _, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] + self._pci_hole_minimum, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") break else: From dd8fcce2ef683aa4bad211ee9273a865a59e84e1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Apr 2022 22:30:14 +0100 Subject: [PATCH 058/181] Layers: QEMU improvements suggested by @cstation --- volatility3/framework/layers/qemu.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index c755b8a9b..e3d70ba6e 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -54,7 +54,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): # and location of the memory gap. Deviating hole sizes could eventually be detected for Linux by e.g. scanning # for dmesg entries with a regex like rb'\[mem (0x[0-9a-f]{4,10})-0x[0-9a-f]{4,10}\] available for PCI devices' - distro_re = r"(artful|eoan|rhel[\d\.]+)" + distro_re = r"(\w+[\d\.]?)" pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), re.compile(r"^pc-i440fx-[01]\.\d$"): (0xe0000000, 0xe0000000, 0x100000000), @@ -141,8 +141,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) - if size_array.get(b'pc.ram', - max([x[0] for x in self.pci_hole_table.values()]) + 1) <= self._pci_hole_minimum: + highest_possible_maximum = max([x[0] for x in self.pci_hole_table.values()]) + 1 + if size_array.get(b'pc.ram', highest_possible_maximum) < self._pci_hole_minimum: # Turns off the pci_hole if it's not supposed to be there vollog.debug( f"QEVM tunrning off PCI hole due to small image size: {size_array.get(b'pc.ram'):x} < {self._pci_hole_minimum:x}") @@ -264,7 +264,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): def _fallback_determine_architecture(self) -> str: architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|[\w\d\.]+)' - old_suffix = "-2.0" + default_suffix = "-2.0" base_layer = self.context.layers[self._base_layer] vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") @@ -281,10 +281,10 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for device in self._configuration.get('devices', []): device_name = device.get('vmsd_name', '').lower() if 'i440fx' in device_name or 'piix' in device_name: - architecture = 'pc-i440fx' + old_suffix + architecture = 'pc-i440fx' + default_suffix break elif 'ich9' in device_name: - architecture = 'pc-q35' + old_suffix + architecture = 'pc-q35' + default_suffix break if architecture: return architecture @@ -295,7 +295,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for offset in base_layer.scan(context = self.context, scanner = res): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) - architecture = "pc-" + regex_results.groups()[0].decode().lower() + old_suffix + architecture = "pc-" + regex_results.groups()[0].decode().lower() + default_suffix return architecture vollog.warning("Could not determine QEMU target architecture!") From 7ce95117484e13ab0ba6a4a50051b56252cca978 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 23 May 2022 01:20:46 +0100 Subject: [PATCH 059/181] Layers: QEMU recommendations from @cstation --- volatility3/framework/layers/qemu.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index e3d70ba6e..985c4534a 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -54,7 +54,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): # and location of the memory gap. Deviating hole sizes could eventually be detected for Linux by e.g. scanning # for dmesg entries with a regex like rb'\[mem (0x[0-9a-f]{4,10})-0x[0-9a-f]{4,10}\] available for PCI devices' - distro_re = r"(\w+[\d\.]?)" + distro_re = r"(\w+[\d{1,2}\.]*)" pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), re.compile(r"^pc-i440fx-[01]\.\d$"): (0xe0000000, 0xe0000000, 0x100000000), @@ -145,7 +145,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if size_array.get(b'pc.ram', highest_possible_maximum) < self._pci_hole_minimum: # Turns off the pci_hole if it's not supposed to be there vollog.debug( - f"QEVM tunrning off PCI hole due to small image size: {size_array.get(b'pc.ram'):x} < {self._pci_hole_minimum:x}") + f"QEVM tunrning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}") self._pci_hole_start, self._pci_hole_end = 0, 0 if flags & (self.SEGMENT_FLAG_COMPRESS | self.SEGMENT_FLAG_PAGE): @@ -179,8 +179,6 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): index = 8 section_info = dict() current_section_id = -1 - version_id = -1 - name = None arch_detected = False while section_byte != self.QEVM_EOF and index <= base_layer.maximum_address: if index > 20 and not arch_detected: @@ -263,7 +261,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}') def _fallback_determine_architecture(self) -> str: - architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|[\w\d\.]+)' + architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|\w+[\d{1,2}\.]*)' default_suffix = "-2.0" base_layer = self.context.layers[self._base_layer] @@ -287,6 +285,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): architecture = 'pc-q35' + default_suffix break if architecture: + vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}') return architecture # Still haven't found architecture, switch to fallback-method @@ -296,6 +295,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) architecture = "pc-" + regex_results.groups()[0].decode().lower() + default_suffix + vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}') return architecture vollog.warning("Could not determine QEMU target architecture!") From 6276b984008868161638971c8e7bf0eeddc1ea1a Mon Sep 17 00:00:00 2001 From: ikelos Date: Mon, 23 May 2022 01:48:07 +0100 Subject: [PATCH 060/181] Update volatility3/framework/layers/qemu.py Fix typo courtesy of @digitalisx Co-authored-by: Donghyun Kim --- volatility3/framework/layers/qemu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 985c4534a..907116e99 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -145,7 +145,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if size_array.get(b'pc.ram', highest_possible_maximum) < self._pci_hole_minimum: # Turns off the pci_hole if it's not supposed to be there vollog.debug( - f"QEVM tunrning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}") + f"QEVM turning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}") self._pci_hole_start, self._pci_hole_end = 0, 0 if flags & (self.SEGMENT_FLAG_COMPRESS | self.SEGMENT_FLAG_PAGE): From a3c63fbdf893324df2754b999e51db6655f3b06a Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 23 May 2022 09:25:30 +0300 Subject: [PATCH 061/181] rename variable --- volatility3/framework/layers/physical.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 5d5fd17a9..728fdcf31 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -113,14 +113,15 @@ class FileLayer(interfaces.layers.DataLayerInterface): def maximum_address(self) -> int: """Returns the largest available address in the space.""" # Zero based, so we return the size of the file minus 1 - if self._size: - return self._size + if self._maximum_address + return self._maximum_address with self._lock: orig = self._file.tell() self._file.seek(0, 2) - self._size = self._file.tell() - 1 + self._size = self._file.tell() self._file.seek(orig) - return self._size + self._maximum_address = self._size - 1 + return self._maximum_address @property def minimum_address(self) -> int: From be82c1639c051f929cbc17c6aa8e7250d6711f8b Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 23 May 2022 09:40:25 +0300 Subject: [PATCH 062/181] fix missing --- volatility3/framework/layers/physical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 728fdcf31..5d757f482 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -113,7 +113,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): def maximum_address(self) -> int: """Returns the largest available address in the space.""" # Zero based, so we return the size of the file minus 1 - if self._maximum_address + if self._maximum_address: return self._maximum_address with self._lock: orig = self._file.tell() From e15fa0ebad1a46fd990d31181d2dbe8f6b5b994d Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 23 May 2022 09:43:49 +0300 Subject: [PATCH 063/181] declared in constructor --- volatility3/framework/layers/physical.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 5d757f482..5cf0b776d 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -88,6 +88,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): self._accessor = resources.ResourceAccessor() self._file_: Optional[IO[Any]] = None self._size: Optional[int] = None + self._maximum_address: Optional[int] = None # Construct the lock now (shared if made before threading) in case we ever need it self._lock: Union[DummyLock, threading.Lock] = DummyLock() if constants.PARALLELISM == constants.Parallelism.Threading: From 946d2302bbc92e781b1281632c5ea5a669228bd0 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 23 May 2022 17:44:52 +0300 Subject: [PATCH 064/181] log resource cache usage --- volatility3/framework/layers/resources.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index ac25b5cc2..8a0e96208 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -171,6 +171,8 @@ class ResourceAccessor(object): cache_file.write(block) block = fp.read(block_size) cache_file.close() + else: + vollog.debug(f"Using already cached file at: {temp_filename}") # Re-open the cache with a different mode # Since we don't want people thinking they're able to save to the cache file, # open it in read mode only and allow breakages to happen if they wanted to write From 4dd8114dcb565daddbd105809252b5517f87a5a1 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 29 May 2022 11:28:40 +0300 Subject: [PATCH 065/181] check return value from is_valid --- .../framework/symbols/windows/extensions/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 69e8ba94e..e9264e0a0 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -746,7 +746,10 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): trans_layer = self._context.layers[layer] try: - trans_layer.is_valid(self.vol.offset) + is_valid = trans_layer.is_valid(self.vol.offset) + if not is_valid: + return + link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: return @@ -762,7 +765,9 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): obj_offset = link.vol.offset - relative_offset try: - trans_layer.is_valid(obj_offset) + is_valid = trans_layer.is_valid(obj_offset) + if not is_valid: + return except exceptions.InvalidAddressException: return From cdbe41dbf5a2a43714d3bc3579746a057055e07a Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 29 May 2022 12:20:24 +0300 Subject: [PATCH 066/181] removed redundant try catch --- .../framework/symbols/windows/extensions/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index e9264e0a0..b5ee272a0 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -764,11 +764,7 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): while link.vol.offset not in seen: obj_offset = link.vol.offset - relative_offset - try: - is_valid = trans_layer.is_valid(obj_offset) - if not is_valid: - return - except exceptions.InvalidAddressException: + if not trans_layer.is_valid(obj_offset): return obj = self._context.object(symbol_type, From b9694e109a03f9589f124cb3d88c4ec4e58cc719 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 02:48:46 +0900 Subject: [PATCH 067/181] Add: JSON EOF for config file --- volatility3/cli/__init__.py | 1 + volatility3/cli/volshell/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index e3fb726a1..8851e2b18 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -332,6 +332,7 @@ class CommandLine: parser.error(f"Cannot write configuration: file {args.save_config} already exists") with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + f.write("\n") except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 30fe75e06..769e958fd 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -246,6 +246,7 @@ class VolShell(cli.CommandLine): parser.error(f"Cannot write configuration: file {args.save_config} already exists") with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + f.write("\n") except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") From bd332261dede65118114e6484c4d8ce446d3b165 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 05:39:37 +0900 Subject: [PATCH 068/181] Fix: __del__ to __exit__ --- volatility3/framework/layers/physical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 5cf0b776d..0633637ca 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -191,7 +191,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): """Closes the file handle.""" self._file.close() - def __del__(self) -> None: + def __exit__(self) -> None: self.destroy() @classmethod From a4e162c7f1dd8597cd0b9a0a8e175573c7fb8c62 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 07:24:45 +0900 Subject: [PATCH 069/181] Fix: minor for better code --- volatility3/framework/plugins/mac/kauth_listeners.py | 2 +- volatility3/framework/plugins/windows/skeleton_key_check.py | 2 +- volatility3/framework/symbols/metadata.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/mac/kauth_listeners.py b/volatility3/framework/plugins/mac/kauth_listeners.py index 7002d88e2..fba6a8e0a 100644 --- a/volatility3/framework/plugins/mac/kauth_listeners.py +++ b/volatility3/framework/plugins/mac/kauth_listeners.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index cd4a5baec..4a1b48c9a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -9,7 +9,7 @@ # For a thorough walkthrough on how the R&D was performed to develop this plugin, # please see our blogpost here: # -# +# https://volatility-labs.blogspot.com/2021/10/memory-forensics-r-illustrated.html import io import logging diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 7cde686ee..350bb0a53 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -38,4 +38,4 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): class LinuxMetadata(interfaces.symbols.MetadataInterface): - """Class to handle the etadata from a Linux symbol table.""" + """Class to handle the metadata from a Linux symbol table.""" From 1ef8c5167722aaed65e163be3ab1d1f06c6117bb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 08:29:13 +0900 Subject: [PATCH 070/181] Fix: minor code for improve --- volatility3/framework/plugins/linux/check_syscall.py | 2 +- volatility3/framework/plugins/linux/mountinfo.py | 1 - volatility3/framework/plugins/windows/ssdt.py | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 87d252cd5..50fd05fa5 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -80,7 +80,7 @@ class Check_syscall(plugins.PluginInterface): def _get_table_info_disassembly(self, ptr_sz, vmlinux): """Find the size of the system call table by disassembling functions - that immediately reference it in their first isntruction This is in the + that immediately reference it in their first instruction This is in the form 'cmp reg,NR_syscalls'.""" table_size = 0 diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 6f3cb712d..551d128ad 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -1,7 +1,6 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -# Author: Gustavo Moreira import logging from collections import namedtuple diff --git a/volatility3/framework/plugins/windows/ssdt.py b/volatility3/framework/plugins/windows/ssdt.py index 0d921535d..78fd72630 100644 --- a/volatility3/framework/plugins/windows/ssdt.py +++ b/volatility3/framework/plugins/windows/ssdt.py @@ -95,10 +95,10 @@ class SSDT(plugins.PluginInterface): if is_kernel_64: array_subtype = "long" - def kvo_calulator(func: int) -> int: + def kvo_calculator(func: int) -> int: return kvo + service_table_address + (func >> 4) - find_address = kvo_calulator + find_address = kvo_calculator else: array_subtype = "unsigned long" From 8b128b05f834c210ce607ab40d386718b5b363b5 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 22:17:33 +0900 Subject: [PATCH 071/181] Fix: typo for code comments --- volatility3/framework/objects/templates.py | 2 +- volatility3/framework/plugins/linux/psaux.py | 2 +- volatility3/framework/plugins/linux/pstree.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/objects/templates.py b/volatility3/framework/objects/templates.py index b544d117f..56754d255 100644 --- a/volatility3/framework/objects/templates.py +++ b/volatility3/framework/objects/templates.py @@ -63,7 +63,7 @@ class ObjectTemplate(interfaces.objects.Template): object_info: interfaces.objects.ObjectInformation) -> interfaces.objects.ObjectInterface: """Constructs the object. - Returns: an object adhereing to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface` + Returns: an object adhering to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface` """ arguments: Dict[str, Any] = {} for arg in self.vol: diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index c62712907..ed91c66f2 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -40,7 +40,7 @@ class PsAux(plugins.PluginInterface): name: string name of the process (from task.comm) """ - # kernel theads never have an mm as they do not have userland mappings + # kernel threads never have an mm as they do not have userland mappings try: mm = task.mm except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index a44310147..3ad5f3e19 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -19,7 +19,7 @@ class PsTree(pslist.PsList): """Finds how deep the PID is in the tasks hierarchy. Args: - pid: PID to find the level in the hierachy + pid: PID to find the level in the hierarchy """ seen = set([pid]) level = 0 From bb80d7067e99d55748e1067e6d11e22c2ffe5e4d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 31 May 2022 23:24:49 +0900 Subject: [PATCH 072/181] Fix: typo of timeliner parameter --- volatility3/framework/plugins/timeliner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 8785f62e1..c1d29062d 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -101,7 +101,7 @@ class Timeliner(interfaces.plugins.PluginInterface): return [sortable(timestamp) for timestamp in data[2:]] - def _generator(self, runable_plugins: List[TimeLinerInterface]) -> Optional[Iterable[Tuple[int, Tuple]]]: + def _generator(self, runnable_plugins: List[TimeLinerInterface]) -> Optional[Iterable[Tuple[int, Tuple]]]: """Takes a timeline, sorts it and output the data from each relevant row from each plugin.""" # Generate the results for each plugin @@ -115,9 +115,9 @@ class Timeliner(interfaces.plugins.PluginInterface): file_data = None fp = None - for plugin in runable_plugins: + for plugin in runnable_plugins: plugin_name = plugin.__class__.__name__ - self._progress_callback((runable_plugins.index(plugin) * 100) // len(runable_plugins), + self._progress_callback((runnable_plugins.index(plugin) * 100) // len(runnable_plugins), f"Running plugin {plugin_name}...") try: vollog.log(logging.INFO, f"Running {plugin_name}") From 0abd2e53abefd0856c83bfad2cf61ab500868d38 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 5 Jun 2022 10:56:42 +0100 Subject: [PATCH 073/181] Pyinstaller: Fix path need to current directory to be correct --- vol.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vol.spec b/vol.spec index 42b69af3f..666526dde 100644 --- a/vol.spec +++ b/vol.spec @@ -26,7 +26,7 @@ except ImportError: # Volatility must be findable in sys.path in order for collect_submodules to work # This adds the current working directory, which should usually do the trick -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.abspath(SPEC))) vol_analysis = Analysis(['vol.py'], pathex = [], From db3408bdaa978de5b23eecd2d4411df8a6de1f16 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 5 Jun 2022 22:23:30 +0100 Subject: [PATCH 074/181] Windows: Extend the pdb support to modules --- .../framework/symbols/windows/pdbutil.py | 75 +++++++++++++++---- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 585e96b6d..41037d464 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -10,10 +10,10 @@ import os import re import struct from typing import Any, Dict, Generator, List, Optional, Tuple, Union -from urllib import request, parse +from urllib import parse, request from volatility3 import symbols -from volatility3.framework import constants, interfaces, exceptions +from volatility3.framework import constants, contexts, exceptions, interfaces from volatility3.framework.configuration.requirements import SymbolTableRequirement from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbconv @@ -24,7 +24,7 @@ vollog = logging.getLogger(__name__) class PDBUtility(interfaces.configuration.VersionableInterface): """Class to handle and manage all getting symbols based on MZ header""" - _version = (1, 0, 0) + _version = (1, 0, 1) _required_framework_version = (2, 0, 0) @classmethod @@ -131,14 +131,14 @@ class PDBUtility(interfaces.configuration.VersionableInterface): # Check it is actually the MZ header if mz_sig != b"MZ": return None - + nt_header_start, = struct.unpack(" str: + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: """Creates symbol table for a module in the specified layer_name. Searches the memory section of the loaded module for its PDB GUID @@ -307,6 +307,19 @@ class PDBUtility(interfaces.configuration.VersionableInterface): Returns: The name of the constructed and loaded symbol table """ + _, symbol_table_name = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size) + return symbol_table_name + + @classmethod + def _modtable_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None, + create_module: bool = False) -> Tuple[Optional[str], Optional[str]]: + + if module_offset is None: + module_offset = context.layers[layer_name].minimum_address + if module_size is None: + module_size = context.layers[layer_name].maximum_address - module_offset guids = list( cls.pdbname_scan(context, @@ -323,12 +336,46 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - return cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) + module_name = guid["pdb_name"].strip('.pdb') + + symbol_table_name = cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) + + new_module_name = None + if create_module: + new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], + symbol_table_name = symbol_table_name) + new_module_name = new_module.name + + return new_module_name, symbol_table_name + + @classmethod + def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: + """Creates a module in the specified layer_name based on a pdb name. + + Searches the memory section of the loaded module for its PDB GUID + and loads the associated symbol table into the symbol space. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + config_path: The config path where to find symbol files + layer_name: The name of the layer on which to operate + module_offset: This memory dump's module image offset + module_size: The size of the module for this dump + + Returns: + The name of the constructed and loaded symbol table + """ + + module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size, create_module = True) + + return module_name class PdbSignatureScanner(interfaces.layers.ScannerInterface): From 21d916be0a08eccc91bbd4884f458ae6ff489b95 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 6 Jun 2022 14:53:52 +0100 Subject: [PATCH 075/181] Pyinstaller: Support pyinstaller 5 and later --- volatility3/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/__init__.py b/volatility3/__init__.py index db52aa9b0..b6da6e01e 100644 --- a/volatility3/__init__.py +++ b/volatility3/__init__.py @@ -37,9 +37,9 @@ class WarningFindSpec(abc.MetaPathFinder): first.""" if fullname.startswith("volatility3.framework.plugins."): warning = "Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins" - # Pyinstaller uses walk_packages to import, but needs to read the modules to figure out dependencies - # As such, we only print the warning when directly imported rather than from within walk_packages - if inspect.stack()[-2].function != 'walk_packages': + # Pyinstaller uses walk_packages/_collect_submodules to import, but needs to read the modules to figure out dependencies + # As such, we only print the warning when directly imported rather than from within walk_packages/_collect_submodules + if inspect.stack()[-2].function in ['walk_packages', '_collect_submodules']: raise Warning(warning) From aa06ed6e674761c8ec1238daeff9a64603aff392 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 12 Jun 2022 20:32:06 +0900 Subject: [PATCH 076/181] Add: new options for vol-cli.rst --- doc/source/vol-cli.rst | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 9db29c818..902787c9c 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -9,7 +9,11 @@ Synopsis **volatility** [-h] [-c CONFIG] [--parallelism [{processes,threads,off}]] [-e EXTEND] [-p PLUGIN_DIRS] [-s SYMBOL_DIRS] [-v] [-l LOG] [-o OUTPUT_DIR] [-q] [-r RENDERER] [-f FILE] - [--write-config] [--single-location SINGLE_LOCATION] + [--write-config] [--save-config SAVE_CONFIG] + [--clear-cache] [--cache-path CACHE_PATH] + [--offline] + [--single-location SINGLE_LOCATION] + [--stackers [STACKERS ...]] [--single-swap-locations SINGLE_SWAP_LOCATIONS] ... @@ -105,11 +109,31 @@ Options other plugins, but there's no guarantee that plugins use the same configuration options. +--save-config + This flag specifies that volatility should write or overwrite a file + called config.json in the current directory. The file will contain + the necessary JSON configuration to recreate the environment that the + plugin was previously run in. This configuration *may* be accepted by + other plugins, but there's no guarantee that plugins use the same + configuration options. + +--clear-cache + Clears out all short-term cached items. + +--cache-path + Change the default path ({constants.CACHE_PATH}) used to store the cache. + +--offline + Do not search online for additional JSON files. + --single-location SINGLE_LOCATION This specifies a URL which will be downloaded if necessary, and built upon by the automagic and, since most plugins require a single memory image, can be considered the input for the program. +--stackers STACKERS + + --single-swap-locations SINGLE_SWAP_LOCATIONS A comma-separated list of swap files to be considered as part of the memory image specified by the single-location or file parameters. From 2d14e4e012d6745862fafa1a857414102a412eb1 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 12 Jun 2022 20:50:03 +0900 Subject: [PATCH 077/181] Add: descriptions of new options --- doc/source/vol-cli.rst | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 902787c9c..b9e16623d 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -102,12 +102,8 @@ Options attempt to build upon, and can be considered the input for the program. --write-config - This flag specifies that volatility should write or overwrite a file - called config.json in the current directory. The file will contain - the necessary JSON configuration to recreate the environment that the - plugin was previously run in. This configuration *may* be accepted by - other plugins, but there's no guarantee that plugins use the same - configuration options. + *Deprecated* + Use of `--write-config` has been deprecated, replaced by `--save-config` --save-config This flag specifies that volatility should write or overwrite a file @@ -121,19 +117,18 @@ Options Clears out all short-term cached items. --cache-path - Change the default path ({constants.CACHE_PATH}) used to store the cache. + Change the default path used to store the cache. --offline Do not search online for additional JSON files. + Run offline mode (defaults to false) and for + remote windows symbol tables, linux/mac banner repositories. --single-location SINGLE_LOCATION This specifies a URL which will be downloaded if necessary, and built upon by the automagic and, since most plugins require a single memory image, can be considered the input for the program. ---stackers STACKERS - - --single-swap-locations SINGLE_SWAP_LOCATIONS A comma-separated list of swap files to be considered as part of the memory image specified by the single-location or file parameters. From 3ef505641eb2f7d3d76174effb18cba69434298f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 12 Jun 2022 20:55:15 +0900 Subject: [PATCH 078/181] Add: stacker descriptions --- doc/source/vol-cli.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index b9e16623d..cc6f7fe6a 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -129,6 +129,9 @@ Options upon by the automagic and, since most plugins require a single memory image, can be considered the input for the program. +--stackers STACKERS + Creates the list of stackers to use based on the config option. + --single-swap-locations SINGLE_SWAP_LOCATIONS A comma-separated list of swap files to be considered as part of the memory image specified by the single-location or file parameters. From e1f3f65202d7eb23901a4c9639ad1523f4429369 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 14 Jun 2022 21:03:33 +0900 Subject: [PATCH 079/181] Fix: typo for code comment, requirements name --- volatility3/framework/interfaces/configuration.py | 2 +- volatility3/framework/interfaces/layers.py | 4 ++-- volatility3/framework/plugins/mac/kevents.py | 2 +- volatility3/framework/plugins/windows/modscan.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index c39dba680..e271ef6d4 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -523,7 +523,7 @@ class ConstructableRequirementInterface(RequirementInterface): must happen after the class configuration value has been provided). These values are then provided to the object's constructor by name as arguments (as well as the standard `context` and `config_path` - arguments. + arguments). """ def __init__(self, *args, **kwargs) -> None: diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index a42282c39..7ff110c6e 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -307,7 +307,7 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla while length > 0: chunk_size = min(length, scanner.chunk_size + scanner.overlap) yield [(layer_name, mapped_offset, chunk_size)], offset + chunk_size - # It we've got more than the scanner's chunk_size, only move up by the chunk_size + # If we've got more than the scanner's chunk_size, only move up by the chunk_size if chunk_size > scanner.chunk_size: chunk_size -= scanner.overlap length -= chunk_size @@ -517,7 +517,7 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): yield output, chunk_position output = [] chunk_position = chunk_start - # Take from chunk_position as far as far as the block can go, + # Take from chunk_position as far as the block can go, # or as much left of a scanner chunk as we can chunk_size = min(block_end - chunk_position, scanner.chunk_size + scanner.overlap - (chunk_position - chunk_start)) diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 6f82c75cd..4a82d81cd 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -74,7 +74,7 @@ class Kevents(interfaces.plugins.PluginInterface): @classmethod def _walk_klist_array(cls, kernel, fdp, array_pointer_member, array_size_member): """ - Convience wrapper for walking an array of lists of kernel events + Convenience wrapper for walking an array of lists of kernel events Handles invalid address references """ try: diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index b661d71d7..e352c21fe 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -25,7 +25,7 @@ class ModScan(interfaces.plugins.PluginInterface): return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'poolerscanner', + requirements.VersionRequirement(name = 'poolscanner', component = poolscanner.PoolScanner, version = (1, 0, 0)), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), From dd92955a99249fe9e8863cb2754229e01a917d73 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 16 Jun 2022 05:40:26 +0900 Subject: [PATCH 080/181] Remove: unreachable code --- volatility3/cli/volshell/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 769e958fd..5eeef77cf 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -257,7 +257,6 @@ class VolShell(cli.CommandLine): constructed.run() except exceptions.VolatilityException as excp: self.process_exceptions(excp) - parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") def main(): From 9f525dfa733dd65769458540d3996917a1daaa96 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 22 Jun 2022 19:57:47 +0900 Subject: [PATCH 081/181] Refactor: simplify comparision --- volatility3/framework/automagic/pdbscan.py | 2 +- volatility3/framework/plugins/windows/ldrmodules.py | 12 ++++++------ volatility3/framework/plugins/windows/vadinfo.py | 2 +- .../framework/symbols/linux/extensions/__init__.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 5db66a3d0..cedbc4919 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -148,7 +148,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vollog.debug("Kernel base determination - optimized scan virtual layer") valid_kernel = self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, True, False, progress_callback) - if valid_kernel != None: + if valid_kernel is not None: return valid_kernel vollog.debug("Kernel base determination - slow scan virtual layer") diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index e7c96e946..284d1afc2 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -71,14 +71,14 @@ class LdrModules(interfaces.plugins.PluginInterface): mem_mod = mem_order_mod.get(base, None) yield (0, [int(proc.UniqueProcessId), - str(proc.ImageFileName.cast("string", + str(proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, errors = 'replace')), - format_hints.Hex(base), - load_mod != None, - init_mod != None, - mem_mod != None, - mapped_files[base]]) + format_hints.Hex(base), + load_mod is not None, + init_mod is not None, + mem_mod is not None, + mapped_files[base]]) def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 9fa1458d1..e357b150a 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -132,7 +132,7 @@ class VadInfo(interfaces.plugins.PluginInterface): vollog.debug("Unable to find the starting/ending VPN member") return None - if maxsize > 0 and (vad_end - vad_start) > maxsize: + if 0 < maxsize < (vad_end - vad_start): vollog.debug(f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit") return None diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6792ab19c..73f31115a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -409,7 +409,7 @@ class vm_area_struct(objects.StructType): fname = linux.LinuxUtilities.path_for_file(context, task, self.vm_file) elif self.vm_start <= task.mm.start_brk and self.vm_end >= task.mm.brk: fname = "[heap]" - elif self.vm_start <= task.mm.start_stack and self.vm_end >= task.mm.start_stack: + elif self.vm_start <= task.mm.start_stack <= self.vm_end: fname = "[stack]" elif self.vm_mm.context.has_member("vdso") and self.vm_start == self.vm_mm.context.vdso: fname = "[vdso]" From 7ba27a75ca9cbecbe796f6475340718e6bce0dd0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 22 Jun 2022 14:49:57 +0100 Subject: [PATCH 082/181] Documentation: Improve the simple-plugin example --- doc/source/simple-plugin.rst | 54 ++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 8446b0ef5..904c586c7 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -30,6 +30,9 @@ to be able to run properly. Any that are defined as optional need not necessari :: + _version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + @classmethod def get_requirements(cls): return [requirements.TranslationLayerRequirement(name = 'primary', @@ -37,13 +40,13 @@ to be able to run properly. Any that are defined as optional need not necessari architectures = ["Intel32", "Intel64"]), requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (1, 0, 0)), requirements.ListRequirement(name = 'pid', element_type = int, description = "Process IDs to include (all other processes are excluded)", - optional = True)] + optional = True), + requirements.PluginRequirement(name = 'pslist', + plugin = pslist.PsList, + version = (1, 0, 0))] This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how @@ -91,29 +94,44 @@ name of the :py:class:`SymbolTable Date: Wed, 22 Jun 2022 15:12:40 +0100 Subject: [PATCH 083/181] Documentation: Update the documentation to the latest framework --- doc/source/simple-plugin.rst | 103 ++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 904c586c7..543451b88 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -35,11 +35,8 @@ to be able to run properly. Any that are defined as optional need not necessari @classmethod def get_requirements(cls): - return [requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.ListRequirement(name = 'pid', element_type = int, description = "Process IDs to include (all other processes are excluded)", @@ -54,45 +51,73 @@ to instantiate the plugin). At the moment these requirements are fairly straigh :: - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), -This requirement indicates that the plugin will operate on a single -:py:class:`TranslationLayer `. The name of the -loaded layer will appear in the plugin's configuration under the name ``primary``. Requirement values can be -accessed within the plugin through the plugin's `config` attribute (for example ``self.config['pid']``). +This requirement specifies the need for a particular submodule. Each module requires a +:py:class:`TranslationLayer ` and a +:py:class:`SymbolTable `, which are fulfilled by two +subrequirements: a +:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` and a +:py:class:`~volatility3.framework.configuration.requirements.SymbolTableRequirement`. At the moment, the automagic +only fills `ModuleRequirements` with kernels, and so has relatively few parameters. It requires the architecture for +the underlying TranslationLayer, and the offset of the module within that layer. -.. note:: The name itself is dynamic depending on the other layers already present in the Context. Always use the value - from the configuration rather than attempting to guess what the layer will be called. +The name of the module will be stored in the ``kernel`` configuration option, and the module object itself +can be accessed from the ``context.modules`` collection. This requirement is a Complex Requirement and therefore will +not be requested directly from the user. -Finally, this defines that the translation layer must be on the Intel Architecture. At the moment, this acts as a filter, -failing to be satisfied by memory images that do not match the architecture required. -Most plugins will only operate on a single layer, but it is entirely possible for a plugin to request two different -layers, for example a plugin that carries out some form of difference or statistics against multiple memory images. +.. note:: -This requirement (and the next two) are known as Complex Requirements, and user interfaces will likely not directly -request a value for this from a user. The value stored in the configuration tree for a -:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` is -the string name of a layer present in the context's memory that satisfies the requirement. + In previous versions of volatility 3, there was no `ModuleRequirement`, and instead two requirements were defined + a :py:class:`TranslationLayer ` and a `SymbolTableRequirement`. These still exist, and can be used, most plugins just + define a single `ModuleRequirement` for the kernel, which the automagic will populate. The `ModuleRequirement` has + two automatic sub-requirements, a `TranslationLayerRequirement` and a `SymbolTableRequirement`, but the module also + includes the offset of the module, and will allow future expansion to specify specific modules when application + level plugins become more common. Below are how the requirements would be specified: -:: + :: - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), -This requirement specifies the need for a particular -:py:class:`SymbolTable ` -to be loaded. This gets populated by various -:py:class:`Automagic ` as the nearest sibling to a particular -:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`. -This means that if the :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` -is satisfied and the :py:class:`Automagic ` can determine -the appropriate :py:class:`SymbolTable `, the -name of the :py:class:`SymbolTable ` will be stored in the configuration. + This requirement indicates that the plugin will operate on a single + :py:class:`TranslationLayer `. The name of the + loaded layer will appear in the plugin's configuration under the name ``primary``. Requirement values can be + accessed within the plugin through the plugin's `config` attribute (for example ``self.config['pid']``). -This requirement is also a Complex Requirement and therefore will not be requested directly from the user. + .. note:: The name itself is dynamic depending on the other layers already present in the Context. Always use the value + from the configuration rather than attempting to guess what the layer will be called. + + Finally, this defines that the translation layer must be on the Intel Architecture. At the moment, this acts as a filter, + failing to be satisfied by memory images that do not match the architecture required. + + Most plugins will only operate on a single layer, but it is entirely possible for a plugin to request two different + layers, for example a plugin that carries out some form of difference or statistics against multiple memory images. + + This requirement (and the next two) are known as Complex Requirements, and user interfaces will likely not directly + request a value for this from a user. The value stored in the configuration tree for a + :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` is + the string name of a layer present in the context's memory that satisfies the requirement. + + :: + + requirements.SymbolTableRequirement(name = "nt_symbols", + description = "Windows kernel symbols"), + + This requirement specifies the need for a particular + :py:class:`SymbolTable ` + to be loaded. This gets populated by various + :py:class:`Automagic ` as the nearest sibling to a particular + :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`. + This means that if the :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` + is satisfied and the :py:class:`Automagic ` can determine + the appropriate :py:class:`SymbolTable `, the + name of the :py:class:`SymbolTable ` will be stored in the configuration. + + This requirement is also a Complex Requirement and therefore will not be requested directly from the user. :: @@ -147,6 +172,7 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] return renderers.TreeGrid([("PID", int), ("Process", str), @@ -155,8 +181,8 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. ("Name", str), ("Path", str)], self._generator(pslist.PsList.list_processes(self.context, - self.config['primary'], - self.config['nt_symbols'], + kernel.layer_name, + kernel.symbol_table_name, filter_func = filter_func))) In this instance, the plugin constructs a filter (using the PsList plugin's *classmethod* for creating filters). @@ -175,7 +201,8 @@ the :py:class:`~volatility3.plugins.windows.pslist.PsList` plugin. That plugin so that other plugins can call it. As such, it takes all the necessary parameters rather than accessing them from a configuration. Since it must be portable code, it takes a context, as well as the layer name, symbol table and optionally a filter. In this instance we unconditionally -pass it the values from the configuration for the ``primary`` and ``nt_symbols`` requirements. This will generate a list +pass it the values from the configuration for the layer and symbol table from the kernel module object, constructed from +the ``kernel`` configuration requirement. This will generate a list of :py:class:`~volatility3.framework.symbols.windows.extensions.EPROCESS` objects, as provided by the :py:class:`~volatility.plugins.windows.pslist.PsList` plugin, and is not covered here but is used as an example for how to share code across plugins (both as the provider and the consumer of the shared code). From fd524a6b314750bd07779f65257d010ed635b1f6 Mon Sep 17 00:00:00 2001 From: ikelos Date: Wed, 22 Jun 2022 17:08:18 +0100 Subject: [PATCH 084/181] Update doc/source/simple-plugin.rst Yep, that seems fine. Co-authored-by: Donghyun Kim --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 543451b88..d03f7c7d6 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -43,7 +43,7 @@ to be able to run properly. Any that are defined as optional need not necessari optional = True), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, - version = (1, 0, 0))] + version = (2, 0, 0))] This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how From a386de72f5a22d176ecad730e83f804e2f62c633 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 22 Jun 2022 17:12:24 +0100 Subject: [PATCH 085/181] Documentation: Fix pslist plugin requirement --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index d03f7c7d6..1c7b91205 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -134,7 +134,7 @@ being defined within the configuration tree at all. requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, - version = (1, 0, 0)) + version = (2, 0, 0))] This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements on that plugin. The version is specified in terms of Semantic Versioning meaning that, to be compatible, the major From aed87346cdd362fb59fce772cbd62b0dded51bf5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 23 Jun 2022 09:23:31 +0100 Subject: [PATCH 086/181] Core: Add support to templates to get child templates --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/interfaces/objects.py | 11 +++++++++++ volatility3/framework/objects/__init__.py | 17 +++++++++++++++++ volatility3/framework/objects/templates.py | 7 +++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 472a743e6..f08819f29 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 2 # Number of changes that only add to the interface +VERSION_MINOR = 3 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 2240c58c9..3cc23e759 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -241,6 +241,13 @@ class ObjectInterface(metaclass = abc.ABCMeta): the child member.""" raise KeyError(f"Template does not contain any children: {template.vol.type_name}") + @classmethod + @abc.abstractmethod + def child_template(cls, template: 'Template', child: str) -> interfaces.objects.Template: + """Returns the template of the child member from the parent.""" + raise KeyError(f"Template does not contain any children: {template.vol.type_name}") + + @classmethod @abc.abstractmethod def has_member(cls, template: 'Template', member_name: str) -> bool: @@ -305,6 +312,10 @@ class Template: """Returns the relative offset of the `child` member from its parent offset.""" + @abc.abstractmethod + def child_template(self, child: str) -> interfaces.objects.Template: + """Returns the `child` member template from its parent.""" + @abc.abstractmethod def replace_child(self, old_child: 'Template', new_child: 'Template') -> None: """Replaces `old_child` with `new_child` in the list of children.""" diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index e0f927ec9..feb49a089 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -602,6 +602,14 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): return 0 raise IndexError(f"Member not present in array template: {child}") + @classmethod + def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: + """Returns the template of the child member.""" + if 'subtype' in template.vol and child == 'subtype'@ + return template.vol.subtype + raise IndexError(f"Member not present in array template: {child}") + + @overload def __getitem__(self, i: int) -> interfaces.objects.Template: ... @@ -715,6 +723,15 @@ class AggregateType(interfaces.objects.ObjectInterface): raise IndexError(f"Member not present in template: {child}") return retlist[0] + @classmethod + def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: + """Returns the template of a child to its parent.""" + retlist = template.vol.members.get(child, None) + if retlist is None: + raise IndexError(f"Member not present in template: {child}") + return retlist[1] + + @classmethod def has_member(cls, template: interfaces.objects.Template, member_name: str) -> bool: """Returns whether the object would contain a member called diff --git a/volatility3/framework/objects/templates.py b/volatility3/framework/objects/templates.py index 56754d255..e8b523373 100644 --- a/volatility3/framework/objects/templates.py +++ b/volatility3/framework/objects/templates.py @@ -48,6 +48,12 @@ class ObjectTemplate(interfaces.objects.Template): plateProxy`)""" return self.vol.object_class.VolTemplateProxy.relative_child_offset(self, child) + def child_template(self, child: str) -> interfaces.objects.Template: + """Returns the template of a child of the templated object (see + :class:`~volatility3.framework.interfaces.objects.ObjectInterface.VolTem + plateProxy`)""" + return self.vol.object_class.VolTemplateProxy.child_template(self, child) + def replace_child(self, old_child: interfaces.objects.Template, new_child: interfaces.objects.Template) -> None: """Replaces `old_child` for `new_child` in the templated object's child list (see :class:`~volatility3.framework.interfaces.objects.ObjectInterf @@ -99,6 +105,7 @@ class ReferenceTemplate(interfaces.objects.Template): size: ClassVar[Any] = property(_unresolved) replace_child: ClassVar[Any] = _unresolved relative_child_offset: ClassVar[Any] = _unresolved + child_template: ClassVar[Any] = _unresolved has_member: ClassVar[Any] = _unresolved def __call__(self, context: interfaces.context.ContextInterface, object_info: interfaces.objects.ObjectInformation): From 6982650c188a7c8fccbbee3c1d7d1f47f309df28 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Jun 2022 15:38:48 +0100 Subject: [PATCH 087/181] Volshell: Fixes use of old config variables Closes #780 --- volatility3/cli/volshell/linux.py | 4 ++-- volatility3/cli/volshell/mac.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 97a488743..0f2a90c7e 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -56,13 +56,13 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['vmlinux'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: - symbol_table = self.config['vmlinux'] + symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) @property diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 305f80505..6744f3394 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -56,7 +56,7 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['darwin'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): From e8e6bacb194933de3402a182ffc3dd070256e32b Mon Sep 17 00:00:00 2001 From: ikelos Date: Thu, 30 Jun 2022 09:57:09 +0100 Subject: [PATCH 088/181] Update volatility3/framework/objects/__init__.py Fix typo Co-authored-by: Donghyun Kim --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index feb49a089..62e6de553 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -605,7 +605,7 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): @classmethod def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: """Returns the template of the child member.""" - if 'subtype' in template.vol and child == 'subtype'@ + if 'subtype' in template.vol and child == 'subtype': return template.vol.subtype raise IndexError(f"Member not present in array template: {child}") From 951a0f5d508b8db4985d54751ea93ca78e57b191 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 30 Jun 2022 11:43:38 +0100 Subject: [PATCH 089/181] Documentation: Clarify that the code is just an example Clarifies for #773 and #776 --- doc/source/simple-plugin.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 1c7b91205..e2143f1b7 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -6,6 +6,12 @@ This guide will step through how to construct a simple plugin using Volatility 3 The example plugin we'll use is :py:class:`~volatility3.plugins.windows.dlllist.DllList`, which features the main traits of a normal plugin, and reuses other plugins appropriately. +.. note:: + + This document will not include the complete code necessary for a + working plugin (such as imports, etc) since it's designed to focus on the necessary componets for writing a plugin. + For complete and functioning plugins, the ``framework/plugins`` directory should be consulted. + Inherit from PluginInterface ---------------------------- From 6d7095fa3bf01aa4f2a9fceb1887cf28ed463e58 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 2 Jul 2022 19:30:00 +0900 Subject: [PATCH 090/181] Add: exceptions code --- .../plugins/windows/registry/certificates.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index d2fb61f02..e2fe662fc 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -17,10 +17,8 @@ class Certificates(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)), requirements.BooleanRequirement(name = 'dump', @@ -58,10 +56,12 @@ class Certificates(interfaces.plugins.PluginInterface): def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: + kernel = self.context.modules[self.config['kernel']] + for hive in hivelist.HiveList.list_hives(self.context, base_config_path = self.config_path, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols']): + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name): for top_key in [ "Microsoft\\SystemCertificates", @@ -87,6 +87,12 @@ class Certificates(interfaces.plugins.PluginInterface): # Key wasn't found in this hive, carry on vollog.log(constants.LOGLEVEL_VVVV, "Key wasn't found in this hive") pass + except exceptions.SwappedInvalidAddressException as exp: + vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)") + pass + except exceptions.PagedInvalidAddressException as exp: + vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)") + pass def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), From c40aecdfdacfde5ac17b658b581aa85595272b85 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 2 Jul 2022 19:38:12 +0900 Subject: [PATCH 091/181] Remove: invalid exceptions code --- volatility3/plugins/windows/registry/certificates.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index e2fe662fc..e873fd1d6 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -90,9 +90,6 @@ class Certificates(interfaces.plugins.PluginInterface): except exceptions.SwappedInvalidAddressException as exp: vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)") pass - except exceptions.PagedInvalidAddressException as exp: - vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)") - pass def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), From dcc774787cf333ac574ae1dd402f752616e946a4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Jul 2022 22:11:24 +0100 Subject: [PATCH 092/181] Core: Convert try/except/pass to contextlib.supress --- volatility3/framework/automagic/pdbscan.py | 13 ++++---- volatility3/framework/interfaces/objects.py | 5 ++- volatility3/framework/layers/crash.py | 5 ++- volatility3/framework/layers/registry.py | 21 ++++++------ volatility3/framework/layers/resources.py | 4 +-- volatility3/framework/layers/vmware.py | 26 +++++++-------- .../framework/plugins/linux/check_syscall.py | 8 ++--- .../framework/plugins/windows/dlllist.py | 13 ++++---- .../framework/plugins/windows/envars.py | 21 ++++-------- .../framework/plugins/windows/mftscan.py | 7 ++-- .../plugins/windows/registry/userassist.py | 18 ++++------- volatility3/framework/renderers/conversion.py | 6 ++-- .../symbols/mac/extensions/__init__.py | 32 +++++++------------ .../framework/symbols/windows/__init__.py | 14 ++++---- .../symbols/windows/extensions/__init__.py | 21 ++++-------- .../symbols/windows/extensions/pool.py | 28 +++++++--------- .../symbols/windows/extensions/registry.py | 6 ++-- 17 files changed, 97 insertions(+), 151 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index cedbc4919..5cbdbfe0e 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -7,10 +7,11 @@ from loaded PE files. This module contains a standalone scanner, and also a :class:`~volatility3.framework.interfaces.layers.ScannerInterface` based scanner for use within the framework by calling :func:`~volatility3.framework.interfaces.layers.DataLayerInterface.scan`. """ +import contextlib import logging import math import os -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union, Callable +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.configuration import requirements @@ -139,7 +140,8 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]: + def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ + ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): # Rule out kernels that couldn't find a suitable MZ header @@ -159,7 +161,8 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - def test_physical_kernel(physical_layer_name:str , virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]: + def test_physical_kernel(physical_layer_name: str, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ + ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): # Rule out kernels that couldn't find a suitable MZ header @@ -274,7 +277,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] virtual_layer_name = vlayer.name - try: + with contextlib.suppress(exceptions.InvalidAddressException): if vlayer.read(address, 0x2) == b'MZ': res = list( PDBUtility.pdbname_scan(ctx = context, @@ -286,8 +289,6 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): end = address + self.max_pdb_size)) if res: valid_kernel = (virtual_layer_name, address, res[0]) - except exceptions.InvalidAddressException: - pass return valid_kernel # List of methods to be run, in order, to determine the valid kernels diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 2240c58c9..0f8e742fb 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -6,6 +6,7 @@ interpreted values of data from a layer.""" import abc import collections import collections.abc +import contextlib import logging from typing import Any, Dict, List, Mapping, Optional @@ -187,11 +188,9 @@ class ObjectInterface(metaclass = abc.ABCMeta): """ if self.has_member(member_name): # noinspection PyBroadException - try: + with contextlib.suppress(Exception): _ = getattr(self, member_name) return True - except Exception: - pass return False def has_valid_members(self, member_names: List[str]) -> bool: diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index c690c8d8f..6194501ee 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -1,6 +1,7 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib import logging import struct from typing import Tuple, Optional @@ -202,11 +203,9 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: for layer in [WindowsCrashDump32Layer, WindowsCrashDump64Layer]: - try: + with contextlib.suppress(WindowsCrashDumpFormatException): layer.check_header(context.layers[layer_name]) new_name = context.layers.free_layer_name(layer.__name__) context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name return layer(context, new_name, new_name) - except WindowsCrashDumpFormatException: - pass return None diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 55a6e5186..ec7aed217 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import logging from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union @@ -92,11 +92,9 @@ class RegistryHive(linear.LinearlyMappedLayer): @property def root_cell_offset(self) -> int: """Returns the offset for the root cell in this hive.""" - try: + with contextlib.suppress(InvalidAddressException): if self._base_block.Signature.cast("string", max_length = 4, encoding = "latin-1") == 'regf': return self._base_block.RootCell - except InvalidAddressException: - pass return 0x20 def get_cell(self, cell_offset: int) -> 'objects.StructType': @@ -201,11 +199,11 @@ class RegistryHive(linear.LinearlyMappedLayer): if offset & 0x7fffffff > self._get_hive_maxaddr(volatile): vollog.log(constants.LOGLEVEL_VVV, "Layer {} couldn't translate offset {}, greater than {} in {} store of {}".format( - self.name, - hex(offset & 0x7fffffff), - hex(self._get_hive_maxaddr(volatile)), - "volative" if volatile else "non-volatile", - self.get_name())) + self.name, + hex(offset & 0x7fffffff), + hex(self._get_hive_maxaddr(volatile)), + "volative" if volatile else "non-volatile", + self.get_name())) raise RegistryInvalidIndex(self.name, "Mapping request for value greater than maxaddr") storage = self.hive.Storage[volatile] @@ -252,14 +250,13 @@ class RegistryHive(linear.LinearlyMappedLayer): def is_valid(self, offset: int, length: int = 1) -> bool: """Returns a boolean based on whether the offset is valid or not.""" - try: + with contextlib.suppress(exceptions.InvalidAddressException): # Pass this to the lower layers for now return all([ self.context.layers[layer].is_valid(offset, length) for (_, _, offset, length, layer) in self.mapping(offset, length) ]) - except exceptions.InvalidAddressException: - return False + return False @property def minimum_address(self) -> int: diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 8a0e96208..dca215c85 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -184,14 +184,12 @@ class ResourceAccessor(object): stop = False while not stop: detected = None - try: + with contextlib.suppress(AttributeError, IOError): # Detect the content detected = magic.detect_from_fobj(curfile) IMPORTED_MAGIC = True # This is because python-magic and file provide a magic module # Only file's python has magic.detect_from_fobj - except (AttributeError, IOError): - pass if detected: if detected.mime_type == 'application/x-xz': diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 85e961b24..ae4a7d55e 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -1,14 +1,14 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import logging import struct from typing import Any, Dict, List, Optional -from volatility3.framework import interfaces, constants, exceptions +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.layers import physical, segmented, resources +from volatility3.framework.layers import physical, resources, segmented from volatility3.framework.symbols import native vollog = logging.getLogger(__name__) @@ -87,13 +87,13 @@ class VmwareLayer(segmented.SegmentedLayer): offset = offset + name_len + 2 + (index * index_len), layer_name = self._meta_layer)) data_len = flags & 0x3f - + if data_len in [62, 63]: # Handle special data sizes that indicate a longer data stream data_len = 4 if version == 0 else 8 # Read the size of the data data_size = self._context.object(self._choose_type(data_len), - layer_name = self._meta_layer, - offset = offset + 2 + name_len + (indices_len * index_len)) + layer_name = self._meta_layer, + offset = offset + 2 + name_len + (indices_len * index_len)) # Skip two bytes of padding (as it seems?) # Read the actual data data = self._context.object("vmware!bytes", @@ -113,9 +113,9 @@ class VmwareLayer(segmented.SegmentedLayer): if tags[("regionsCount", ())][1] == 0: raise VmwareFormatException(self.name, "VMware VMEM is not split into regions") for region in range(tags[("regionsCount", ())][1]): - offset = tags[("regionPPN", (region, ))][1] * self._page_size - mapped_offset = tags[("regionPageNum", (region, ))][1] * self._page_size - length = tags[("regionSize", (region, ))][1] * self._page_size + offset = tags[("regionPPN", (region,))][1] * self._page_size + mapped_offset = tags[("regionPageNum", (region,))][1] * self._page_size + length = tags[("regionSize", (region,))][1] * self._page_size self._segments.append((offset, mapped_offset, length, length)) @property @@ -153,23 +153,19 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): current_layer_name) vmss_success = False - try: + with contextlib.suppress(IOError): _ = resources.ResourceAccessor().open(vmss).read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmss_success = True - except IOError: - pass vmsn_success = False if not vmss_success: - try: + with contextlib.suppress(IOError): _ = resources.ResourceAccessor().open(vmsn).read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmsn context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmsn_success = True - except IOError: - pass vollog.log(constants.LOGLEVEL_VVVV, f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})") diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 50fd05fa5..6ec5fd354 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -3,11 +3,11 @@ # """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" +import contextlib import logging from typing import List -from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers, constants +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.renderers import format_hints @@ -40,11 +40,9 @@ class Check_syscall(plugins.PluginInterface): symbol_list = [] for sn in vmlinux.symbols: - try: + with contextlib.suppress(exceptions.SymbolError): # When requesting the symbol from the module, a full resolve is performed symbol_list.append((vmlinux.get_symbol(sn).address, sn)) - except exceptions.SymbolError: - pass sorted_symbols = sorted(symbol_list) sym_address = 0 diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 2fd7deeaf..cb7626dfa 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -1,18 +1,19 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib import datetime import logging import ntpath from typing import List, Optional, Type -from volatility3.framework import exceptions, renderers, interfaces, constants +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins import timeliner -from volatility3.plugins.windows import pslist, info +from volatility3.plugins.windows import info, pslist vollog = logging.getLogger(__name__) @@ -28,7 +29,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), requirements.ListRequirement(name = 'pid', @@ -107,12 +108,10 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for entry in proc.load_order_modules(): BaseDllName = FullDllName = renderers.UnreadableValue() - try: + with contextlib.suppress(exceptions.InvalidAddressException): BaseDllName = entry.BaseDllName.get_string() # We assume that if the BaseDllName points to an invalid buffer, so will FullDllName FullDllName = entry.FullDllName.get_string() - except exceptions.InvalidAddressException: - pass if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index 9791fa580..e9015280a 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -1,9 +1,10 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +import contextlib import logging from typing import List -from volatility3.framework import renderers, interfaces, objects, exceptions, constants +from volatility3.framework import constants, exceptions, interfaces, objects, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.plugins.windows import pslist @@ -23,7 +24,7 @@ class Envars(interfaces.plugins.PluginInterface): # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -61,13 +62,11 @@ class Envars(interfaces.plugins.PluginInterface): key = hive.get_key('CurrentControlSet\\Control\\Session Manager\\Environment') sys = True except KeyError: - try: + with contextlib.suppress(KeyError): key = hive.get_key('ControlSet001\\Control\\Session Manager\\Environment') sys = True - except KeyError: - pass if sys: - try: + with contextlib.suppress(KeyError): for node in key.get_values(): try: value_node_name = node.get_name() @@ -78,17 +77,13 @@ class Envars(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVV, "Error while parsing global environment variables keys (some keys might be excluded)") continue - except KeyError: - pass ## The user-specific variables - try: + with contextlib.suppress(KeyError): key = hive.get_key('Environment') ntuser = True - except KeyError: - pass if ntuser: - try: + with contextlib.suppress(KeyError): for node in key.get_values(): try: value_node_name = node.get_name() @@ -99,8 +94,6 @@ class Envars(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVV, "Error while parsing user environment variables keys (some keys might be excluded)") continue - except KeyError: - pass ## The volatile user variables try: diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 654e26db7..c96fd9522 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -1,7 +1,7 @@ # This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import datetime import logging @@ -56,7 +56,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): - try: + with contextlib.suppress(exceptions.PagedInvalidAddressException): mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset @@ -131,9 +131,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) - except exceptions.PagedInvalidAddressException: - pass - def generate_timeline(self): for row in self._generator(): _depth, row_data = row diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index a788f058f..30b5db695 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -3,17 +3,18 @@ # import codecs +import contextlib import datetime import json import logging import os -from typing import Any, List, Tuple, Generator +from typing import Any, Generator, List, Tuple -from volatility3.framework import exceptions, renderers, constants, interfaces +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers.physical import BufferDataLayer from volatility3.framework.layers.registry import RegistryHive -from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.plugins.windows.registry import hivelist @@ -38,7 +39,7 @@ class UserAssist(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.IntRequirement(name = 'offset', description = "Hive Offset", default = None, optional = True), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) ] @@ -126,11 +127,9 @@ class UserAssist(interfaces.plugins.PluginInterface): hive_name = hive.hive.cast(kernel.symbol_table_name + constants.BANG + "_CMHIVE").get_name() if self._win7 is None: - try: + with contextlib.suppress(exceptions.SymbolError): self._win7 = self._win7_or_later() - except exceptions.SymbolError: # self._win7 will be None and only registry value rawdata will be output - pass self._determine_userassist_type() @@ -163,7 +162,6 @@ class UserAssist(interfaces.plugins.PluginInterface): # output any subkeys under Count for subkey in countkey.get_subkeys(): - subkey_name = subkey.get_name() result = (1, ( renderers.format_hints.Hex(hive.hive_offset), @@ -185,10 +183,8 @@ class UserAssist(interfaces.plugins.PluginInterface): for value in countkey.get_values(): value_name = value.get_name() - try: + with contextlib.suppress(UnicodeDecodeError): value_name = codecs.encode(value_name, "rot_13") - except UnicodeDecodeError: - pass if self._win7: guid = value_name.split("\\")[0] diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index 996cf03a5..3ce49bbde 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import datetime import ipaddress import socket @@ -27,10 +27,8 @@ def unixtime_to_datetime(unixtime: int) -> Union[interfaces.renderers.BaseAbsent ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = renderers.UnparsableValue() if unixtime > 0: - try: + with contextlib.suppress(ValueError): ret = datetime.datetime.utcfromtimestamp(unixtime) - except ValueError: - pass return ret diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 94045d2e7..a66bfb534 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -1,19 +1,18 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib +import logging from typing import Generator, Iterable, Optional, Set, Tuple -import logging - -from volatility3.framework import constants, objects, renderers -from volatility3.framework import exceptions, interfaces +from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic vollog = logging.getLogger(__name__) + class proc(generic.GenericIntelProcess): def get_task(self): @@ -29,10 +28,8 @@ class proc(generic.GenericIntelProcess): if not isinstance(parent_layer, interfaces.layers.TranslationLayerInterface): raise TypeError("Parent layer is not a translation layer, unable to construct process layer") - try: + with contextlib.suppress(exceptions.InvalidAddressException): dtb = self.get_task().map.pmap.pm_cr3 - except exceptions.InvalidAddressException: - return None if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.p_pid}" @@ -41,10 +38,8 @@ class proc(generic.GenericIntelProcess): return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - try: + with contextlib.suppress(exceptions.InvalidAddressException): task = self.get_task() - except exceptions.InvalidAddressException: - return try: current_map = task.map.hdr.links.next @@ -55,9 +50,9 @@ class proc(generic.GenericIntelProcess): for i in range(task.map.hdr.nentries): if (not current_map or - current_map.vol.offset in seen or - not self._context.layers[task.vol.native_layer_name].is_valid(current_map.dereference().vol.offset, current_map.dereference().vol.size)): - + current_map.vol.offset in seen or + not self._context.layers[task.vol.native_layer_name].is_valid(current_map.dereference().vol.offset, + current_map.dereference().vol.size)): vollog.log(constants.LOGLEVEL_VVV, "Breaking process maps iteration due to invalid state.") break @@ -102,10 +97,8 @@ class fileglob(objects.StructType): if self.has_member("fg_type"): ret = self.fg_type elif self.fg_ops != 0: - try: + with contextlib.suppress(exceptions.InvalidAddressException): ret = self.fg_ops.fo_type - except exceptions.InvalidAddressException: - pass if ret: ret = str(ret.description).replace("DTYPE_", "") @@ -456,7 +449,7 @@ class queue_entry(objects.StructType): seen = set() for attr in ['next', 'prev']: - try: + with contextlib.suppress(exceptions.InvalidAddressException): n = getattr(self, attr).dereference().cast(type_name) while n is not None and n.vol.offset != list_head: @@ -473,9 +466,6 @@ class queue_entry(objects.StructType): n = getattr(n.member(attr = member_name), attr).dereference().cast(type_name) - except exceptions.InvalidAddressException: - pass - class ifnet(objects.StructType): diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index 899b89dc2..cfac87e2c 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -1,10 +1,11 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import extensions -from volatility3.framework.symbols.windows.extensions import registry, pool, pe +from volatility3.framework.symbols.windows.extensions import pe, pool, registry class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -39,26 +40,23 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('_VACB', extensions.VACB) self.set_type_class('_POOL_TRACKER_BIG_PAGES', pool.POOL_TRACKER_BIG_PAGES) self.set_type_class('_IMAGE_DOS_HEADER', pe.IMAGE_DOS_HEADER) - + # Might not necessarily defined in every version of windows self.optional_set_type_class('_IMAGE_NT_HEADERS', pe.IMAGE_NT_HEADERS) self.optional_set_type_class('_IMAGE_NT_HEADERS64', pe.IMAGE_NT_HEADERS) # This doesn't exist in very specific versions of windows - try: + with contextlib.suppress(ValueError): if self.get_type("_POOL_TRACKER_BIG_PAGES").has_member("PoolType"): self.set_type_class('_POOL_HEADER', pool.POOL_HEADER_VISTA) else: self.set_type_class('_POOL_HEADER', pool.POOL_HEADER) - except ValueError: - pass # these don't exist in windows XP self.optional_set_type_class('_MMADDRESS_NODE', extensions.MMVAD_SHORT) - + # these were introduced starting in windows 8 self.optional_set_type_class('_MM_AVL_NODE', extensions.MMVAD_SHORT) - + # these were introduced starting in windows 7 self.optional_set_type_class('_RTL_BALANCED_NODE', extensions.MMVAD_SHORT) - \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b5ee272a0..7be9c4791 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -3,6 +3,7 @@ # import collections.abc +import contextlib import datetime import functools import logging @@ -305,7 +306,7 @@ class MMVAD(MMVAD_SHORT): file_name = renderers.NotApplicableValue() - try: + with contextlib.suppress(exceptions.InvalidAddressException): # this is for xp and 2003 if self.has_member("ControlArea"): filename_obj = self.ControlArea.FilePointer.FileName @@ -318,9 +319,6 @@ class MMVAD(MMVAD_SHORT): if filename_obj.Length > 0: file_name = filename_obj.get_string() - except exceptions.InvalidAddressException: - pass - return file_name @@ -364,6 +362,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): yield device device = device.AttachedDevice.dereference() + class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" @@ -374,7 +373,7 @@ class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" - device = self.DeviceObject.dereference() + device = self.DeviceObject.dereference() while device: yield device device = device.NextDevice.dereference() @@ -413,15 +412,11 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. if self._context.layers[self.vol.native_layer_name].is_valid(self.DeviceObject): - try: + with contextlib.suppress(ValueError): name = f"\\Device\\{self.DeviceObject.get_device_name()}" - except ValueError: - pass - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): name += self.FileName.String - except (TypeError, exceptions.InvalidAddressException): - pass return name @@ -1114,12 +1109,10 @@ class SHARED_CACHE_MAP(objects.StructType): iterval = 0 while (iterval < full_blocks) and (full_blocks <= 4): vacb_obj = self.InitialVacbs[iterval] - try: + with contextlib.suppress(exceptions.InvalidAddressException): # Make sure that the SharedCacheMap member of the VACB points back to the parent object. if vacb_obj.SharedCacheMap == self.vol.offset: self.save_vacb(vacb_obj, vacb_list) - except exceptions.InvalidAddressException: - pass iterval += 1 # We also have to account for the spill over data that is not found in the full blocks. diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index 368765497..79ea60027 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -1,12 +1,14 @@ +import contextlib import functools import logging import struct -from typing import Optional, Tuple, List, Dict, Union +from typing import Dict, List, Optional, Tuple, Union -from volatility3.framework import objects, interfaces, constants, symbols, exceptions, renderers -from volatility3.framework.renderers import conversion from volatility3.plugins.windows.poolscanner import PoolConstraint +from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols +from volatility3.framework.renderers import conversion + vollog = logging.getLogger(__name__) @@ -138,7 +140,7 @@ class POOL_HEADER(objects.StructType): if addr - optional_headers_length >= padding_length > addr: continue - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): mem_object = self._context.object(symbol_table_name + constants.BANG + type_name, layer_name = self.vol.layer_name, offset = addr + body_offset + start_offset, @@ -147,15 +149,13 @@ class POOL_HEADER(objects.StructType): if mem_object.is_valid(): yield mem_object - except (TypeError, exceptions.InvalidAddressException): - pass - # use the bottom up approach for windows 7 and earlier else: type_size = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + type_name).size if constraint.additional_structures: for additional_structure in constraint.additional_structures: - type_size += self._context.symbol_space.get_type(symbol_table_name + constants.BANG + additional_structure).size + type_size += self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + additional_structure).size rounded_size = conversion.round(type_size, alignment, up = True) @@ -164,11 +164,9 @@ class POOL_HEADER(objects.StructType): offset = self.vol.offset + self.BlockSize * alignment - rounded_size, native_layer_name = native_layer_name) - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): if mem_object.is_valid(): yield mem_object - except (TypeError, exceptions.InvalidAddressException): - pass @classmethod @functools.lru_cache() @@ -177,20 +175,18 @@ class POOL_HEADER(objects.StructType): headers = [] sizes = [] for header in [ - 'CREATOR_INFO', 'NAME_INFO', 'HANDLE_INFO', 'QUOTA_INFO', 'PROCESS_INFO', 'AUDIT_INFO', 'EXTENDED_INFO', - 'HANDLE_REVOCATION_INFO', 'PADDING_INFO' + 'CREATOR_INFO', 'NAME_INFO', 'HANDLE_INFO', 'QUOTA_INFO', 'PROCESS_INFO', 'AUDIT_INFO', 'EXTENDED_INFO', + 'HANDLE_REVOCATION_INFO', 'PADDING_INFO' ]: - try: + with contextlib.suppress(AttributeError, exceptions.SymbolError): type_name = f"{symbol_table_name}{constants.BANG}_OBJECT_HEADER_{header}" header_type = context.symbol_space.get_type(type_name) headers.append(header) sizes.append(header_type.size) - except (AttributeError, exceptions.SymbolError): # Some of these may not exist, for example: # if build < 9200: PADDING_INFO else: AUDIT_INFO # if build == 10586: HANDLE_REVOCATION_INFO else EXTENDED_INFO # based on what's present and what's not, this list should be the right order and the right length - pass return headers, sizes def is_free_pool(self): diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 47ff24506..c71fcf49b 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import enum import logging import struct @@ -75,12 +75,10 @@ class CMHIVE(objects.StructType): """ for attr in ["FileFullPath", "FileUserName", "HiveRootPath"]: - try: + with contextlib.suppress(AttributeError, exceptions.InvalidAddressException): name = getattr(self, attr) if name.Length > 0: return name.get_string() - except (AttributeError, exceptions.InvalidAddressException): - pass return None From 3679134f01abcd901f430be1292d99de21c093fc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Jul 2022 22:45:11 +0100 Subject: [PATCH 093/181] Core: Prevent circular dependency on imports --- volatility3/framework/interfaces/objects.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 3cc23e759..d1f442d69 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -243,11 +243,10 @@ class ObjectInterface(metaclass = abc.ABCMeta): @classmethod @abc.abstractmethod - def child_template(cls, template: 'Template', child: str) -> interfaces.objects.Template: + def child_template(cls, template: 'Template', child: str) -> 'interfaces.objects.Template': """Returns the template of the child member from the parent.""" raise KeyError(f"Template does not contain any children: {template.vol.type_name}") - @classmethod @abc.abstractmethod def has_member(cls, template: 'Template', member_name: str) -> bool: From b1b4d21bbcfd0095a10d363e99d8b4bcbfb1a015 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Jul 2022 22:46:31 +0100 Subject: [PATCH 094/181] Core: Prevent circular dependency on imports - take 2 --- volatility3/framework/interfaces/objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index d1f442d69..fcf3c8d6c 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -312,7 +312,7 @@ class Template: offset.""" @abc.abstractmethod - def child_template(self, child: str) -> interfaces.objects.Template: + def child_template(self, child: str) -> 'interfaces.objects.Template': """Returns the `child` member template from its parent.""" @abc.abstractmethod From ec78fe7d8dd015c8ebfc366a5937cebe2bf92b3e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 4 Jul 2022 15:49:03 +0900 Subject: [PATCH 095/181] Fix: try/except/pass to contextlib.supress by #782 --- volatility3/plugins/windows/registry/certificates.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index e873fd1d6..429db96a6 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,3 +1,4 @@ +import contextlib import logging import struct from typing import List, Iterator, Optional, Tuple, Type @@ -67,7 +68,7 @@ class Certificates(interfaces.plugins.PluginInterface): "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - try: + with contextlib.suppress(KeyError, exceptions.SwappedInvalidAddressException): # Walk it node_path = hive.get_key(top_key, return_list = True) for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): @@ -83,13 +84,6 @@ class Certificates(interfaces.plugins.PluginInterface): file_handle.close() yield (0, (top_key, reg_section, key_hash, name)) - except KeyError: - # Key wasn't found in this hive, carry on - vollog.log(constants.LOGLEVEL_VVVV, "Key wasn't found in this hive") - pass - except exceptions.SwappedInvalidAddressException as exp: - vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)") - pass def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), From 2f25312a5c96376b58817772bde1371b424d3f49 Mon Sep 17 00:00:00 2001 From: Malware Utkonos Date: Mon, 4 Jul 2022 13:08:30 -0400 Subject: [PATCH 096/181] Refactor try to reduce size of clause to only what is needed. Based on feedback, memory_object.get_available_pages() might raise this type of exception, so it's still inside the try clause. --- .../framework/plugins/windows/dumpfiles.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 58166ee7f..2c3f8c2d5 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -63,29 +63,28 @@ class DumpFiles(interfaces.plugins.PluginInterface): :return: result status """ filedata = open_method(desired_file_name) - try: - # Description of these variables: - # memoffset: offset in the specified layer where the page begins - # fileoffset: write to this offset in the destination file - # datasize: size of the page + # Description of these variables: + # memoffset: offset in the specified layer where the page begins + # fileoffset: write to this offset in the destination file + # datasize: size of the page - # track number of bytes written so we don't write empty files to disk - bytes_written = 0 + # track number of bytes written so we don't write empty files to disk + bytes_written = 0 + try: for memoffset, fileoffset, datasize in memory_object.get_available_pages(): data = layer.read(memoffset, datasize, pad = True) bytes_written += len(data) filedata.seek(fileoffset) filedata.write(data) - - if not bytes_written: - vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") - return None - else: - vollog.debug(f"Stored {filedata.preferred_filename}") - return filedata except exceptions.InvalidAddressException: vollog.debug(f"Unable to dump file at {file_object.vol.offset:#x}") return None + if not bytes_written: + vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") + return None + vollog.debug(f"Stored {filedata.preferred_filename}") + + return filedata @classmethod def process_file_object(cls, context: interfaces.context.ContextInterface, primary_layer_name: str, From 772ae98eb1966b4b0fa7451673352c6f8f97095c Mon Sep 17 00:00:00 2001 From: Malware Utkonos Date: Mon, 4 Jul 2022 13:27:17 -0400 Subject: [PATCH 097/181] Style changes including yapf according to .style.yapf in package root --- .../framework/plugins/windows/dumpfiles.py | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 58166ee7f..26b637fc3 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -5,6 +5,7 @@ import logging import ntpath from typing import List, Tuple, Type, Optional, Generator + from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -32,8 +33,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.IntRequirement(name = 'pid', description = "Process ID to include (all other processes are excluded)", optional = True), @@ -98,12 +100,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): :param open_method: class for constructing output files :param file_obj: the FILE_OBJECT """ - # Filtering by these types of devices prevents us from processing other types of devices that # use the "File" object type, such as \Device\Tcp and \Device\NamedPipe. if file_obj.DeviceObject.DeviceType not in [FILE_DEVICE_DISK, FILE_DEVICE_NETWORK_FILE_SYSTEM]: - vollog.log(constants.LOGLEVEL_VVV, - f"The file object at {file_obj.vol.offset:#x} is not a file on disk") + vollog.log(constants.LOGLEVEL_VVV, f"The file object at {file_obj.vol.offset:#x} is not a file on disk") return # Depending on the type of object (DataSection, ImageSection, SharedCacheMap) we may need to @@ -120,7 +120,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): # layer to read from, # file extension to apply, # ) - dump_parameters = [] + dump_parameters = list() # The DataSectionObject and ImageSectionObject caches are handled in basically the same way. # We carve these "pages" from the memory_layer. @@ -131,8 +131,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if control_area.is_valid(): dump_parameters.append((control_area, memory_layer, extension)) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") # The SharedCacheMap is handled differently than the caches above. # We carve these "pages" from the primary_layer. @@ -142,8 +141,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if shared_cache_map.is_valid(): dump_parameters.append((shared_cache_map, primary_layer, "vacb")) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] @@ -151,7 +149,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): memory_object.vol.offset, cache_name, ntpath.basename(obj_name), extension) - file_handle = DumpFiles.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) + file_handle = cls.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) file_output = "Error dumping file" if file_handle: @@ -185,8 +183,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): try: object_table = proc.ObjectTable except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") continue for entry in handles_plugin.handles(object_table): @@ -218,12 +215,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue - for result in self.process_file_object(self.context, kernel.layer_name, self.open, - file_obj): + for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot extract file from VAD at {vad.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"Cannot extract file from VAD at {vad.vol.offset:#x}") elif offsets: # Now process any offsets explicitly requested by the user. @@ -234,10 +229,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not is_virtual: layer_name = self.context.layers[layer_name].config["memory_layer"] - file_obj = self.context.object( - kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", + file_obj = self.context.object(kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", layer_name = layer_name, - native_layer_name = kernel.layer_name, + native_layer_name = kernel.layer_name, offset = offset) for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) @@ -246,9 +240,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): def run(self): # a list of tuples (, ) where is the address and is True for virtual. - offsets = [] + offsets = list() # a list of processes matching the pid filter. all files for these process(es) will be dumped. - procs = [] + procs = list() kernel = self.context.modules[self.config['kernel']] if self.config.get("virtaddr", None) is not None: From b30cb5d96842178085967f9462487e1b08b3ec19 Mon Sep 17 00:00:00 2001 From: Malware Utkonos Date: Mon, 4 Jul 2022 13:47:23 -0400 Subject: [PATCH 098/181] Move debug logging based on feedback. --- volatility3/framework/plugins/windows/dumpfiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 2c3f8c2d5..7e1480cbe 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -82,8 +82,8 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not bytes_written: vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") return None - vollog.debug(f"Stored {filedata.preferred_filename}") + vollog.debug(f"Stored {filedata.preferred_filename}") return filedata @classmethod From 84d26ba4bdf46b0280b343b3bd3c3b7ee54238c8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 5 Jul 2022 11:08:41 +0100 Subject: [PATCH 099/181] Core: Add in API_CHANGES updates --- API_CHANGES.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/API_CHANGES.md b/API_CHANGES.md index 274d1d8bb..4d8733286 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,14 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.3.0 +===== +Add in `child_template` to template class + +2.2.0 +===== +Changes to linux core calls + 2.1.0 ===== Add in the linux `task.get_threads` method to the API. From df277b9e802899368186aa04c4d106ba06de1e9b Mon Sep 17 00:00:00 2001 From: Frank Gomulka Date: Wed, 13 Jul 2022 13:49:30 -0500 Subject: [PATCH 100/181] Add testing framework --- .github/workflows/test.yaml | 54 +++++ test/README.md | 34 +++ test/conftest.py | 40 ++++ test/known_files.json | 19 ++ test/requirements-testing.txt | 8 + test/test_volatility.py | 381 ++++++++++++++++++++++++++++++++++ 6 files changed, 536 insertions(+) create mode 100644 .github/workflows/test.yaml create mode 100644 test/README.md create mode 100644 test/conftest.py create mode 100644 test/known_files.json create mode 100644 test/requirements-testing.txt create mode 100644 test/test_volatility.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..5a3f90565 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,54 @@ +name: Test Volatility3 +on: [push] +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install Cmake + pip install setuptools wheel + pip install -U pytest + pip install -r ./test/requirements-testing.txt + + - name: Build PyPi packages + run: | + python setup.py sdist --formats=gztar,zip + python setup.py bdist_wheel + + - name: Download images + run: | + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz" + gunzip linux-sample-1.bin.gz + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" + gunzip win-xp-laptop-2005-06-25.img.gz + + - name: Download and Extract symbols + run: | + cd ./volatility3/symbols + curl -sLO https://downloads.volatilityfoundation.org/volatility3/symbols/linux.zip + unzip linux.zip + cd - + + - name: Testing... + run: | + py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v + py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + + - name: Clean up post-test + run: | + rm -rf *.lime + rm -rf *.img + cd volatility3/symbols + rm -rf linux + rm -rf linux.zip + cd - diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..dcbe289b0 --- /dev/null +++ b/test/README.md @@ -0,0 +1,34 @@ +# Volatility 3 Testing Framework + +## Requirements + +The Volatility 3 Testing Framework requires the same version of Python as Volatility3 itself. To install the current set of dependencies that the framework requires, use a command like this: + +```shell +pip3 install -r requirements-testing.txt +``` + +NOTE: `requirements-testing.txt` can be found in this current `test/` directory. + +## Quick Start: Manual Testing + +1. To test Volatility 3 on an image, first download one with a command such as: + +```shell +curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" +gunzip win-xp-laptop-2005-06-25.img.gz +``` + +2. In many cases, more symbols are required to be downloaded to the `./volatility3/symbols` directory. + +3. To manually run the tests, run a command, such as: + +```shell +py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows +``` + +The above command runs all available tests for windows on the `win-xp-laptop-2005-06-25.img` image. To choose a more specific set of tests, change the phrase after `-k` in this command. + +## Github Actions + +This framework currently tests two images (one linux image and one windows image) after every push on any branch. For more information/context, find the actions setup in `./github/workflows/test.yaml` \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 000000000..9d3d27fc5 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,40 @@ +# This file is used to augment the test configuration + +import os +import pytest + +def pytest_addoption(parser): + parser.addoption("--volatility", action="store", default=None, + required=True, + help="path to the volatility script") + + parser.addoption("--python", action="store", default="python3", + help="The name of the interpreter to use when running the volatility script") + + parser.addoption("--image", action="append", default=[], + help="path to an image to test") + + parser.addoption("--image-dir", action="append", default=[], + help="path to a directory containing images to test") + +def pytest_generate_tests(metafunc): + """Parameterize tests based on image names""" + + images = metafunc.config.getoption('image') + for d in metafunc.config.getoption('image_dir'): + images = images + [os.path.join(d, x) for x in os.listdir(d)] + + # tests with "image" parameter are run against images + if 'image' in metafunc.fixturenames: + metafunc.parametrize("image", + images, + ids=[os.path.basename(image) for image in images]) + +# Fixtures +@pytest.fixture +def volatility(request): + return request.config.getoption("--volatility") + +@pytest.fixture +def python(request): + return request.config.getoption("--python") diff --git a/test/known_files.json b/test/known_files.json new file mode 100644 index 000000000..fbc40e48b --- /dev/null +++ b/test/known_files.json @@ -0,0 +1,19 @@ +{ + "windows_dumpfiles": { + "win-xp-laptop-2005-06-25.img": { + "0x82220e78": [ + "9bdd5532286f1660f3778e68bc36efe6", + "e3bc1e9e7370e3b5a661ebe591ecf4ec" + ], + "0x82350bf8": [ + "e5c5e8d97b6280745b41f6572c85d1f0", + "8589f1463422884dbf1411aaad278465" + ], + "0x81eaf418": [ + "f7a1ae2060a58f8470b97affdb46dccf", + "54fd611021fa784912530b8007545986" + ], + "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" + } + } + } \ No newline at end of file diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt new file mode 100644 index 000000000..d37dc93c3 --- /dev/null +++ b/test/requirements-testing.txt @@ -0,0 +1,8 @@ +# These packages are required for core functionality. +pefile>=2017.8.1 #foo + +# The following packages are optional. +# If certain packages are not necessary, place a comment (#) at the start of the line. + +# This is required for the yara plugins +yara-python>=3.8.0 diff --git a/test/test_volatility.py b/test/test_volatility.py new file mode 100644 index 000000000..527d86dd3 --- /dev/null +++ b/test/test_volatility.py @@ -0,0 +1,381 @@ +# volatility3 tests +# + +# +# IMPORTS +# + +import os +import subprocess +import sys +import shutil +import tempfile +import hashlib +import ntpath +import json + +import pytest + +# +# HELPER FUNCTIONS +# + +def runvol(args, volatility, python): + volpy = volatility + python_cmd = python + + cmd = [python_cmd, volpy] + args + print(" ".join(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + print("stdout:") + sys.stdout.write(str(stdout)) + print("") + print("stderr:") + sys.stdout.write(str(stderr)) + print("") + + return p.returncode, stdout, stderr + +def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): + args = globalargs + [ + "--single-location", + img, + "-q", + plugin, + ] + pluginargs + + return runvol(args, volatility, python) + +# +# TESTS +# + +# WINDOWS + +def test_windows_pslist(image, volatility, python): + rc, out, err = runvol_plugin("windows.pslist.PsList", image, volatility, python) + out = out.lower() + assert out.find(b"system") != -1 + assert out.find(b"csrss.exe") != -1 + assert out.find(b"svchost.exe") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + + rc, out, err = runvol_plugin( + "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"]) + out = out.lower() + assert out.find(b"system") != -1 + assert out.count(b"\n") < 10 + assert rc == 0 + assert rc == 0 + +def test_windows_psscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) + out = out.lower() + assert out.find(b"system") != -1 + assert out.find(b"csrss.exe") != -1 + assert out.find(b"svchost.exe") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_dlllist(image, volatility, python): + rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) + out = out.lower() + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_modules(image, volatility, python): + rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) + out = out.lower() + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_hivelist(image, volatility, python): + rc, out, err = runvol_plugin("windows.registry.hivelist.HiveList", image, volatility, python) + out = out.lower() + + not_xp = out.find(b"\\systemroot\\system32\\config\\software") + if not_xp == -1: + assert out.find(b"\\device\\harddiskvolume1\\windows\\system32\\config\\software") != -1 + + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_windows_dumpfiles(image, volatility, python): + + json_file = open('./test/known_files.json') + + known_files = json.load(json_file) + + failed_chksms = 0 + + if sys.platform == 'win32': + file_name = ntpath.basename(image) + else: + file_name = os.path.basename(image) + + try: + for addr in known_files["windows_dumpfiles"][file_name]: + + path = tempfile.mkdtemp() + + rc, out, err = runvol_plugin("windows.dumpfiles.DumpFiles", image, volatility, python, globalargs=["-o", path], pluginargs=["--virtaddr", addr]) + + for file in os.listdir(path): + fp = open(os.path.join(path, file), "rb") + if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: + failed_chksms += 1 + fp.close() + + shutil.rmtree(path) + + json_file.close() + + assert failed_chksms == 0 + assert rc == 0 + except Exception as e: + json_file.close() + print("Key Error raised on " + str(e)) + assert False + +def test_windows_handles(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"System Pid 4") != -1 + assert out.find(b"MACHINE\\SYSTEM\\CONTROLSET001\\CONTROL\\SESSION MANAGER\\MEMORY MANAGEMENT\\PREFETCHPARAMETERS") != -1 + assert out.find(b"MACHINE\\SYSTEM\\SETUP") != -1 + assert out.count(b"\n") > 500 + assert rc == 0 + +def test_windows_svcscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) + + assert out.find(b"Microsoft ACPI Driver") != -1 + assert out.count(b"\n") > 250 + assert rc == 0 + +def test_windows_privileges(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"SeCreateTokenPrivilege") != -1 + assert out.find(b"SeCreateGlobalPrivilege") != -1 + assert out.find(b"SeAssignPrimaryTokenPrivilege") != -1 + assert out.count(b"\n") > 20 + assert rc == 0 + +def test_windows_getsids(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"Local System") != -1 + assert out.find(b"Administrators") != -1 + assert out.find(b"Everyone") != -1 + assert out.find(b"Authenticated Users") != -1 + assert rc == 0 + +def test_windows_envars(image, volatility, python): + rc, out, err = runvol_plugin("windows.envars.Envars", image, volatility, python) + + assert out.find(b"PATH") != -1 + assert out.find(b"PROCESSOR_ARCHITECTURE") != -1 + assert out.find(b"USERNAME") != -1 + assert out.find(b"SystemRoot") != -1 + assert out.find(b"CommonProgramFiles") != -1 + assert out.count(b"\n") > 500 + assert rc == 0 + +def test_windows_callbacks(image, volatility, python): + rc, out, err = runvol_plugin("windows.callbacks.Callbacks", image, volatility, python) + + assert out.find(b"PspCreateProcessNotifyRoutine") != -1 + assert out.find(b"KeBugCheckCallbackListHead") != -1 + assert out.find(b"KeBugCheckReasonCallbackListHead") != -1 + assert out.count(b"KeBugCheckReasonCallbackListHead ") > 5 + assert rc == 0 + +# LINUX + +def test_linux_pslist(image, volatility, python): + rc, out, err = runvol_plugin("linux.pslist.PsList", image, volatility, python) + out = out.lower() + + assert ((out.find(b"init") != -1) or (out.find(b"systemd") != -1)) + assert out.find(b"watchdog") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_check_idt(image, volatility, python): + rc, out, err = runvol_plugin("linux.check_idt.Check_idt", image, volatility, python) + out = out.lower() + + assert out.count(b"__kernel__") >= 10 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_check_syscall(image, volatility, python): + rc, out, err = runvol_plugin("linux.check_syscall.Check_syscall", image, volatility, python) + out = out.lower() + + assert out.find(b"sys_close") != -1 + assert out.find(b"sys_open") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_linux_lsmod(image, volatility, python): + rc, out, err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_lsof(image, volatility, python): + rc, out, err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) + out = out.lower() + + assert out.count(b"socket:") >= 10 + assert out.count(b"\n") > 35 + assert rc == 0 + +def test_linux_proc_maps(image, volatility, python): + rc, out, err = runvol_plugin("linux.proc.Maps", image, volatility, python) + out = out.lower() + + assert out.count(b"anonymous mapping") >= 10 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_linux_tty_check(image, volatility, python): + rc, out, err = runvol_plugin("linux.tty_check.tty_check", image, volatility, python) + out = out.lower() + + assert out.find(b"__kernel__") != -1 + assert out.count(b"\n") >= 5 + assert rc == 0 + +# MAC + +def test_mac_pslist(image, volatility, python): + rc, out, err = runvol_plugin("mac.pslist.PsList", image, volatility, python) + out = out.lower() + + assert ((out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1)) + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_check_syscall(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_syscall.Check_syscall", image, volatility, python) + out = out.lower() + + assert out.find(b"chmod") != -1 + assert out.find(b"chown") != -1 + assert out.find(b"nosys") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_mac_check_sysctl(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_sysctl.Check_sysctl", image, volatility, python) + out = out.lower() + + assert out.find(b"__kernel__") != -1 + assert out.count(b"\n") > 250 + assert rc == 0 + +def test_mac_check_trap_table(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_trap_table.Check_trap_table", image, volatility, python) + out = out.lower() + + assert out.count(b"kern_invalid") >= 10 + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_ifconfig(image, volatility, python): + rc, out, err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) + out = out.lower() + + assert out.find(b"127.0.0.1") != -1 + assert out.find(b"false") != -1 + assert out.count(b"\n") > 9 + assert rc == 0 + +def test_mac_lsmod(image, volatility, python): + rc, out, err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) + out = out.lower() + + assert out.find(b"com.apple") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_lsof(image, volatility, python): + rc, out, err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_malfind(image, volatility, python): + rc, out, err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 20 + assert rc == 0 + +def test_mac_mount(image, volatility, python): + rc, out, err = runvol_plugin("mac.mount.Mount", image, volatility, python) + out = out.lower() + + assert out.find(b"/dev") != -1 + assert out.count(b"\n") > 7 + assert rc == 0 + +def test_mac_netstat(image, volatility, python): + rc, out, err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) + + assert out.find(b"TCP") != -1 + assert out.find(b"UDP") != -1 + assert out.find(b"UNIX") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_proc_maps(image, volatility, python): + rc, out, err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) + out = out.lower() + + assert out.find(b"[heap]") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_mac_psaux(image, volatility, python): + rc, out, err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) + out = out.lower() + + assert out.find(b"executable_path") != -1 + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_socket_filters(image, volatility, python): + rc, out, err = runvol_plugin("mac.socket_filters.Socket_filters", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 9 + assert rc == 0 + +def test_mac_timers(image, volatility, python): + rc, out, err = runvol_plugin("mac.timers.Timers", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 6 + assert rc == 0 + +def test_mac_trustedbsd(image, volatility, python): + rc, out, err = runvol_plugin("mac.trustedbsd.Trustedbsd", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 From f96f004b13c15b3c0334f7ca96b2c0459a8f9cf1 Mon Sep 17 00:00:00 2001 From: Frank Gomulka Date: Fri, 15 Jul 2022 18:01:48 -0500 Subject: [PATCH 101/181] @digitalisx suggested changes --- .github/workflows/test.yaml | 6 +++--- test/test_volatility.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5a3f90565..a3ecd7c7e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,5 +1,5 @@ name: Test Volatility3 -on: [push] +on: [push, pull_request] jobs: build: @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.x + - name: Set up Python 3.6 uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.6' - name: Install dependencies run: | diff --git a/test/test_volatility.py b/test/test_volatility.py index 527d86dd3..a55dffb27 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -127,10 +127,9 @@ def test_windows_dumpfiles(image, volatility, python): rc, out, err = runvol_plugin("windows.dumpfiles.DumpFiles", image, volatility, python, globalargs=["-o", path], pluginargs=["--virtaddr", addr]) for file in os.listdir(path): - fp = open(os.path.join(path, file), "rb") - if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: - failed_chksms += 1 - fp.close() + with open(os.path.join(path, file), "rb") as fp: + if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: + failed_chksms += 1 shutil.rmtree(path) From f295e5d91a6b7ecc7a1f03099d2984a98cf43eda Mon Sep 17 00:00:00 2001 From: fgomulka <60993471+fgomulka@users.noreply.github.com> Date: Sat, 16 Jul 2022 16:34:47 -0500 Subject: [PATCH 102/181] Add newline Co-authored-by: Donghyun Kim --- test/known_files.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/known_files.json b/test/known_files.json index fbc40e48b..089896714 100644 --- a/test/known_files.json +++ b/test/known_files.json @@ -16,4 +16,5 @@ "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" } } - } \ No newline at end of file + } + \ No newline at end of file From 3e748e7d488eeb96904d94039ca02d57c6368fff Mon Sep 17 00:00:00 2001 From: fgomulka <60993471+fgomulka@users.noreply.github.com> Date: Sat, 16 Jul 2022 16:35:26 -0500 Subject: [PATCH 103/181] Use more descriptive variable names Co-authored-by: Donghyun Kim --- test/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 9d3d27fc5..9057e1676 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -21,8 +21,8 @@ def pytest_generate_tests(metafunc): """Parameterize tests based on image names""" images = metafunc.config.getoption('image') - for d in metafunc.config.getoption('image_dir'): - images = images + [os.path.join(d, x) for x in os.listdir(d)] + for image_dir in metafunc.config.getoption('image_dir'): + images = images + [os.path.join(image_dir, dir) for dir in os.listdir(image_dir)] # tests with "image" parameter are run against images if 'image' in metafunc.fixturenames: From 5bc517aa42f09bb467136866d92811760a92169b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 00:15:57 +0000 Subject: [PATCH 104/181] Automagic: Use sqlite to cache identifiers --- volatility3/framework/automagic/linux.py | 35 +- volatility3/framework/automagic/mac.py | 28 +- .../framework/automagic/symbol_cache.py | 480 ++++++++++++------ .../framework/automagic/symbol_finder.py | 25 +- .../framework/configuration/requirements.py | 12 +- volatility3/framework/constants/__init__.py | 7 +- volatility3/framework/interfaces/automagic.py | 9 +- volatility3/framework/plugins/isfinfo.py | 39 +- volatility3/framework/symbols/intermed.py | 3 +- .../framework/symbols/windows/pdbutil.py | 58 +-- 10 files changed, 417 insertions(+), 279 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f1d6c91e4..2c152996d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -5,8 +5,9 @@ import logging from typing import Optional, Tuple, Type -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder +from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import linux @@ -23,6 +24,13 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify linux within this layer.""" + # Version check the SQlite cache + required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + vollog.info( + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + return None + # Bail out by default unless we can stack properly layer = context.layers[layer_name] join = interfaces.configuration.path_join @@ -32,7 +40,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - linux_banners = LinuxBannerCache.load_banners() + linux_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + operating_system = 'linux') # If we have no banners, don't bother scanning if not linux_banners: vollog.info("No Linux banners found - if this is a linux plugin, please check your symbol files location") @@ -43,15 +52,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = linux_banners.get(banner, None) - if symbol_files: - if len(symbol_files) > 1: - using = "*" - vollog.warning(f"Multiple symbol files identified (using {using}):") - for symbol_file in symbol_files: - vollog.warning(f" {using} {symbol_file}") - using = " " - isf_path = symbol_files[0] + isf_path = linux_banners.get(banner, None) + if isf_path: table_name = context.symbol_space.free_table_name('LintelStacker') table = linux.LinuxKernelIntermedSymbols(context, 'temporary.' + table_name, @@ -147,20 +149,11 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): return addr - 0xc0000000 -class LinuxBannerCache(symbol_cache.SymbolBannerCache): - """Caches the banners found in the Linux symbol files.""" - - os = "linux" - symbol_name = "linux_banner" - banner_path = constants.LINUX_BANNERS_PATH - exclusion_list = ['mac', 'windows'] - - class LinuxSymbolFinder(symbol_finder.SymbolFinder): """Linux symbol loader based on uname signature strings.""" banner_config_key = "kernel_banner" - banner_cache = LinuxBannerCache + operating_system = 'linux' symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] exclusion_list = ['mac', 'windows'] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index c37aef463..246462878 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -6,8 +6,9 @@ import logging import struct from typing import Optional -from volatility3.framework import interfaces, constants, layers, exceptions +from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.automagic import symbol_cache, symbol_finder +from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import mac @@ -24,6 +25,13 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify mac within this layer.""" + # Version check the SQlite cache + required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + vollog.info( + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + return None + # Bail out by default unless we can stack properly layer = context.layers[layer_name] new_layer = None @@ -34,7 +42,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - mac_banners = MacBannerCache.load_banners() + mac_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + operating_system = 'mac') # If we have no banners, don't bother scanning if not mac_banners: vollog.info("No Mac banners found - if this is a mac plugin, please check your symbol files location") @@ -46,9 +55,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = mac_banners.get(banner, None) - if symbol_files: - isf_path = symbol_files[0] + isf_path = mac_banners.get(banner, None) + if isf_path: table_name = context.symbol_space.free_table_name('MacintelStacker') table = mac.MacKernelIntermedSymbols(context = context, config_path = join('temporary', table_name), @@ -197,19 +205,11 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): yield offset, banner -class MacBannerCache(symbol_cache.SymbolBannerCache): - """Caches the banners found in the Mac symbol files.""" - os = "mac" - symbol_name = "version" - banner_path = constants.MAC_BANNERS_PATH - exclusion_list = ['windows', 'linux'] - - class MacSymbolFinder(symbol_finder.SymbolFinder): """Mac symbol loader based on uname signature strings.""" banner_config_key = 'kernel_banner' - banner_cache = MacBannerCache + operating_system = 'mac' find_aslr = MacIntelStacker.find_aslr symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols" exclusion_list = ['windows', 'linux'] diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 7b6adf9b4..fe717b8be 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -2,18 +2,20 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import base64 -import gc import json import logging import os -import pickle +import sqlite3 import urllib import urllib.parse import urllib.request -import zipfile -from typing import Dict, List, Optional +from abc import abstractmethod +from typing import Dict, Generator, List, Optional -from volatility3.framework import constants, exceptions, interfaces +import volatility3.framework +import volatility3.schemas +from volatility3.framework import constants, interfaces +from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources from volatility3.framework.symbols import intermed @@ -22,164 +24,324 @@ vollog = logging.getLogger(__name__) BannersType = Dict[bytes, List[str]] -class SymbolBannerCache(interfaces.automagic.AutomagicInterface): - """Runs through all symbols tables and caches their banners.""" +### Identifiers - # Since this is necessary for ConstructionMagic, we set a lower priority - # The user would run it eventually either way, but running it first means it can be used that run +class IdentifierProcessor: + operating_system = None + + def __init__(self): + pass + + @classmethod + @abstractmethod + def get_identifier(cls, json) -> Optional[bytes]: + """Method to extract the identifier from a particular operating system's JSON + + Returns: + identifier is valid or None if not found + """ + raise NotImplemented("This base class has no get_identifier method defined") + + +class WindowsIdentifier(IdentifierProcessor): + operating_system = 'windows' + separator = '|' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + """Returns the identifier for the file if one can be found""" + windows_metadata = json.get('metadata', {}).get('windows', {}).get('pdb', {}) + if windows_metadata: + guid = windows_metadata.get('GUID', None) + age = windows_metadata.get('age', None) + database = windows_metadata.get('database', None) + if guid and age and database: + return cls.generate(database, guid, age) + return None + + @classmethod + def generate(cls, pdb_name: str, guid: str, age: int) -> bytes: + return bytes(cls.separator.join([pdb_name, guid.upper(), str(age)]), 'latin-1') + + +class MacIdentifier(IdentifierProcessor): + operating_system = 'mac' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + mac_banner = json.get('symbols', {}).get('version', {}).get('constant_data', None) + if mac_banner: + return base64.b64decode(mac_banner) + return None + + +class LinuxIdentifier(IdentifierProcessor): + operating_system = 'linux' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + linux_banner = json.get('symbols', {}).get('linux_banner', {}).get('constant_data', None) + if linux_banner: + return base64.b64decode(linux_banner) + return None + + +### CacheManagers + +class CacheManagerInterface(interfaces.configuration.VersionableInterface): + def __init__(self, filename: str): + super().__init__() + self._filename = filename + self._classifiers = {} + for subclazz in volatility3.framework.class_subclasses(IdentifierProcessor): + self._classifiers[subclazz.operating_system] = subclazz + + def add_identifier(self, location: str, operating_system: str, identifier: str): + """Adds an identifier to the store""" + pass + + def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + """Returns the location of the symbol file given the identifier + + Args: + identifier: string that uniquely identifies a particular symbolt table + operating_system: optional string to restrict identifiers to just those for a particular operating system + + Returns: + The location of the symbols file that matches the identifier + """ + pass + + def get_local_locations(self) -> List[str]: + """Returns a list of all the local locations""" + pass + + def update(self): + """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. + This also updates remote locations based on a cache timeout. + + """ + pass + + def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ + Dict[bytes, str]: + """Returns a dictionary of identifiers and locations + + Args: + operating_system: If set, limits responses to a specific operating system + local_only: Returns only local locations + + Returns: + A dictionary of identifiers mapped to a location + """ + pass + + def get_identifier(self, location: str) -> Optional[bytes]: + """Returns an identifier based on a specific location or None""" + pass + + def get_identifiers(self, operating_system: Optional[str]): + """Returns all identifiers for a particular operating system""" + pass + + +class SqliteCache(CacheManagerInterface): + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + def __init__(self, filename: str): + super().__init__(filename) + try: + self._database = self._connect_storage(filename) + except sqlite3.DatabaseError: + os.unlink(filename) + self._database = self._connect_storage(filename) + + def _connect_storage(self, path: str): + database = sqlite3.connect(path, isolation_level = None) + database.row_factory = sqlite3.Row + database.cursor().execute( + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + return database + + def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + """Returns the location of the symbol file given the identifier. + If multiple locations exist for an identifier, the last found is returned + + Args: + identifier: string that uniquely identifies a particular symbolt table + operating_system: optional string to restrict identifiers to just those for a particular operating system + + Returns: + The location of the symbols file that matches the identifier or None + """ + statement = 'SELECT location FROM cache WHERE identifier = ?' + parameters = (identifier,) + if operating_system is not None: + statement = 'SELECT location FROM cache WHERE identifier = ? AND operating_system = ?' + parameters = (identifier, operating_system) + results = self._database.cursor().execute(statement, parameters).fetchall() + result = None + for row in results: + result = row['location'] + return result + + def get_local_locations(self) -> Generator[str, None, None]: + result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = True').fetchall() + for row in result: + yield row['location'] + + def is_url_local(self, url: str) -> bool: + """Determines whether an url is local or not""" + parsed = urllib.parse.urlparse(url) + if parsed.scheme in ['file', 'jar']: + return True + + def get_identifier(self, location: str) -> Optional[bytes]: + results = self._database.cursor().execute('SELECT identifier FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['identifier'] + return None + + def update(self, progress_callback = None): + """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. + This also updates remote locations based on a cache timeout. + + """ + on_disk_locations = set([filename for filename in intermed.IntermediateSymbolTable.file_symbol_url('')]) + cached_locations = set(self.get_local_locations()) + + new_locations = on_disk_locations.difference(cached_locations) + missing_locations = cached_locations.difference(on_disk_locations) + + cache_update = set() + files_to_timestamp = on_disk_locations.intersection(cached_locations) + if files_to_timestamp: + result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " + "AND cached < date('now', '-3 days');") + for row in result: + if row['location'] in files_to_timestamp: + cache_update.add(row['location']) + + idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + + counter = 0 + files_to_process = new_locations.union(cache_update) + number_files_to_process = len(files_to_process) + for location in files_to_process: + # Open location + counter += 1 + progress_callback(counter * 100 / number_files_to_process, + "Updating caches for {number_files_to_process} files...") + try: + with resources.ResourceAccessor().open(location) as fp: + json_obj = json.load(fp) + identifier = None + for idextractor in idextractors: + identifier = idextractor.get_identifier(json_obj) + operating_system = idextractor.operating_system + if identifier is not None: + break + if identifier is not None: + # We don't try to validate schemas here, we do that on first use + # Store in database + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") + else: + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + None, + None, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") + except Exception as excp: + vollog.log(constants.LOGLEVEL_VVVV, excp) + + if not constants.OFFLINE and constants.REMOTE_ISF_URL: + remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) + for operating_system in ['mac', 'linux', 'windows']: + identifiers = remote_identifiers.process({}, operating_system = operating_system) + for identifier in identifiers: + for location in identifiers[identifier]: + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now')", + (location, identifier, operating_system, False) + ) + + if missing_locations: + self._database.cursor().execute( + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + + def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ + Dict[bytes, str]: + output = {} + additions = [] + statement = 'SELECT location, identifier FROM cache' + if local_only: + additions.append('local = True') + if operating_system: + additions.append(f"operating_system = '{operating_system}'") + if additions: + statement += f" WHERE {' AND '.join(additions)}" + results = self._database.cursor().execute(statement) + for row in results: + if row['identifier'] in output and row['identifier'] and row['location']: + vollog.debug( + f"Duplicate entry for identifier {row['identifier']}: {row['location']} and {output[row['identifier']]}") + output[row['identifier']] = row['location'] + return output + + def get_identifiers(self, operating_system: Optional[str]): + if operating_system: + results = self._database.cursor().execute('SELECT identifier FROM cache WHERE operating_system = ?', + (operating_system,)).fetchall() + else: + results = self._database.cursor().execute('SELECT identifier FROM cache').fetchall() + output = [] + for row in results: + output.append(row['identifier']) + return output + + +### Automagic + +class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): + """Runs through all symbol tables and caches their identifiers""" priority = 0 - os: Optional[str] = None - symbol_name: str = "banner_name" - banner_path: Optional[str] = None - - @classmethod - def load_banners(cls) -> BannersType: - if not cls.banner_path: - raise ValueError("Banner_path not appropriately set") - banners: BannersType = {} - if os.path.exists(cls.banner_path): - with open(cls.banner_path, "rb") as f: - # We use pickle over JSON because we're dealing with bytes objects - banners.update(pickle.load(f)) - - # Remove possibilities that can't exist locally. - remove_banners = [] - for banner in banners: - for path in banners[banner]: - url = urllib.parse.urlparse(path) - if url.scheme == 'file' and not os.path.exists(urllib.request.url2pathname(url.path)): - vollog.log( - constants.LOGLEVEL_VV, "Removing cached path {} for banner {}: file does not exist".format( - path, str(banner or b'', 'latin-1'))) - banners[banner].remove(path) - # This is probably excessive, but it's here if we need it - if url.scheme == 'jar': - zip_file, zip_path = url.path.split("!") - zip_file = urllib.parse.urlparse(zip_file).path - if ((not os.path.exists(zip_file)) or (zip_path not in zipfile.ZipFile(zip_file).namelist())): - vollog.log(constants.LOGLEVEL_VV, - "Removing cached path {} for banner {}: file does not exist".format(path, banner)) - banners[banner].remove(path) - - if not banners[banner]: - remove_banners.append(banner) - for remove_banner in remove_banners: - del banners[remove_banner] - return banners - - @classmethod - def save_banners(cls, banners): - - with open(cls.banner_path, "wb") as f: - pickle.dump(banners, f) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cache = SqliteCache(constants.IDENTIFIERS_PATH) def __call__(self, context, config_path, configurable, progress_callback = None): """Runs the automagic over the configurable.""" - - # Bomb out if we're just the generic interface - if self.os is None: - return - - # We only need to be called once, so no recursion necessary - banners = self.load_banners() - - cacheables = self.find_new_banner_files(banners, self.os) - - new_banners = self.read_new_banners(context, config_path, cacheables, self.symbol_name, self.os, - progress_callback) - - # Add in any new banners to the existing list - for new_banner in new_banners: - banner_list = banners.get(new_banner, []) - banners[new_banner] = list(set(banner_list + new_banners[new_banner])) - - # Do remote banners *after* the JSON loading, so that it doesn't pull down all the remote JSON - self.remote_banners(banners, self.os) - - # Rewrite the cached banners each run, since writing is faster than the banner_cache validation portion - self.save_banners(banners) - - if progress_callback is not None: - progress_callback(100, f"Built {self.os} caches") + self._cache.update(progress_callback) @classmethod - def read_new_banners(cls, context: interfaces.context.ContextInterface, config_path: str, new_urls: List[str], - symbol_name: str, operating_system: str = None, - progress_callback = None) -> Optional[Dict[bytes, List[str]]]: - """Reads the any new banners for the OS in question""" - if operating_system is None: - return None - - banners = {} - - total = len(new_urls) - if total > 0: - vollog.info(f"Building {operating_system} caches...") - for current in range(total): - if progress_callback is not None: - progress_callback(current * 100 / total, f"Building {operating_system} caches") - isf_url = new_urls[current] - - isf = None - try: - # Loading the symbol table will be very slow until it's been validated - isf = intermed.IntermediateSymbolTable(context, config_path, "temp", isf_url, validate = False) - - # We should store the banner against the filename - # We don't bother with the hash (it'll likely take too long to validate) - # but we should check at least that the banner matches on load. - banner = isf.get_symbol(symbol_name).constant_data - vollog.log(constants.LOGLEVEL_VV, f"Caching banner {banner} for file {isf_url}") - - bannerlist = banners.get(banner, []) - bannerlist.append(isf_url) - banners[banner] = bannerlist - except exceptions.SymbolError: - pass - except json.JSONDecodeError: - vollog.log(constants.LOGLEVEL_VV, f"Caching file {isf_url} failed due to JSON error") - finally: - # Get rid of the loaded file, in case it sits in memory - if isf: - del isf - gc.collect() - return banners - - @classmethod - def find_new_banner_files(cls, banners: Dict[bytes, List[str]], operating_system: str) -> List[str]: - """Gathers all files and remove existing banners""" - cacheables = list(intermed.IntermediateSymbolTable.file_symbol_url(operating_system)) - for banner in banners: - for json_file in banners[banner]: - if json_file in cacheables: - cacheables.remove(json_file) - return cacheables - - @classmethod - def remote_banners(cls, banners: Dict[bytes, List[str]], operating_system = None, banner_location = None): - """Adds remote URLs to the banner list""" - if operating_system is None: - return None - - if banner_location is None: - banner_location = constants.REMOTE_ISF_URL - - if not constants.OFFLINE and banner_location is not None: - try: - rbf = RemoteBannerFormat(banner_location) - rbf.process(banners, operating_system) - except urllib.error.URLError: - vollog.debug(f"Unable to download remote banner list from {banner_location}") + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + """Returns a list of RequirementInterface objects required by this + object.""" + return [requirements.VersionRequirement(name = 'SQLiteCache', component = SqliteCache, version = (1, 0, 0))] -class RemoteBannerFormat: +class RemoteIdentifierFormat: def __init__(self, location: str): self._location = location with resources.ResourceAccessor().open(url = location) as fp: self._data = json.load(fp) if not self._verify(): - raise ValueError("Unsupported version for remote banner list format") + raise ValueError("Unsupported version for remote identifier list format") def _verify(self) -> bool: version = self._data.get('version', 0) @@ -188,23 +350,23 @@ class RemoteBannerFormat: return True return False - def process(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): - raise ValueError("Banner List version not verified") + def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]): + raise ValueError("Identifier List version not verified") - def process_v1(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): + def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]): if operating_system in self._data: - for banner in self._data[operating_system]: - binary_banner = base64.b64decode(banner) - file_list = banners.get(binary_banner, []) - for value in self._data[operating_system][banner]: + for identifier in self._data[operating_system]: + binary_identifier = base64.b64decode(identifier) + file_list = identifiers.get(binary_identifier, []) + for value in self._data[operating_system][identifier]: if value not in file_list: file_list = file_list + [value] - banners[binary_banner] = file_list + identifiers[binary_identifier] = file_list if 'additional' in self._data: for location in self._data['additional']: try: - subrbf = RemoteBannerFormat(location) - subrbf.process(banners, operating_system) + subrbf = RemoteIdentifierFormat(location) + subrbf.process(identifiers, operating_system) except IOError: vollog.debug(f"Remote file not found: {location}") - return banners + return identifiers diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 143abd02e..610ed0e18 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -3,9 +3,9 @@ # import logging -from typing import Any, Iterable, List, Tuple, Type, Optional, Callable +from typing import Any, Callable, Iterable, List, Optional, Tuple -from volatility3.framework import interfaces, constants, layers +from volatility3.framework import constants, interfaces, layers from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners @@ -18,7 +18,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): priority = 40 banner_config_key: str = "banner" - banner_cache: Optional[Type[symbol_cache.SymbolBannerCache]] = None + operating_system: Optional[str] = None symbol_class: Optional[str] = None find_aslr: Optional[Callable] = None @@ -27,14 +27,21 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): self._requirements: List[Tuple[str, interfaces.configuration.RequirementInterface]] = [] self._banners: symbol_cache.BannersType = {} + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.VersionRequirement(name = 'SQLiteCache', + component = symbol_cache.SqliteCache, + version = (1, 0, 0)) + ] + @property def banners(self) -> symbol_cache.BannersType: """Creates a cached copy of the results, but only it's been requested.""" if not self._banners: - if not self.banner_cache: - raise RuntimeError(f"Cache has not been properly defined for {self.__class__.__name__}") - self._banners = self.banner_cache.load_banners() + cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system) return self._banners def __call__(self, @@ -103,8 +110,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): vollog.debug(f"Identified banner: {repr(banner)}") symbol_files = self.banners.get(banner, None) if symbol_files: - isf_path = symbol_files[0] - vollog.debug(f"Using symbol library: {symbol_files[0]}") + isf_path = symbol_files + vollog.debug(f"Using symbol library: {symbol_files}") clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join @@ -117,7 +124,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): break else: if symbol_files: - vollog.debug(f"Symbol library path not found: {symbol_files[0]}") + vollog.debug(f"Symbol library path not found: {symbol_files}") # print("Kernel", banner, hex(banner_offset)) else: vollog.debug("No existing banners found") diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 4edc6d17c..b31c4767f 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -408,13 +408,19 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type config_path = interfaces.configuration.path_join(config_path, self.name) - if len(self._version) > 0 and self._component.version[0] != self._version[0]: - return {config_path: self} - if len(self._version) > 1 and self._component.version[1] < self._version[1]: + if not self.matches_required(self._version, self._component.version): return {config_path: self} context.config[interfaces.configuration.path_join(config_path, self.name)] = True return {} + @classmethod + def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]): + if len(required) > 0 and version[0] != required[0]: + return False + if len(required) > 1 and version[1] < required[1]: + return False + return True + class PluginRequirement(VersionRequirement): diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index f08819f29..322e574e1 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -68,10 +68,13 @@ if sys.platform == 'win32': os.makedirs(CACHE_PATH, exist_ok = True) LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") -""""Default location to record information about available linux banners""" +"""Default location to record information about available linux banners""" MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") -""""Default location to record information about available mac banners""" +"""Default location to record information about available mac banners""" + +IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") +"""Default location to record information about available identifiers""" BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index c96c9bdbe..713f91da0 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -9,9 +9,9 @@ that a user has not filled. """ import logging from abc import ABCMeta -from typing import Any, List, Optional, Tuple, Union, Type +from typing import Any, List, Optional, Tuple, Type, Union -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -47,9 +47,10 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla super().__init__(context, config_path) for requirement in self.get_requirements(): if not isinstance(requirement, (interfaces.configuration.SimpleTypeRequirement, - requirements.ChoiceRequirement, requirements.ListRequirement)): + requirements.ChoiceRequirement, requirements.ListRequirement, + requirements.VersionRequirement)): raise TypeError( - "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement or ListRequirement") + "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement, ListRequirement or VersionRequirement") def __call__(self, context: interfaces.context.ContextInterface, diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 575f25426..b2960733d 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -1,17 +1,16 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import base64 import json import logging import os import pathlib import zipfile -from typing import List, Type, Any, Generator +from typing import Generator, List from volatility3 import schemas, symbols -from volatility3.framework import interfaces, renderers, constants -from volatility3.framework.automagic import mac, linux, symbol_cache +from volatility3.framework import constants, interfaces, renderers +from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import resources @@ -23,7 +22,7 @@ class IsfInfo(plugins.PluginInterface): """Determines information about the currently available ISF files, or a specific one""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -39,7 +38,10 @@ class IsfInfo(plugins.PluginInterface): requirements.BooleanRequirement(name = 'validate', description = 'Validate against schema if possible', default = False, - optional = True) + optional = True), + requirements.VersionRequirement(name = 'SQLiteCache', + component = symbol_cache.SqliteCache, + version = (1, 0, 0)) ] @classmethod @@ -62,14 +64,6 @@ class IsfInfo(plugins.PluginInterface): if filename.endswith(extension): yield pathlib.Path(base_name).as_uri() - def _get_banner(self, clazz: Type[symbol_cache.SymbolBannerCache], data: Any) -> str: - """Gets a banner from an ISF file""" - banner_symbol = data.get('symbols', {}).get(clazz.symbol_name, {}).get('constant_data', - renderers.NotAvailableValue()) - if not isinstance(banner_symbol, interfaces.renderers.BaseAbsentValue): - banner_symbol = str(base64.b64decode(banner_symbol), encoding = 'latin-1') - return banner_symbol - def _generator(self): if self.config.get('isf', None) is not None: file_list = [self.config['isf']] @@ -101,7 +95,6 @@ class IsfInfo(plugins.PluginInterface): # Process the filtered list for entry in filtered_list: num_types = num_enums = num_bases = num_symbols = 0 - windows_info = linux_banner = mac_banner = renderers.NotAvailableValue() valid = "Unknown" with resources.ResourceAccessor().open(url = entry) as fp: try: @@ -111,20 +104,20 @@ class IsfInfo(plugins.PluginInterface): num_enums = len(data.get('enums', [])) num_bases = len(data.get('base_types', [])) - linux_banner = self._get_banner(linux.LinuxBannerCache, data) - mac_banner = self._get_banner(mac.MacBannerCache, data) - if not linux_banner and not mac_banner: - windows_info = os.path.splitext(os.path.basename(entry))[0] + identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifier = identifier_cache.get_identifier(location = entry) + if identifier: + identifier = identifier.decode('utf-8', errors = 'replace') + else: + identifier = renderers.NotAvailableValue() valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {entry}") - yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, windows_info, linux_banner, - mac_banner)) + yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) # Try to open the file, load it as JSON, read the data from it def run(self): return renderers.TreeGrid([("URI", str), ("Valid", str), ("Number of base_types", int), ("Number of types", int), ("Number of symbols", int), - ("Number of enums", int), ("Windows info", str), ("Linux banner", str), - ("Mac banner", str)], self._generator()) + ("Number of enums", int), ("Identifying infomration", str)], self._generator()) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index a6a7a0fae..1fceb1bcc 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -202,8 +202,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): pass # Finally try looking in zip files - zip_path = os.path.join(path, sub_path + ".zip") - if os.path.exists(zip_path): + for zip_path in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + '.zip'): # We have a zipfile, so run through it and look for sub files that match the filename with zipfile.ZipFile(zip_path) as zfile: for name in zfile.namelist(): diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 41037d464..af3741bbe 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -14,6 +14,8 @@ from urllib import parse, request from volatility3 import symbols from volatility3.framework import constants, contexts, exceptions, interfaces +from volatility3.framework.automagic import symbol_cache +from volatility3.framework.configuration import requirements from volatility3.framework.configuration.requirements import SymbolTableRequirement from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbconv @@ -74,9 +76,15 @@ class PDBUtility(interfaces.configuration.VersionableInterface): isf_path = None # Take the first result of search for the intermediate file - for value in intermed.IntermediateSymbolTable.file_symbol_url("windows", filter_string): + if not requirements.VersionRequirement.matches_required((1, 0, 0), symbol_cache.SqliteCache.version): + vollog.debug(f"Required version of SQLiteCache not found") + return None + + value = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).find_location( + symbol_cache.WindowsIdentifier.generate(pdb_name.strip('\x00'), guid.upper(), age), 'windows') + + if value: isf_path = value - break else: # If none are found, attempt to download the pdb, convert it and try again cls.download_pdb_isf(context, guid.upper(), age, pdb_name, progress_callback) @@ -336,46 +344,12 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - module_name = guid["pdb_name"].strip('.pdb') - - symbol_table_name = cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) - - new_module_name = None - if create_module: - new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], - symbol_table_name = symbol_table_name) - new_module_name = new_module.name - - return new_module_name, symbol_table_name - - @classmethod - def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, - pdb_name: str, module_offset: int = None, module_size: int = None) -> str: - """Creates a module in the specified layer_name based on a pdb name. - - Searches the memory section of the loaded module for its PDB GUID - and loads the associated symbol table into the symbol space. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - config_path: The config path where to find symbol files - layer_name: The name of the layer on which to operate - module_offset: This memory dump's module image offset - module_size: The size of the module for this dump - - Returns: - The name of the constructed and loaded symbol table - """ - - module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, - module_size, create_module = True) - - return module_name + return cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) class PdbSignatureScanner(interfaces.layers.ScannerInterface): From 2729d25d89576b3d31785c1326671eb86455495e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 00:40:01 +0000 Subject: [PATCH 105/181] Automagic: speed up caching by db commit when necessary --- .../framework/automagic/symbol_cache.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index fe717b8be..4b27b8e0f 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -10,7 +10,7 @@ import urllib import urllib.parse import urllib.request from abc import abstractmethod -from typing import Dict, Generator, List, Optional +from typing import Dict, Generator, List, Optional, Tuple import volatility3.framework import volatility3.schemas @@ -158,10 +158,11 @@ class SqliteCache(CacheManagerInterface): self._database = self._connect_storage(filename) def _connect_storage(self, path: str): - database = sqlite3.connect(path, isolation_level = None) + database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + database.commit() return database def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: @@ -229,6 +230,7 @@ class SqliteCache(CacheManagerInterface): counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) + cursor = self._database.cursor() for location in files_to_process: # Open location counter += 1 @@ -246,7 +248,7 @@ class SqliteCache(CacheManagerInterface): if identifier is not None: # We don't try to validate schemas here, we do that on first use # Store in database - self._database.cursor().execute( + cursor.execute( "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", ( location, @@ -256,7 +258,7 @@ class SqliteCache(CacheManagerInterface): )) vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") else: - self._database.cursor().execute( + cursor.execute( "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", ( location, @@ -267,21 +269,27 @@ class SqliteCache(CacheManagerInterface): vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") except Exception as excp: vollog.log(constants.LOGLEVEL_VVVV, excp) + self._database.commit() if not constants.OFFLINE and constants.REMOTE_ISF_URL: + progress_callback(0, 'Reading remote ISF list') remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) + progress_callback(50, 'Reading remote ISF list') + cursor = self._database.cursor() for operating_system in ['mac', 'linux', 'windows']: identifiers = remote_identifiers.process({}, operating_system = operating_system) - for identifier in identifiers: - for location in identifiers[identifier]: - self._database.cursor().execute( - "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now')", - (location, identifier, operating_system, False) - ) + for identifier, location in identifiers: + cursor.execute( + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + (location, identifier, operating_system, False) + ) + progress_callback(100, 'Reading remote ISF list') + self._database.commit() if missing_locations: self._database.cursor().execute( f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ Dict[bytes, str]: @@ -350,23 +358,23 @@ class RemoteIdentifierFormat: return True return False - def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]): + def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]) -> Generator[ + Tuple[bytes, str], None, None]: raise ValueError("Identifier List version not verified") - def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]): + def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]) -> Generator[ + Tuple[bytes, str], None, None]: if operating_system in self._data: for identifier in self._data[operating_system]: binary_identifier = base64.b64decode(identifier) file_list = identifiers.get(binary_identifier, []) for value in self._data[operating_system][identifier]: - if value not in file_list: - file_list = file_list + [value] - identifiers[binary_identifier] = file_list + yield binary_identifier, value if 'additional' in self._data: for location in self._data['additional']: try: subrbf = RemoteIdentifierFormat(location) - subrbf.process(identifiers, operating_system) + yield from subrbf.process(identifiers, operating_system) except IOError: vollog.debug(f"Remote file not found: {location}") return identifiers From 57a202ae1d69de5968a6a49e9bc199724d364152 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 01:04:32 +0000 Subject: [PATCH 106/181] Automagic: Use cache delay for remote locations --- volatility3/framework/automagic/symbol_cache.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 4b27b8e0f..3c7049986 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -149,6 +149,8 @@ class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) + cache_period = '-3 days' + def __init__(self, filename: str): super().__init__(filename) try: @@ -220,13 +222,15 @@ class SqliteCache(CacheManagerInterface): files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " - "AND cached < date('now', '-3 days');") + f"AND cached < date('now', {self.cache_period});") for row in result: if row['location'] in files_to_timestamp: cache_update.add(row['location']) idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + # New or not recently updated + counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) @@ -271,11 +275,15 @@ class SqliteCache(CacheManagerInterface): vollog.log(constants.LOGLEVEL_VVVV, excp) self._database.commit() + # Remote Entries + if not constants.OFFLINE and constants.REMOTE_ISF_URL: progress_callback(0, 'Reading remote ISF list') + cursor = self._database.cursor() + cursor.execute( + f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') - cursor = self._database.cursor() for operating_system in ['mac', 'linux', 'windows']: identifiers = remote_identifiers.process({}, operating_system = operating_system) for identifier, location in identifiers: @@ -286,6 +294,8 @@ class SqliteCache(CacheManagerInterface): progress_callback(100, 'Reading remote ISF list') self._database.commit() + # Missing entries + if missing_locations: self._database.cursor().execute( f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) From fe466386406556a17ba2f474558257e4bb4e8457 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 01:36:17 +0000 Subject: [PATCH 107/181] Automagic: Update to use more recent OS categories --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 3c7049986..8bbedf3e8 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -284,7 +284,7 @@ class SqliteCache(CacheManagerInterface): f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') - for operating_system in ['mac', 'linux', 'windows']: + for operating_system in constants.OS_CATEGORIES: identifiers = remote_identifiers.process({}, operating_system = operating_system) for identifier, location in identifiers: cursor.execute( From 371267f38a61f03007bde4f880b9c45a4b4c2e41 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 21:50:08 +0000 Subject: [PATCH 108/181] Automagic: Ensure partial caching survives --- .../framework/automagic/symbol_cache.py | 80 ++++++++++--------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 8bbedf3e8..54ee13ca2 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -222,7 +222,7 @@ class SqliteCache(CacheManagerInterface): files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " - f"AND cached < date('now', {self.cache_period});") + f"AND cached < date('now', '{self.cache_period}');") for row in result: if row['location'] in files_to_timestamp: cache_update.add(row['location']) @@ -235,45 +235,47 @@ class SqliteCache(CacheManagerInterface): files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) cursor = self._database.cursor() - for location in files_to_process: - # Open location - counter += 1 - progress_callback(counter * 100 / number_files_to_process, - "Updating caches for {number_files_to_process} files...") - try: - with resources.ResourceAccessor().open(location) as fp: - json_obj = json.load(fp) - identifier = None - for idextractor in idextractors: - identifier = idextractor.get_identifier(json_obj) - operating_system = idextractor.operating_system + try: + for location in files_to_process: + # Open location + counter += 1 + progress_callback(counter * 100 / number_files_to_process, + f"Updating caches for {number_files_to_process} files...") + try: + with resources.ResourceAccessor().open(location) as fp: + json_obj = json.load(fp) + identifier = None + for idextractor in idextractors: + identifier = idextractor.get_identifier(json_obj) + operating_system = idextractor.operating_system + if identifier is not None: + break if identifier is not None: - break - if identifier is not None: - # We don't try to validate schemas here, we do that on first use - # Store in database - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - identifier, - operating_system, - self.is_url_local(location) - )) - vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") - else: - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - None, - None, - self.is_url_local(location) - )) - vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") - except Exception as excp: - vollog.log(constants.LOGLEVEL_VVVV, excp) - self._database.commit() + # We don't try to validate schemas here, we do that on first use + # Store in database + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") + else: + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + None, + None, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") + except Exception as excp: + vollog.log(constants.LOGLEVEL_VVVV, excp) + finally: + self._database.commit() # Remote Entries From d16861b5925a473c0bf36a0949bc052321197399 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 22:15:35 +0000 Subject: [PATCH 109/181] Documentation: Update documentation for isf caching feature --- doc/source/symbol-tables.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 4dea6077d..d41e8797a 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -12,20 +12,20 @@ Volatility will automatically decompress them on use. It will also cache their under the user's home directory, in :file:`.cache/volatility3`, along with other useful data. The cache directory currently cannot be altered. -Symbol table JSON files live, by default, under the :file:`volatility3/symbols`, underneath an operating system directory -(currently one of :file:`windows`, :file:`mac` or :file:`linux`). The symbols directory is configurable within the framework and can -usually be set within the user interface. +Symbol table JSON files live, by default, under the :file:`volatility3/symbols` directory. The symbols directory is +configurable within the framework and can usually be set within the user interface. These files can also be compressed into ZIP files, which Volatility will process in order to locate symbol files. -The ZIP file must be named after the appropriate operating system (such as `linux.zip`, `mac.zip` or `windows.zip`). -Inside the ZIP file, the directory structure should match the uncompressed operating system directory. + +Volatility maintains a cache mapping the appropriate identifier for each symbol file against its filename. This cache +is update by automagic called as part of the standard automagic that's run each time a plugin is run. Windows symbol tables --------------------- For Windows systems, Volatility accepts a string made up of the GUID and Age of the required PDB file. It then -searches all files under the configured symbol directories under the windows subdirectory. Any that match the filename -pattern of :file:`/-.json` (or any compressed variant) will be used. If such a symbol table cannot be found, then +searches all files under the configured symbol directories under the windows subdirectory. Any that contain metadata +which matches the pdb name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then the associated PDB file will be downloaded from Microsoft's Symbol Server and converted into the appropriate JSON format, and will be saved in the correct location. @@ -41,11 +41,10 @@ or a virtual environment. Mac/Linux symbol tables ----------------------- -For Mac/Linux systems, both use the same mechanism for identification. JSON files live under the symbol directories, -under either the :file:`linux` or :file:`mac` directories. The generated files contain an identifying string (the operating system +For Mac/Linux systems, both use the same mechanism for identification. The generated files contain an identifying string (the operating system banner), which Volatility's automagic can detect. Volatility caches the mapping between the strings and the symbol tables they come from, meaning the precise file names don't matter and can be organized under any necessary hierarchy -under the operating system directory. +under the symbols directory. Linux and Mac symbol tables can be generated from a DWARF file using a tool called `dwarf2json `_. Currently a kernel with debugging symbols is the only suitable means for recovering all the information required by From 2d64deb18ec0b341a40f416429da3e8b0d1ddb44 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 22:52:04 +0000 Subject: [PATCH 110/181] Plugins: Update isfinfo to use the cache unless --live --- .../framework/automagic/symbol_cache.py | 96 +++++++++++++++---- volatility3/framework/constants/__init__.py | 3 + volatility3/framework/plugins/isfinfo.py | 56 ++++++----- 3 files changed, 112 insertions(+), 43 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 54ee13ca2..c09904713 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -104,7 +104,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns the location of the symbol file given the identifier Args: - identifier: string that uniquely identifies a particular symbolt table + identifier: string that uniquely identifies a particular symbol table operating_system: optional string to restrict identifiers to just those for a particular operating system Returns: @@ -144,6 +144,18 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns all identifiers for a particular operating system""" pass + def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + """Returns ISF statistics based on the location + + Returns: + A tuple of base_types, types, enums, symbols, or None is location not found""" + + def get_verified(self, location: str) -> bool: + """Returns whether a location ISF has been verified against its schema""" + + def set_verified(self, location: str, state: bool = True) -> None: + """Sets the verified state of a location based on whether it has been successfully verified against its schema""" + class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) @@ -163,7 +175,23 @@ class SqliteCache(CacheManagerInterface): database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( - 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCEMA_VERSION})') + schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() + if not schema_version: + database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCEMA_VERSION})') + elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCEMA_VERSION: + # All good, so pass and move on + pass + else: + vollog.info(f"Previous cache schema version found: {schema_version['schema_version']}") + # TODO: Implement code if the schema changes + # Current this should never happen so we start over again + database.close() + os.unlink(path) + return self._connect_storage(path) + database.cursor().execute( + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, verified BOOL DEFAULT False,' + 'stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)') database.commit() return database @@ -207,6 +235,25 @@ class SqliteCache(CacheManagerInterface): return row['identifier'] return None + def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + results = self._database.cursor().execute( + 'SELECT stats_base_types, stats_types, stats_enums, stats_symbols FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['stats_base_types'], row['stats_types'], row['stats_enums'], row['stats_symbols'] + return None + + def get_verified(self, location: str) -> bool: + results = self._database.cursor().execute('SELECT verified FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['verified'] + return False + + def set_verified(self, location: str, state: bool = True) -> None: + self._database.cursor().execute('UPDATE cache (verified) VALUES (?) WHERE location = ?', + (state, location,)) + def update(self, progress_callback = None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. @@ -245,32 +292,39 @@ class SqliteCache(CacheManagerInterface): with resources.ResourceAccessor().open(location) as fp: json_obj = json.load(fp) identifier = None + + # Get stats + stats_base_types = len(json_obj.get('base_types', {})) + stats_types = len(json_obj.get('types', {})) + stats_enums = len(json_obj.get('enums', {})) + stats_symbols = len(json_obj.get('symbols', {})) + + operating_system = None for idextractor in idextractors: identifier = idextractor.get_identifier(json_obj) - operating_system = idextractor.operating_system if identifier is not None: + operating_system = idextractor.operating_system break + + # We don't try to validate schemas here, we do that on first use + # Store in database + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, " + "stats_base_types, stats_types, stats_enums, stats_symbols, " + "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + stats_base_types, + stats_types, + stats_enums, + stats_symbols, + self.is_url_local(location) + )) if identifier is not None: - # We don't try to validate schemas here, we do that on first use - # Store in database - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - identifier, - operating_system, - self.is_url_local(location) - )) vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") else: - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - None, - None, - self.is_url_local(location) - )) vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") except Exception as excp: vollog.log(constants.LOGLEVEL_VVVV, excp) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 322e574e1..3b499adea 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -76,6 +76,9 @@ MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") """Default location to record information about available identifiers""" +CACHE_SQLITE_SCEMA_VERSION = 1 +"""Version for the sqlite3 cache schema""" + BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" ProgressCallback = Optional[Callable[[float, str], None]] diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index b2960733d..b94cfd69a 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -41,7 +41,11 @@ class IsfInfo(plugins.PluginInterface): optional = True), requirements.VersionRequirement(name = 'SQLiteCache', component = symbol_cache.SqliteCache, - version = (1, 0, 0)) + version = (1, 0, 0)), + requirements.BooleanRequirement(name = 'live', + description = 'Traverse all files, rather than use the cache', + default = False, + optional = True) ] @classmethod @@ -92,28 +96,36 @@ class IsfInfo(plugins.PluginInterface): def check_valid(data): return "Unknown" - # Process the filtered list - for entry in filtered_list: - num_types = num_enums = num_bases = num_symbols = 0 - valid = "Unknown" - with resources.ResourceAccessor().open(url = entry) as fp: - try: - data = json.load(fp) - num_symbols = len(data.get('symbols', [])) - num_types = len(data.get('user_types', [])) - num_enums = len(data.get('enums', [])) - num_bases = len(data.get('base_types', [])) + if self.config['live']: + # Process the filtered list + for entry in filtered_list: + num_types = num_enums = num_bases = num_symbols = 0 + valid = "Unknown" + with resources.ResourceAccessor().open(url = entry) as fp: + try: + data = json.load(fp) + num_symbols = len(data.get('symbols', [])) + num_types = len(data.get('user_types', [])) + num_enums = len(data.get('enums', [])) + num_bases = len(data.get('base_types', [])) - identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) - identifier = identifier_cache.get_identifier(location = entry) - if identifier: - identifier = identifier.decode('utf-8', errors = 'replace') - else: - identifier = renderers.NotAvailableValue() - valid = check_valid(data) - except (UnicodeDecodeError, json.decoder.JSONDecodeError): - vollog.warning(f"Invalid ISF: {entry}") - yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) + identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifier = identifier_cache.get_identifier(location = entry) + if identifier: + identifier = identifier.decode('utf-8', errors = 'replace') + else: + identifier = renderers.NotAvailableValue() + valid = check_valid(data) + except (UnicodeDecodeError, json.decoder.JSONDecodeError): + vollog.warning(f"Invalid ISF: {entry}") + yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) + else: + cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + valid = 'Unknown' + for identifier, location in cache.get_identifier_dictionary().items(): + num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) + if identifier: + yield (0, (location, valid, num_bases, num_types, num_symbols, num_enums, str(identifier))) # Try to open the file, load it as JSON, read the data from it From 1f02fea5d10be5c193f2b100bb18c973335504be Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 23:40:47 +0000 Subject: [PATCH 111/181] Automagic: Change database to store ISF hash instead of verified state --- .../framework/automagic/symbol_cache.py | 27 ++++++++----------- volatility3/framework/plugins/isfinfo.py | 12 +++++++++ volatility3/schemas/__init__.py | 16 +++++++++-- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index c09904713..77bc46265 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -14,6 +14,7 @@ from typing import Dict, Generator, List, Optional, Tuple import volatility3.framework import volatility3.schemas +from volatility3 import schemas from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -150,11 +151,8 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A tuple of base_types, types, enums, symbols, or None is location not found""" - def get_verified(self, location: str) -> bool: - """Returns whether a location ISF has been verified against its schema""" - - def set_verified(self, location: str, state: bool = True) -> None: - """Sets the verified state of a location based on whether it has been successfully verified against its schema""" + def get_hash(self, location: str) -> bool: + """Returns the hash of the JSON from within a location ISF""" class SqliteCache(CacheManagerInterface): @@ -190,7 +188,7 @@ class SqliteCache(CacheManagerInterface): os.unlink(path) return self._connect_storage(path) database.cursor().execute( - 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, verified BOOL DEFAULT False,' + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, hash TEXT,' 'stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)') database.commit() return database @@ -243,16 +241,11 @@ class SqliteCache(CacheManagerInterface): return row['stats_base_types'], row['stats_types'], row['stats_enums'], row['stats_symbols'] return None - def get_verified(self, location: str) -> bool: - results = self._database.cursor().execute('SELECT verified FROM cache WHERE location = ?', + def get_hash(self, location: str) -> Optional[str]: + results = self._database.cursor().execute('SELECT hash FROM cache WHERE location = ?', (location,)).fetchall() for row in results: - return row['verified'] - return False - - def set_verified(self, location: str, state: bool = True) -> None: - self._database.cursor().execute('UPDATE cache (verified) VALUES (?) WHERE location = ?', - (state, location,)) + return row['hash'] def update(self, progress_callback = None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. @@ -291,6 +284,7 @@ class SqliteCache(CacheManagerInterface): try: with resources.ResourceAccessor().open(location) as fp: json_obj = json.load(fp) + hash = schemas.create_json_hash(json_obj) identifier = None # Get stats @@ -309,13 +303,14 @@ class SqliteCache(CacheManagerInterface): # We don't try to validate schemas here, we do that on first use # Store in database cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, " + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, hash," "stats_base_types, stats_types, stats_enums, stats_symbols, " - "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", + "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", ( location, identifier, operating_system, + hash, stats_base_types, stats_types, stats_enums, diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index b94cfd69a..af095b69d 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -125,6 +125,18 @@ class IsfInfo(plugins.PluginInterface): for identifier, location in cache.get_identifier_dictionary().items(): num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) if identifier: + json_hash = cache.get_hash(location) + if json_hash and json_hash in schemas.cached_validations: + valid = 'True (cached)' + if self.config['validate']: + # Even if we're not live, if we've been explicitly asked to validate, then do-so + with resources.ResourceAccessor().open(url = location) as fp: + try: + data = json.load(fp) + valid = check_valid(data) + except (UnicodeDecodeError, json.decoder.JSONDecodeError): + vollog.warning(f"Invalid ISF: {location}") + yield (0, (location, valid, num_bases, num_types, num_symbols, num_enums, str(identifier))) # Try to open the file, load it as JSON, read the data from it diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 65329a4f5..8666680b3 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -6,7 +6,7 @@ import hashlib import json import logging import os -from typing import Set, Any, Dict +from typing import Any, Dict, Optional, Set from volatility3.framework import constants @@ -51,9 +51,21 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: return valid(input, schema, use_cache) -def create_json_hash(input: Dict[str, Any], schema: Dict[str, Any]) -> str: +def create_json_hash(input: Dict[str, Any], schema: Optional[Dict[str, Any]] = None) -> Optional[str]: """Constructs the hash of the input and schema to create a unique identifier for a particular JSON file.""" + if schema is None: + format = input.get('metadata', {}).get('format', None) + if not format: + vollog.debug("No schema format defined") + return None + basepath = os.path.abspath(os.path.dirname(__file__)) + schema_path = os.path.join(basepath, 'schema-' + format + '.json') + if not os.path.exists(schema_path): + vollog.debug(f"Schema for format not found: {schema_path}") + return None + with open(schema_path, 'r') as s: + schema = json.load(s) return hashlib.sha1(bytes(json.dumps((input, schema), sort_keys = True), 'utf-8')).hexdigest() From bda200168a378f79c76acacf93edb6500f766e55 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sat, 28 May 2022 23:49:15 +0100 Subject: [PATCH 112/181] Update volatility3/framework/plugins/isfinfo.py Yep, good spot as ever, thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/plugins/isfinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index af095b69d..6b13f10b6 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -144,4 +144,4 @@ class IsfInfo(plugins.PluginInterface): def run(self): return renderers.TreeGrid([("URI", str), ("Valid", str), ("Number of base_types", int), ("Number of types", int), ("Number of symbols", int), - ("Number of enums", int), ("Identifying infomration", str)], self._generator()) + ("Number of enums", int), ("Identifying information", str)], self._generator()) From ebab09e53a0c56632edc45260e013b42f8097af2 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sat, 28 May 2022 23:50:23 +0100 Subject: [PATCH 113/181] Update volatility3/framework/automagic/symbol_cache.py Cool, I always forget about that, I think it's just what I'm used to, thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 77bc46265..2908774c0 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -276,7 +276,7 @@ class SqliteCache(CacheManagerInterface): number_files_to_process = len(files_to_process) cursor = self._database.cursor() try: - for location in files_to_process: + for counter, location in enumerate(files_to_process): # Open location counter += 1 progress_callback(counter * 100 / number_files_to_process, From a4aa93f05945ab3c3778a25a0c0d3e4709e62c01 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 28 May 2022 23:51:54 +0100 Subject: [PATCH 114/181] Core: Clean up unneeded counter variable, now we're using enumerate --- volatility3/framework/automagic/symbol_cache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 2908774c0..156a1e8c2 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -271,14 +271,12 @@ class SqliteCache(CacheManagerInterface): # New or not recently updated - counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) cursor = self._database.cursor() try: for counter, location in enumerate(files_to_process): # Open location - counter += 1 progress_callback(counter * 100 / number_files_to_process, f"Updating caches for {number_files_to_process} files...") try: From 1e80bb54deb5c8a7cf82057f996bf197058828e7 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:34:44 +0100 Subject: [PATCH 115/181] Update volatility3/framework/configuration/requirements.py Yep, not sure why I forgot, thanks 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index b31c4767f..cc4f05ae6 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -414,7 +414,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): return {} @classmethod - def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]): + def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]) -> bool: if len(required) > 0 and version[0] != required[0]: return False if len(required) > 1 and version[1] < required[1]: From 6f34e1350e67ca893d0f1c5984c45813d9892b5a Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:35:42 +0100 Subject: [PATCH 116/181] Update volatility3/framework/automagic/symbol_cache.py Hehehe, I guess I'm just a little shy about handing out complex objects, but you're right and it is a private method. 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 156a1e8c2..efe1ce601 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -169,7 +169,7 @@ class SqliteCache(CacheManagerInterface): os.unlink(filename) self._database = self._connect_storage(filename) - def _connect_storage(self, path: str): + def _connect_storage(self, path: str) -> sqlite3.Connection: database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( From 98f7fe17433b11a2ef3950e87fabaab8e757e67d Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:49:20 +0100 Subject: [PATCH 117/181] Update volatility3/framework/automagic/symbol_cache.py Quite right, thanks for the catch! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index efe1ce601..47ff66121 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -151,7 +151,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A tuple of base_types, types, enums, symbols, or None is location not found""" - def get_hash(self, location: str) -> bool: + def get_hash(self, location: str) -> Optional[str]: """Returns the hash of the JSON from within a location ISF""" From 504229e46886d9f6d8d3c6a6b8782d67e3656600 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 May 2022 10:52:29 +0100 Subject: [PATCH 118/181] Automagic: include fixes from @digitalisx on review --- volatility3/framework/automagic/symbol_cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 47ff66121..378424ef5 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -10,7 +10,7 @@ import urllib import urllib.parse import urllib.request from abc import abstractmethod -from typing import Dict, Generator, List, Optional, Tuple +from typing import Dict, Generator, Iterable, List, Optional, Tuple import volatility3.framework import volatility3.schemas @@ -113,7 +113,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """ pass - def get_local_locations(self) -> List[str]: + def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" pass @@ -141,7 +141,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns an identifier based on a specific location or None""" pass - def get_identifiers(self, operating_system: Optional[str]): + def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" pass @@ -369,7 +369,7 @@ class SqliteCache(CacheManagerInterface): output[row['identifier']] = row['location'] return output - def get_identifiers(self, operating_system: Optional[str]): + def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: if operating_system: results = self._database.cursor().execute('SELECT identifier FROM cache WHERE operating_system = ?', (operating_system,)).fetchall() From 47accf520bb322040e0cbf1facfa74e32dc944bb Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:55:17 +0100 Subject: [PATCH 119/181] Update volatility3/framework/automagic/symbol_cache.py Yep, you're quite right, not sure how that got left behind. Thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 378424ef5..ed64746fd 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -426,7 +426,6 @@ class RemoteIdentifierFormat: if operating_system in self._data: for identifier in self._data[operating_system]: binary_identifier = base64.b64decode(identifier) - file_list = identifiers.get(binary_identifier, []) for value in self._data[operating_system][identifier]: yield binary_identifier, value if 'additional' in self._data: From c475b792a53305fe7769134f46d1cf502e29e9b6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jun 2022 17:34:22 +0100 Subject: [PATCH 120/181] Automgic: Fix removing stale entries --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index ed64746fd..46431b773 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -347,7 +347,7 @@ class SqliteCache(CacheManagerInterface): if missing_locations: self._database.cursor().execute( - f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", [x for x in missing_locations]) self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ From 225c36631403fe3fa58208befc9d36ad93424b83 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jun 2022 18:54:00 +0100 Subject: [PATCH 121/181] Windows: Update PDB to store correct age value --- volatility3/framework/symbols/windows/pdbconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index da8254ffd..15b5c733a 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -521,7 +521,7 @@ class PdbReader: self.metadata['windows']['pdb'] = { "GUID": self.convert_bytes_to_guid(pdb_info.GUID), - "age": pdb_info.age, + "age": self._dbiheader.age, "database": self._database_name or 'unknown.pdb', "machine_type": self._dbiheader.machine } From ae48a8ab479cc1f60550eef6fe9502b0d49f2e74 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 20 Jul 2022 20:40:23 +0100 Subject: [PATCH 122/181] Documentation: Update text about long cache updates --- README.md | 3 +++ doc/source/symbol-tables.rst | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9f9c1bbb7..348121e44 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,9 @@ Symbol tables zip files must be placed, as named, into the `volatility3/symbols` Windows symbols that cannot be found will be queried, downloaded, generated and cached. Mac and Linux symbol tables must be manually produced by a tool such as [dwarf2json](https://github.com/volatilityfoundation/dwarf2json). +Important: The first run of volatility with new symbol files will require the cache to be updated. The symbol packs contain a large number of symbol files and so may take some time to update! +However, this process only needs to be run once on each new symbol file, so assuming the pack stays in the same location will not need to be done again. Please also note it can be interrupted and next run will restart itself. + Please note: These are representative and are complete up to the point of creation for Windows and Mac. Due to the ease of compiling Linux kernels and the inability to uniquely distinguish them, an exhaustive set of Linux symbol tables cannot easily be supplied. ## Documentation diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index d41e8797a..fd8b8933e 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -18,7 +18,9 @@ configurable within the framework and can usually be set within the user interfa These files can also be compressed into ZIP files, which Volatility will process in order to locate symbol files. Volatility maintains a cache mapping the appropriate identifier for each symbol file against its filename. This cache -is update by automagic called as part of the standard automagic that's run each time a plugin is run. +is updated by automagic called as part of the standard automagic that's run each time a plugin is run. If a large number of new +symbols file are detected, this may take some time, but can be safely interrupted and restarted and will not need to run again +as long as the symbol files stay in the same location. Windows symbol tables --------------------- @@ -92,4 +94,4 @@ file, the banners must match exactly (down to the compilation date). * Copy the `.json` file to the symbols directory into `[symbols directory]/linux` - * For Mac change `linux` to `mac` \ No newline at end of file + * For Mac change `linux` to `mac` From d5c7ef1a9e61fca00b40db1dab7ed52343637278 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:16 +0900 Subject: [PATCH 123/181] Remove: pytest install command --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a3ecd7c7e..cf70b66cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -17,7 +17,6 @@ jobs: python -m pip install --upgrade pip pip install Cmake pip install setuptools wheel - pip install -U pytest pip install -r ./test/requirements-testing.txt - name: Build PyPi packages From 5db182d4303db48dbc4ba401f22b9ad5ff354f7c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:39 +0900 Subject: [PATCH 124/181] Add: .gitignore for test --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index d26e17d91..b3c86d49b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ ENV/ # Memory dump files *.dmp *.vmem +*.img + +# PyTest cache files +.pytest_cache/ \ No newline at end of file From 723fd9b4293b5e5231a2dacd05aa973567c0a9ca Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:51 +0900 Subject: [PATCH 125/181] Fix: json prettier --- test/known_files.json | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/test/known_files.json b/test/known_files.json index 089896714..a579c8053 100644 --- a/test/known_files.json +++ b/test/known_files.json @@ -1,20 +1,19 @@ { - "windows_dumpfiles": { - "win-xp-laptop-2005-06-25.img": { - "0x82220e78": [ - "9bdd5532286f1660f3778e68bc36efe6", - "e3bc1e9e7370e3b5a661ebe591ecf4ec" - ], - "0x82350bf8": [ - "e5c5e8d97b6280745b41f6572c85d1f0", - "8589f1463422884dbf1411aaad278465" - ], - "0x81eaf418": [ - "f7a1ae2060a58f8470b97affdb46dccf", - "54fd611021fa784912530b8007545986" - ], - "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" - } + "windows_dumpfiles": { + "win-xp-laptop-2005-06-25.img": { + "0x82220e78": [ + "9bdd5532286f1660f3778e68bc36efe6", + "e3bc1e9e7370e3b5a661ebe591ecf4ec" + ], + "0x82350bf8": [ + "e5c5e8d97b6280745b41f6572c85d1f0", + "8589f1463422884dbf1411aaad278465" + ], + "0x81eaf418": [ + "f7a1ae2060a58f8470b97affdb46dccf", + "54fd611021fa784912530b8007545986" + ], + "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" } } - \ No newline at end of file +} From 8e9bf4f27cf26cb4578a053808d5c305c15d171c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:50:46 +0900 Subject: [PATCH 126/181] Add: pytest in requirements-test.txt --- test/requirements-testing.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index d37dc93c3..e47f72fa2 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -6,3 +6,5 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 + +pytest>=7.1.2 \ No newline at end of file From 929d19aa50b8b7eef492bcc4215f39b29055fc16 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:51:54 +0900 Subject: [PATCH 127/181] Add: EOF in requirements-test.txt --- test/requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index e47f72fa2..e5966906c 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -7,4 +7,4 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 -pytest>=7.1.2 \ No newline at end of file +pytest>=7.1.2 From 3587828820a21d02fdb901a6d2c61dd1733ae935 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:54:36 +0900 Subject: [PATCH 128/181] Add: EOF in requirements-test.txt --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b3c86d49b..328ba5f83 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,4 @@ ENV/ *.img # PyTest cache files -.pytest_cache/ \ No newline at end of file +.pytest_cache/ From 986088b1a1b0084c8739200b7ff0792f025cc79b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:56:17 +0900 Subject: [PATCH 129/181] Fix: pytest version --- test/requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index e5966906c..7afe19b94 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -7,4 +7,4 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 -pytest>=7.1.2 +pytest>=7.0.0 From 7755328226af96ba811a63d06c8d222877cf28a8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 22 Jul 2022 01:11:35 +0900 Subject: [PATCH 130/181] Fix: psscan required framework version bump --- volatility3/framework/plugins/windows/psscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index cc030b4bf..335624672 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -22,7 +22,7 @@ vollog = logging.getLogger(__name__) class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for processes present in a particular windows memory image.""" - _required_framework_version = (2, 2, 1) + _required_framework_version = (2, 3, 1) _version = (1, 1, 0) @classmethod From 0fe1f47c9c4978c993de4b9fdb6af3919f72370f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 22 Jul 2022 21:10:43 +0900 Subject: [PATCH 131/181] Fix: support swapped exceptions --- volatility3/framework/plugins/windows/devicetree.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index 8e92de0cc..1b6b55cb7 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -78,7 +78,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): """Listing tree based on drivers and attached devices in a particular windows memory image.""" _required_framework_version = (2, 0, 3) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -96,7 +96,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): try: try: driver_name = driver.get_driver_name() - except (ValueError, exceptions.PagedInvalidAddressException): + except (ValueError, exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Failed to get Driver name : {driver.vol.offset:x}") driver_name = renderers.UnparsableValue() @@ -114,7 +114,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): for device in driver.get_devices(): try: device_name = device.get_device_name() - except (ValueError, exceptions.PagedInvalidAddressException): + except (ValueError, exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Failed to get Device name : {device.vol.offset:x}") device_name = renderers.UnparsableValue() @@ -134,7 +134,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): for level, attached_device in enumerate(device.get_attached_devices(), start=2): try: device_name = attached_device.get_device_name() - except (ValueError, exceptions.PagedInvalidAddressException): + except (ValueError, exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Failed to get Attached Device Name: {attached_device.vol.offset:x}") device_name = renderers.UnparsableValue() @@ -151,7 +151,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): attached_device_type )) - except(exceptions.PagedInvalidAddressException): + except(exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in drivers and devices: {driver.vol.offset:x}") continue From 65d825626e6d847d1a007e9672aa686138bf447b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 22 Jul 2022 21:11:01 +0900 Subject: [PATCH 132/181] Add: test for windows.devicetree --- test/test_volatility.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index a55dffb27..eb713783b 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -199,6 +199,17 @@ def test_windows_callbacks(image, volatility, python): assert out.count(b"KeBugCheckReasonCallbackListHead ") > 5 assert rc == 0 +def test_windows_devicetree(image, volatility, python): + rc, out, err = runvol_plugin("windows.devicetree.DeviceTree", image, volatility, python) + + assert out.find(b"DEV") != -1 + assert out.find(b"DRV") != -1 + assert out.find(b"ATT") != -1 + assert out.find(b"FILE_DEVICE_CONTROLLER") != -1 + assert out.find(b"FILE_DEVICE_DISK") != -1 + assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 + assert rc == 0 + # LINUX def test_linux_pslist(image, volatility, python): From 5a33acd8f955df513f8660d2a5d449a025818262 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 28 Jul 2022 17:44:03 +0300 Subject: [PATCH 133/181] bugfix --- volatility3/framework/automagic/symbol_cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 46431b773..c7cb6a5b8 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -216,7 +216,7 @@ class SqliteCache(CacheManagerInterface): return result def get_local_locations(self) -> Generator[str, None, None]: - result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = True').fetchall() + result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = 1').fetchall() for row in result: yield row['location'] @@ -261,7 +261,7 @@ class SqliteCache(CacheManagerInterface): cache_update = set() files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: - result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " + result = self._database.cursor().execute("SELECT location FROM cache WHERE local = 1 " f"AND cached < date('now', '{self.cache_period}');") for row in result: if row['location'] in files_to_timestamp: @@ -330,7 +330,7 @@ class SqliteCache(CacheManagerInterface): progress_callback(0, 'Reading remote ISF list') cursor = self._database.cursor() cursor.execute( - f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") + f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') for operating_system in constants.OS_CATEGORIES: @@ -356,7 +356,7 @@ class SqliteCache(CacheManagerInterface): additions = [] statement = 'SELECT location, identifier FROM cache' if local_only: - additions.append('local = True') + additions.append('local = 1') if operating_system: additions.append(f"operating_system = '{operating_system}'") if additions: From 95c0ca4ffa0756854a8e16d657175aa67bd7077a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 01:47:18 +0900 Subject: [PATCH 134/181] Remove: pytest module --- test/test_volatility.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index eb713783b..515bef1cc 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -14,8 +14,6 @@ import hashlib import ntpath import json -import pytest - # # HELPER FUNCTIONS # @@ -61,7 +59,6 @@ def test_windows_pslist(image, volatility, python): assert out.find(b"svchost.exe") != -1 assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 rc, out, err = runvol_plugin( "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"]) @@ -69,7 +66,6 @@ def test_windows_pslist(image, volatility, python): assert out.find(b"system") != -1 assert out.count(b"\n") < 10 assert rc == 0 - assert rc == 0 def test_windows_psscan(image, volatility, python): rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) @@ -79,21 +75,18 @@ def test_windows_psscan(image, volatility, python): assert out.find(b"svchost.exe") != -1 assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 def test_windows_dlllist(image, volatility, python): rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 def test_windows_modules(image, volatility, python): rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 def test_windows_hivelist(image, volatility, python): rc, out, err = runvol_plugin("windows.registry.hivelist.HiveList", image, volatility, python) From 64621d90e97cbb2300689559142f009f1af83f9e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:20:01 +0900 Subject: [PATCH 135/181] Add: VSL for frameworkinfo plugin --- volatility3/framework/plugins/frameworkinfo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/frameworkinfo.py b/volatility3/framework/plugins/frameworkinfo.py index b7c887d5c..63ba24d09 100644 --- a/volatility3/framework/plugins/frameworkinfo.py +++ b/volatility3/framework/plugins/frameworkinfo.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from typing import List from volatility3 import framework From 9bfa80e59cb1907163b5adfc054a6a192332630f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:26:21 +0900 Subject: [PATCH 136/181] Add: VSL for initialize file --- volatility3/framework/layers/codecs/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index 550161e6d..e019bcbcd 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + """Codecs used for encoding or decoding data should live here From 54f11d7e18b12c7a8fd384fa333406e5c3dded25 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:26:47 +0900 Subject: [PATCH 137/181] Add: VSL for automagic/module --- volatility3/framework/automagic/module.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index 3d2bb584a..6810a58e2 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from volatility3.framework import interfaces, constants, configuration From a78bf32bd8df8fc075f52fed211e0bf4a9bb7840 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:26:58 +0900 Subject: [PATCH 138/181] Add: VSL for layers/avml --- volatility3/framework/layers/avml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index acc4493f4..f31737232 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + """Functions that read AVML files. The user of the file doesn't have to worry about the compression, From 85a94efd67c41993b15d066343c545da05c2c898 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:27:08 +0900 Subject: [PATCH 139/181] Add: VSL for layers/leechcore --- volatility3/framework/layers/leechcore.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index 8c492ca85..fb0442cfe 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import io import logging import urllib.parse From 5c76dc88e9bf4ec809f1e914a35ace5f14b446c2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:27:15 +0900 Subject: [PATCH 140/181] Add: VSL for layers/linear --- volatility3/framework/layers/linear.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/linear.py b/volatility3/framework/layers/linear.py index c5cb47bdc..383f3d558 100644 --- a/volatility3/framework/layers/linear.py +++ b/volatility3/framework/layers/linear.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import functools from typing import List, Optional, Tuple, Iterable From 6f991f8d4f6d663bd69d33b8c83f61ce37ea39f2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Aug 2022 22:01:21 +0100 Subject: [PATCH 141/181] Core: Fix up LGTM issues across the codebase --- volatility3/framework/automagic/symbol_cache.py | 13 ++++++------- volatility3/framework/automagic/symbol_finder.py | 5 ++--- volatility3/framework/plugins/linux/psaux.py | 16 ++++++++-------- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index c7cb6a5b8..558bfb2f1 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -12,9 +12,7 @@ import urllib.request from abc import abstractmethod from typing import Dict, Generator, Iterable, List, Optional, Tuple -import volatility3.framework -import volatility3.schemas -from volatility3 import schemas +from volatility3 import framework, schemas from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -41,7 +39,7 @@ class IdentifierProcessor: Returns: identifier is valid or None if not found """ - raise NotImplemented("This base class has no get_identifier method defined") + raise NotImplementedError("This base class has no get_identifier method defined") class WindowsIdentifier(IdentifierProcessor): @@ -94,7 +92,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): super().__init__() self._filename = filename self._classifiers = {} - for subclazz in volatility3.framework.class_subclasses(IdentifierProcessor): + for subclazz in framework.class_subclasses(IdentifierProcessor): self._classifiers[subclazz.operating_system] = subclazz def add_identifier(self, location: str, operating_system: str, identifier: str): @@ -267,7 +265,7 @@ class SqliteCache(CacheManagerInterface): if row['location'] in files_to_timestamp: cache_update.add(row['location']) - idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + idextractors = list(framework.class_subclasses(IdentifierProcessor)) # New or not recently updated @@ -347,7 +345,8 @@ class SqliteCache(CacheManagerInterface): if missing_locations: self._database.cursor().execute( - f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", [x for x in missing_locations]) + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", + [x for x in missing_locations]) self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 610ed0e18..a9221a7cc 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -123,9 +123,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): requirement.construct(context, config_path) break else: - if symbol_files: - vollog.debug(f"Symbol library path not found: {symbol_files}") - # print("Kernel", banner, hex(banner_offset)) + vollog.debug(f"Symbol library path not found for: {banner}") + # print("Kernel", banner, hex(banner_offset)) else: vollog.debug("No existing banners found") # TODO: Fallback to generic regex search? diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index ed91c66f2..d8b844ca4 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -4,11 +4,12 @@ from typing import Optional +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework import symbols, exceptions, renderers, interfaces +from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.plugins.linux import pslist -from volatility3.framework.interfaces import plugins + class PsAux(plugins.PluginInterface): """ Lists processes with their command line arguments """ @@ -29,7 +30,7 @@ class PsAux(plugins.PluginInterface): ] def _get_command_line_args(self, task: interfaces.objects.ObjectInterface, - name: str) -> Optional[str]: + name: str) -> Optional[str]: """ Reads the command line arguments of a process These are stored on the userland stack @@ -104,8 +105,7 @@ class PsAux(plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], - self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) - + self._generator( + pslist.PsList.list_tasks(self.context, + self.config['kernel'], + filter_func = filter_func))) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index af3741bbe..430ad6a30 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -13,7 +13,7 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Union from urllib import parse, request from volatility3 import symbols -from volatility3.framework import constants, contexts, exceptions, interfaces +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.configuration.requirements import SymbolTableRequirement From 4c4ccbf4e0e1893b8eefacdb4264ff14585a8802 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Aug 2022 22:03:26 +0100 Subject: [PATCH 142/181] Core: Fix remaining LGTM error --- volatility3/framework/layers/physical.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 0633637ca..b09055c90 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -5,7 +5,7 @@ import logging import threading from typing import Any, Dict, IO, List, Optional, Union -from volatility3.framework import exceptions, interfaces, constants +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -191,7 +191,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): """Closes the file handle.""" self._file.close() - def __exit__(self) -> None: + def __exit__(self, type, value, traceback) -> None: self.destroy() @classmethod From f8506862c4d92422a5e8927f70778f7faf69faf9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Aug 2022 22:59:45 +0100 Subject: [PATCH 143/181] Core: Move jsonschema to dev requirements --- requirements-dev.txt | 26 ++++++++++++++++++++++++++ requirements.txt | 3 --- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..3ff7c50b8 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,26 @@ +# The following packages are required for core functionality. +pefile>=2017.8.1 + +# The following packages are optional. +# If certain packages are not necessary, place a comment (#) at the start of the line. + +# This is required for the yara plugins +yara-python>=3.8.0 + +# This is required for several plugins that perform malware analysis and disassemble code. +# It can also improve accuracy of Windows 8 and later memory samples. +capstone>=3.0.5 + +# This is required by plugins that decrypt passwords, password hashes, etc. +pycryptodome + +# This can improve error messages regarding improperly configured ISF files, +# but is only recommended for development +# jsonschema>=2.3.0 + +# This is required for memory acquisition via leechcore/pcileech. +leechcorepyc>=2.4.0 + +# This is required for analyzing Linux samples compressed using AVMLs native +# compression format. It is not required for AVML's standard LiME compression. +python-snappy==0.6.0 diff --git a/requirements.txt b/requirements.txt index 290d9ca97..1793012f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,9 +14,6 @@ capstone>=3.0.5 # This is required by plugins that decrypt passwords, password hashes, etc. pycryptodome -# This can improve error messages regarding improperly configured ISF files. -jsonschema>=2.3.0 - # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 From 989b4c73273b1acfd767f64a63599fbc511d77dc Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 5 Aug 2022 03:08:18 +0900 Subject: [PATCH 144/181] Fix: cache path for python of Windows Store version --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3b499adea..e83d27108 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -64,7 +64,7 @@ CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" if sys.platform == 'win32': - CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") + CACHE_PATH = os.path.realpath(os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3")) os.makedirs(CACHE_PATH, exist_ok = True) LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") From 837e1ef39df3f5db6422d541b163b47d8226bb83 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 15:58:07 +0900 Subject: [PATCH 145/181] Fix: error handling for netstat plugin --- volatility3/framework/plugins/windows/netstat.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 486957565..3051b950e 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -433,7 +433,8 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self.context, interfaces.configuration.path_join(self.config_path, 'tcpip'), kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) except exceptions.VolatilityException: - vollog.warning("Unable to locate symbols for the memory image's tcpip module") + vollog.error("Unable to locate symbols for the memory image's tcpip module") + raise for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): From a04cb4e031f0a0092aec57ff72c371485893dd66 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 16:13:29 +0900 Subject: [PATCH 146/181] Fix: return syntax --- volatility3/framework/plugins/windows/netstat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 3051b950e..4d6ec5f62 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -434,7 +434,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) except exceptions.VolatilityException: vollog.error("Unable to locate symbols for the memory image's tcpip module") - raise + return for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): From 4f77be32a541563b35279dc7bfda7ee6a52ca853 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 16:30:14 +0900 Subject: [PATCH 147/181] Remove: dump file namespace --- volatility3/plugins/windows/registry/certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 429db96a6..8ef5abcdd 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -47,7 +47,7 @@ class Certificates(interfaces.plugins.PluginInterface): Optional[interfaces.plugins.FileHandlerInterface]: try: if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) + dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) file_handle = open_method(dump_name) file_handle.write(certificate_data) return file_handle From 0c8d4f75ae63a2396deceef229e1c4d2e26135f3 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 16:57:43 +0900 Subject: [PATCH 148/181] Fix: wide exceptions --- volatility3/plugins/windows/registry/certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 8ef5abcdd..6029c0a5c 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -68,7 +68,7 @@ class Certificates(interfaces.plugins.PluginInterface): "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - with contextlib.suppress(KeyError, exceptions.SwappedInvalidAddressException): + with contextlib.suppress(KeyError, exceptions.InvalidAddressException): # Walk it node_path = hive.get_key(top_key, return_list = True) for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): From 471551fda0bc5871f8738f05b06fef1f35c5f826 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 03:13:40 +0900 Subject: [PATCH 149/181] Add: initialize for windows.joblinks plugin --- test/test_volatility.py | 5 ++ .../framework/plugins/windows/joblinks.py | 72 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 volatility3/framework/plugins/windows/joblinks.py diff --git a/test/test_volatility.py b/test/test_volatility.py index 515bef1cc..1126aa9d7 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -203,6 +203,11 @@ def test_windows_devicetree(image, volatility, python): assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 assert rc == 0 +def test_windows_joblinks(image, volatility, python): + rc, out, err = runvol_plugin("windows.joblinks.JobLinks", image, volatility, python) + + assert rc == 0 + # LINUX def test_linux_pslist(image, volatility, python): diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py new file mode 100644 index 000000000..c440cad29 --- /dev/null +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -0,0 +1,72 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterable, Iterator, List, Tuple + +from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.constants import LOGLEVEL_VVVV +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + +class JobLinks(interfaces.plugins.PluginInterface): + """Print process job link information""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls)-> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.BooleanRequirement(name = 'physical', + description = "Display physical offset instead of virtual", + default = False, + optional = True), + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) + ] + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config['kernel']] + memory = self.context.layers[kernel.layer_name] + + for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, + kernel.symbol_table_name): + try: + if not self.config['physical']: + offset = proc.vol.offset + else: + (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + + job = proc.Job.dereference() + + yield (0, ( + format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), + job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, + renderers.NotApplicableValue(), + "(Original Process)" + )) + + vollog.log(LOGLEVEL_VVVV, proc.JobLinks) + vollog.log(LOGLEVEL_VVVV, job.JobLinks) + vollog.log(LOGLEVEL_VVVV, job.ProcessListHead) + + except (exceptions.InvalidAddressException): + continue + + def run(self)-> renderers.TreeGrid: + offsettype = "(V)" if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT) else "(P)" + + return renderers.TreeGrid([ + (f"Offset{offsettype}", format_hints.Hex), ("Name", str), ("PID", int), + ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), + ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str) + ], self._generator()) \ No newline at end of file From e0edb87d7f15b883aea2fbec5539ec1850307037 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 03:25:21 +0900 Subject: [PATCH 150/181] Add: EOF --- volatility3/framework/plugins/windows/joblinks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index c440cad29..2ca32e09d 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -69,4 +69,4 @@ class JobLinks(interfaces.plugins.PluginInterface): (f"Offset{offsettype}", format_hints.Hex), ("Name", str), ("PID", int), ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str) - ], self._generator()) \ No newline at end of file + ], self._generator()) From 7bec33b07d9ab74e798bee315ae9bb8ed8eab26d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 14:40:35 +0900 Subject: [PATCH 151/181] Add: debug log code --- .../framework/plugins/windows/joblinks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 2ca32e09d..e5a206af2 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -56,8 +56,24 @@ class JobLinks(interfaces.plugins.PluginInterface): )) vollog.log(LOGLEVEL_VVVV, proc.JobLinks) + vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Flink)) + vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Blink)) vollog.log(LOGLEVEL_VVVV, job.JobLinks) + vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Flink)) + vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Blink)) vollog.log(LOGLEVEL_VVVV, job.ProcessListHead) + vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Flink)) + vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Blink)) + vollog.log(LOGLEVEL_VVVV, "") + + for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): + yield (1, ( + format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, + entry.InheritedFromUniqueProcessId, entry.get_session_id(), renderers.NotApplicableValue(), entry.get_is_wow64(), + renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + "(Original Process)" + )) except (exceptions.InvalidAddressException): continue From fa686a9fa69c23361df9410b860823b34e31fe38 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:04:24 +0900 Subject: [PATCH 152/181] Add: Peb.ProcessParameters.ImagePathName --- volatility3/framework/plugins/windows/joblinks.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index e5a206af2..1ee64544f 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -55,24 +55,13 @@ class JobLinks(interfaces.plugins.PluginInterface): "(Original Process)" )) - vollog.log(LOGLEVEL_VVVV, proc.JobLinks) - vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Flink)) - vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Blink)) - vollog.log(LOGLEVEL_VVVV, job.JobLinks) - vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Flink)) - vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Blink)) - vollog.log(LOGLEVEL_VVVV, job.ProcessListHead) - vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Flink)) - vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Blink)) - vollog.log(LOGLEVEL_VVVV, "") - for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): yield (1, ( format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), renderers.NotApplicableValue(), entry.get_is_wow64(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - "(Original Process)" + "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string() )) except (exceptions.InvalidAddressException): From ea65649548d708aabe2b4568aa712ef3e8e58ff2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:08:13 +0900 Subject: [PATCH 153/181] Remove: test_windows_joblinks function for test --- test/test_volatility.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 1126aa9d7..515bef1cc 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -203,11 +203,6 @@ def test_windows_devicetree(image, volatility, python): assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 assert rc == 0 -def test_windows_joblinks(image, volatility, python): - rc, out, err = runvol_plugin("windows.joblinks.JobLinks", image, volatility, python) - - assert rc == 0 - # LINUX def test_linux_pslist(image, volatility, python): From 65e7b5302c12068cb78b712d8358f4870eab49dd Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:18:11 +0900 Subject: [PATCH 154/181] Fix: job detail info to zero --- volatility3/framework/plugins/windows/joblinks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 1ee64544f..321e6f791 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -58,8 +58,8 @@ class JobLinks(interfaces.plugins.PluginInterface): for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): yield (1, ( format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, - entry.InheritedFromUniqueProcessId, entry.get_session_id(), renderers.NotApplicableValue(), entry.get_is_wow64(), - renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), + entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), + 0, 0, 0, "Yes", entry.get_peb().ProcessParameters.ImagePathName.get_string() )) From 3556b2374a3f9593d58564431d057ead5859cea7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:22:07 +0900 Subject: [PATCH 155/181] Fix: indent for prettier code --- volatility3/framework/plugins/windows/joblinks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 321e6f791..e88b7a3d3 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -57,12 +57,12 @@ class JobLinks(interfaces.plugins.PluginInterface): for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): yield (1, ( - format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, - entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), - 0, 0, 0, - "Yes", - entry.get_peb().ProcessParameters.ImagePathName.get_string() - )) + format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, + entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), + 0, 0, 0, + "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string() + )) except (exceptions.InvalidAddressException): continue From 2918c13046b91ecdae1fba6599d291fc7ba95ce7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:24:40 +0900 Subject: [PATCH 156/181] Fix: offset for job entry --- volatility3/framework/plugins/windows/joblinks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index e88b7a3d3..39089a741 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -56,6 +56,12 @@ class JobLinks(interfaces.plugins.PluginInterface): )) for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): + + if not self.config['physical']: + offset = entry.vol.offset + else: + (_, _, offset, _, _) = list(memory.mapping(offset = entry.vol.offset, length = 0))[0] + yield (1, ( format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), From cd6a73939ed19426e47209c532481c962b223204 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:27:39 +0900 Subject: [PATCH 157/181] Refactor: apply code style by yapf --- .../framework/plugins/windows/joblinks.py | 49 ++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 39089a741..e30044538 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -15,6 +15,7 @@ from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) + class JobLinks(interfaces.plugins.PluginInterface): """Print process job link information""" @@ -22,62 +23,54 @@ class JobLinks(interfaces.plugins.PluginInterface): _version = (1, 0, 0) @classmethod - def get_requirements(cls)-> List[interfaces.configuration.RequirementInterface]: + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), requirements.BooleanRequirement(name = 'physical', description = "Display physical offset instead of virtual", default = False, optional = True), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) ] def _generator(self) -> Iterator[Tuple]: kernel = self.context.modules[self.config['kernel']] memory = self.context.layers[kernel.layer_name] - for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, - kernel.symbol_table_name): + for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, kernel.symbol_table_name): try: if not self.config['physical']: offset = proc.vol.offset else: (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] - + job = proc.Job.dereference() - - yield (0, ( - format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, - proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), - job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, - renderers.NotApplicableValue(), - "(Original Process)" - )) + + yield (0, (format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), + job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, + renderers.NotApplicableValue(), "(Original Process)")) for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): - if not self.config['physical']: offset = entry.vol.offset else: (_, _, offset, _, _) = list(memory.mapping(offset = entry.vol.offset, length = 0))[0] - yield (1, ( - format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, - entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), - 0, 0, 0, - "Yes", - entry.get_peb().ProcessParameters.ImagePathName.get_string() - )) + yield (1, (format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), + entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, + entry.get_is_wow64(), 0, 0, 0, "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string())) except (exceptions.InvalidAddressException): continue - def run(self)-> renderers.TreeGrid: + def run(self) -> renderers.TreeGrid: offsettype = "(V)" if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT) else "(P)" - return renderers.TreeGrid([ - (f"Offset{offsettype}", format_hints.Hex), ("Name", str), ("PID", int), - ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), - ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str) - ], self._generator()) + return renderers.TreeGrid([(f"Offset{offsettype}", format_hints.Hex), ("Name", str), + ("PID", int), ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), + ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str)], + self._generator()) From 0aedc6a071c9bc0a2b88a7ef978a80be3a9d2e03 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:40:17 +0900 Subject: [PATCH 158/181] Remove: unused module --- volatility3/framework/plugins/windows/joblinks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index e30044538..40d09b9ea 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -4,11 +4,10 @@ import logging -from typing import Iterable, Iterator, List, Tuple +from typing import Iterator, List, Tuple from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.constants import LOGLEVEL_VVVV from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist From 154659cd0d0049ba7be1911af9a7add6ba3e5fa8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 21 Aug 2022 04:58:54 +0900 Subject: [PATCH 159/181] Fix: typo for cache sqlite schema version --- volatility3/framework/automagic/symbol_cache.py | 6 +++--- volatility3/framework/constants/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 558bfb2f1..ab19965c7 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -171,11 +171,11 @@ class SqliteCache(CacheManagerInterface): database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( - f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCEMA_VERSION})') + f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})') schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() if not schema_version: - database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCEMA_VERSION})') - elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCEMA_VERSION: + database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCHEMA_VERSION})') + elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCHEMA_VERSION: # All good, so pass and move on pass else: diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3b499adea..af3f7c0a0 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -76,7 +76,7 @@ MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") """Default location to record information about available identifiers""" -CACHE_SQLITE_SCEMA_VERSION = 1 +CACHE_SQLITE_SCHEMA_VERSION = 1 """Version for the sqlite3 cache schema""" BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" From 3e071b563d03d69cc06042eb05dfd2136cc49b2e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 21 Aug 2022 05:02:34 +0900 Subject: [PATCH 160/181] Fix: typo for symbol table --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index ab19965c7..a24dc3fd0 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -196,7 +196,7 @@ class SqliteCache(CacheManagerInterface): If multiple locations exist for an identifier, the last found is returned Args: - identifier: string that uniquely identifies a particular symbolt table + identifier: string that uniquely identifies a particular symbol table operating_system: optional string to restrict identifiers to just those for a particular operating system Returns: From ed8d240a7cf1b7d3b39bc467af8ec67d4c8ac190 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 24 Aug 2022 10:24:14 +0300 Subject: [PATCH 161/181] return given layer by base --- volatility3/framework/automagic/windows.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index f5dd720d6..08a5027d1 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -214,6 +214,9 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): context.config[interfaces.configuration.path_join( config_path, "page_map_offset")] = base_layer.metadata['page_map_offset'] layer = layer_type(context, config_path = config_path, name = new_layer_name, metadata = {'os': 'Windows'}) + page_map_offset = context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] + vollog.debug(f"DTB was given to as by base layer: {hex(page_map_offset)}") + return layer # Self Referential finder for description, tests, sections in cls.test_sets: From 253c4b5bb1cc7411255277639a866f8b0c9f87ac Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 24 Aug 2022 10:26:04 +0300 Subject: [PATCH 162/181] typo --- volatility3/framework/automagic/windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 08a5027d1..aaef3e820 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -215,7 +215,7 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): config_path, "page_map_offset")] = base_layer.metadata['page_map_offset'] layer = layer_type(context, config_path = config_path, name = new_layer_name, metadata = {'os': 'Windows'}) page_map_offset = context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] - vollog.debug(f"DTB was given to as by base layer: {hex(page_map_offset)}") + vollog.debug(f"DTB was given to us by base layer: {hex(page_map_offset)}") return layer # Self Referential finder From 9ca83763ba7e1b1012c09af4fb0f416a0b6de7cf Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Tue, 10 May 2022 09:17:03 -0500 Subject: [PATCH 163/181] refs #713 add a vad.get_size() method and fix several off-by-one issues with calculating vad size --- volatility3/framework/plugins/windows/malfind.py | 2 +- .../framework/plugins/windows/skeleton_key_check.py | 2 +- volatility3/framework/plugins/windows/vadinfo.py | 7 ++++--- volatility3/framework/plugins/windows/vadyarascan.py | 4 +--- .../framework/symbols/windows/extensions/__init__.py | 12 ++++++++---- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 700ced8ee..e63b81fb2 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -56,7 +56,7 @@ class Malfind(interfaces.plugins.PluginInterface): all_zero_page = b"\x00" * CHUNK_SIZE offset = 0 - vad_length = vad.get_end() - vad.get_start() + vad_length = vad.get_size() while offset < vad_length: next_addr = vad.get_start() + offset diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index 4a1b48c9a..cb1dd06c6 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -262,7 +262,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): if isinstance(filename, str) and filename.lower().endswith("cryptdll.dll"): base = vad.get_start() - return base, vad.get_end() - base + return base, vad.get_size() return None, None diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index e357b150a..50a69f8fb 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -132,7 +132,7 @@ class VadInfo(interfaces.plugins.PluginInterface): vollog.debug("Unable to find the starting/ending VPN member") return None - if 0 < maxsize < (vad_end - vad_start): + if 0 < maxsize < vad.get_size(): vollog.debug(f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit") return None @@ -151,8 +151,9 @@ class VadInfo(interfaces.plugins.PluginInterface): file_handle = open_method(file_name) chunk_size = 1024 * 1024 * 10 offset = vad_start - while offset < vad_end: - to_read = min(chunk_size, vad_end - offset) + vad_size = vad.get_size() + while offset < vad_start + vad_size: + to_read = min(chunk_size, vad_start + vad_size - offset) data = proc_layer.read(offset, to_read, pad = True) if not data: break diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 06a87d003..3954288eb 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -82,9 +82,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """ vad_root = task.get_vad_root() for vad in vad_root.traverse(): - end = vad.get_end() - start = vad.get_start() - yield (start, end - start) + yield (vad.get_start(), vad.get_size()) def run(self): return renderers.TreeGrid([('Offset', format_hints.Hex), ('PID', int), ('Rule', str), ('Component', str), diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index fe32a0322..bf44d1368 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -197,8 +197,8 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the parent member") - def get_start(self): - """Get the VAD's starting virtual address.""" + def get_start(self) -> int: + """Get the VAD's starting virtual address. This is the first accessible byte in the range.""" if self.has_member("StartingVpn"): @@ -216,8 +216,8 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the starting VPN member") - def get_end(self): - """Get the VAD's ending virtual address.""" + def get_end(self) -> int: + """Get the VAD's ending virtual address. This is the last accessible byte in the range.""" if self.has_member("EndingVpn"): @@ -234,6 +234,10 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the ending VPN member") + def get_size(self) -> int: + """Get the size of the VAD region. The OS ensures page granularity.""" + return (self.get_end() - self.get_start()) + 1 + def get_commit_charge(self): """Get the VAD's commit charge (number of committed pages)""" From 8bbcb51bcb3c27c7871dc6629d50570dc866e6bd Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 25 Aug 2022 01:47:19 +0900 Subject: [PATCH 164/181] Remove: return syntax --- volatility3/framework/plugins/windows/netstat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 4d6ec5f62..93ac3af93 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -434,7 +434,6 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) except exceptions.VolatilityException: vollog.error("Unable to locate symbols for the memory image's tcpip module") - return for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): From a5fe38339a038852cdff47acb0d4942e98fdaefd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Aug 2022 21:15:50 +0100 Subject: [PATCH 165/181] Core: Allow for deprecation of constants gracefully --- volatility3/framework/__init__.py | 2 +- volatility3/framework/automagic/linux.py | 4 ++- volatility3/framework/automagic/mac.py | 4 ++- .../framework/automagic/symbol_cache.py | 3 +- .../framework/automagic/symbol_finder.py | 4 ++- volatility3/framework/constants/__init__.py | 29 ++++++++++++++----- volatility3/framework/plugins/isfinfo.py | 6 ++-- .../framework/symbols/windows/pdbutil.py | 3 +- 8 files changed, 40 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 176eb2242..9b11143b2 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 6, 0) +required_python_version = (3, 7, 0) if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): raise RuntimeError( diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2c152996d..9bb2dae9b 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,6 +3,7 @@ # import logging +import os from typing import Optional, Tuple, Type from volatility3.framework import constants, interfaces @@ -40,7 +41,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - linux_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + linux_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( operating_system = 'linux') # If we have no banners, don't bother scanning if not linux_banners: diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 246462878..9bb3ad5f0 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -3,6 +3,7 @@ # import logging +import os import struct from typing import Optional @@ -42,7 +43,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - mac_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + mac_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( operating_system = 'mac') # If we have no banners, don't bother scanning if not mac_banners: diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 558bfb2f1..d69009721 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -388,7 +388,8 @@ class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._cache = SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + self._cache = SqliteCache(identifiers_path) def __call__(self, context, config_path, configurable, progress_callback = None): """Runs the automagic over the configurable.""" diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index a9221a7cc..7a197dffc 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -3,6 +3,7 @@ # import logging +import os from typing import Any, Callable, Iterable, List, Optional, Tuple from volatility3.framework import constants, interfaces, layers @@ -40,7 +41,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): """Creates a cached copy of the results, but only it's been requested.""" if not self._banners: - cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + cache = symbol_cache.SqliteCache(identifiers_path) self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system) return self._banners diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3b499adea..1f646416b 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,6 +9,7 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys +import warnings from typing import Callable, Optional import volatility3.framework.constants.linux @@ -67,13 +68,7 @@ if sys.platform == 'win32': CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") os.makedirs(CACHE_PATH, exist_ok = True) -LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") -"""Default location to record information about available linux banners""" - -MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") -"""Default location to record information about available mac banners""" - -IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") +IDENTIFIERS_FILENAME = "identifier.cache" """Default location to record information about available identifiers""" CACHE_SQLITE_SCEMA_VERSION = 1 @@ -107,3 +102,23 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" + +### +# DEPRECATED VALUES +### + +_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, 'linux_banners.cache') +"""This value is deprecated and is no longer used within volatility""" + +_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, 'mac_banners.cache') +"""This value is deprecated and is no longer used within volatility""" + +_deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) +"""This value is deprecated in favour of CACHE_PATH joined to IDENTIFIER_FILENAME""" + + +def __getattr__(name): + deprecated_tag = '_deprecated_' + if name in [x[len(deprecated_tag):] for x in globals() if x.startswith(deprecated_tag)]: + warnings.warn(f"{name} is deprecated", FutureWarning) + return globals()[f"{deprecated_tag}{name}"] diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 6b13f10b6..efffa9b87 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -109,7 +109,8 @@ class IsfInfo(plugins.PluginInterface): num_enums = len(data.get('enums', [])) num_bases = len(data.get('base_types', [])) - identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + identifier_cache = symbol_cache.SqliteCache(identifiers_path) identifier = identifier_cache.get_identifier(location = entry) if identifier: identifier = identifier.decode('utf-8', errors = 'replace') @@ -120,7 +121,8 @@ class IsfInfo(plugins.PluginInterface): vollog.warning(f"Invalid ISF: {entry}") yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) else: - cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + cache = symbol_cache.SqliteCache(identifiers_path) valid = 'Unknown' for identifier, location in cache.get_identifier_dictionary().items(): num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 430ad6a30..079b0e826 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -80,7 +80,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Required version of SQLiteCache not found") return None - value = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).find_location( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + value = symbol_cache.SqliteCache(identifiers_path).find_location( symbol_cache.WindowsIdentifier.generate(pdb_name.strip('\x00'), guid.upper(), age), 'windows') if value: From a337ec732a6feaf70032c405a48f1f3ceae39ae5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Aug 2022 21:20:56 +0100 Subject: [PATCH 166/181] Test: Update build tests to new minimum python version --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cf70b66cd..2d3729981 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.7' - name: Install dependencies run: | From d7301d653fca9c1195f83642c7133514c1f6a9a7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 25 Aug 2022 10:55:58 +0100 Subject: [PATCH 167/181] Core: Additional updates with the bump to python 3.7.0 Kindly pointed out by @digitalisx --- README.md | 2 +- setup.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 348121e44..502e26f10 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/setup.py b/setup.py index f6bb687f2..bce21ca66 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,10 @@ from volatility3.framework import constants with open("README.md", "r", encoding = "utf-8") as fh: long_description = fh.read() + def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -19,6 +20,7 @@ def get_install_requires(): requirements.append(stripped_line) return requirements + setuptools.setup(name = "volatility3", description = "Memory forensics framework", version = constants.PACKAGE_VERSION, @@ -34,7 +36,7 @@ setuptools.setup(name = "volatility3", "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires = '>=3.6.0', + python_requires = '>=3.7.0', include_package_data = True, exclude_package_data = { '': ['development', 'development.*'], From e8b4944f9a61e0c833354d8765174576069f48c4 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 27 Aug 2022 01:06:06 +0900 Subject: [PATCH 168/181] Fix: typo for simple-plugin.rst --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index e2143f1b7..c4908caf3 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -9,7 +9,7 @@ of a normal plugin, and reuses other plugins appropriately. .. note:: This document will not include the complete code necessary for a - working plugin (such as imports, etc) since it's designed to focus on the necessary componets for writing a plugin. + working plugin (such as imports, etc) since it's designed to focus on the necessary components for writing a plugin. For complete and functioning plugins, the ``framework/plugins`` directory should be consulted. Inherit from PluginInterface From 1f1355711d08e5e62b27156e5d186e3ed59366b2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 30 Aug 2022 10:54:26 +0100 Subject: [PATCH 169/181] Windows: Fix faulty pdbutil API Commit 5bc517aa appears to have been a broken merge that removed some of the changes made to the pdbutil API unintentionally. This was kindly pointed out in PR #822 by @digitalisx. --- .../framework/symbols/windows/pdbutil.py | 48 ++++++++++++++++--- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 430ad6a30..137d5f4a2 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -13,7 +13,7 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Union from urllib import parse, request from volatility3 import symbols -from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework import constants, contexts, exceptions, interfaces from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.configuration.requirements import SymbolTableRequirement @@ -344,12 +344,46 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - return cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) + module_name = guid["pdb_name"].strip('.pdb') + + symbol_table_name = cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) + + new_module_name = None + if create_module: + new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], + symbol_table_name = symbol_table_name) + new_module_name = new_module.name + + return new_module_name, symbol_table_name + + @classmethod + def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: + """Creates a module in the specified layer_name based on a pdb name. + + Searches the memory section of the loaded module for its PDB GUID + and loads the associated symbol table into the symbol space. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + config_path: The config path where to find symbol files + layer_name: The name of the layer on which to operate + module_offset: This memory dump's module image offset + module_size: The size of the module for this dump + + Returns: + The name of the constructed and loaded symbol table + """ + + module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size, create_module = True) + + return module_name class PdbSignatureScanner(interfaces.layers.ScannerInterface): From 4ed534bc8411408194399dc9698cd688a8d6cf44 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 2 Sep 2022 15:48:46 +0900 Subject: [PATCH 170/181] Fix: typo for yapf style file --- .style.yapf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.style.yapf b/.style.yapf index 8159be910..3f154e07b 100644 --- a/.style.yapf +++ b/.style.yapf @@ -107,7 +107,7 @@ each_dict_entry_on_separate_line=True i18n_comment= # The i18n function call names. The presence of this function stops -# reformattting on that line, because the string it has cannot be moved +# reformatting on that line, because the string it has cannot be moved # away from the i18n comment. i18n_function_call= From a49e7cfeca434e622d68f14d3d9fd567c7d450e6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 02:43:05 +0900 Subject: [PATCH 171/181] Fix: duplicate comments --- volatility3/framework/plugins/windows/cachedump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index ddfa856b9..f77c6257b 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -46,7 +46,7 @@ class Cachedump(interfaces.plugins.PluginInterface): rc4 = ARC4.new(rc4key) data = rc4.encrypt(edata) # lgtm [py/weak-cryptographic-algorithm] else: - # based on Based on code from http://lab.mediaservice.net/code/cachedump.rb + # Based on code from http://lab.mediaservice.net/code/cachedump.rb aes = AES.new(nlkm[16:32], AES.MODE_CBC, ch) data = b"" for i in range(0, len(edata), 16): From 3da028c7346d34cea11198dd897cf817d1e8f621 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 02:54:26 +0900 Subject: [PATCH 172/181] Remove: unused module --- volatility3/framework/plugins/windows/ldrmodules.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 284d1afc2..ba8d049a6 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -1,5 +1,4 @@ -from volatility3.framework import interfaces, constants -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed From 9e578e66da923121c44b8940aa1c0c691352f616 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 02:59:26 +0900 Subject: [PATCH 173/181] Remove: duplicate paragraph --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 96f222187..2a37fd0ed 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -31,7 +31,7 @@ If you make any Additions available to others, such as by providing copies of th - You are responsible to ensure you have rights in Additions necessary to comply with this section. Contributing -If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution." +If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution." Trademarks This license grants you no rights to any trademarks or service marks. From 626e352b18c9288b70dcf1cebb615a8b03379989 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 03:15:28 +0900 Subject: [PATCH 174/181] Add: api changes description for 2.3.1 version --- API_CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/API_CHANGES.md b/API_CHANGES.md index 4d8733286..f74f754f9 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,10 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.3.1 +===== +Update in the windows `_EPROCESS.owning_process` method for support Windows Vista and later versions. + 2.3.0 ===== Add in `child_template` to template class From 97638ffc0dd05c587d031303f431f646ca3752f8 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 12 Sep 2022 16:35:31 +0300 Subject: [PATCH 175/181] fix lineterminator --- volatility3/cli/text_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ecb5179e0..623153fae 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -224,7 +224,7 @@ class CSVRenderer(CLIRenderer): # Ignore the type because namedtuples don't realize they have accessible attributes header_list.append(f"{column.name}") - writer = csv.DictWriter(outfd, header_list) + writer = csv.DictWriter(outfd, header_list, lineterminator='\n') writer.writeheader() def visitor(node: interfaces.renderers.TreeNode, accumulator): From ee3895867f3c124f3aa80c5e2f4add5e02ada33b Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Wed, 21 Sep 2022 13:40:28 -0500 Subject: [PATCH 176/181] refs #713 bump VERSION_MINOR to 4 --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/malfind.py | 2 +- volatility3/framework/plugins/windows/skeleton_key_check.py | 2 +- volatility3/framework/plugins/windows/vadinfo.py | 2 +- volatility3/framework/plugins/windows/vadyarascan.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 6eec88d26..0e661a474 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 3 # Number of changes that only add to the interface +VERSION_MINOR = 4 # Number of changes that only add to the interface VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index e63b81fb2..9b5fab3f5 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -17,7 +17,7 @@ vollog = logging.getLogger(__name__) class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) @classmethod def get_requirements(cls): diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index cb1dd06c6..f6f41864a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -41,7 +41,7 @@ vollog = logging.getLogger(__name__) class Skeleton_Key_Check(interfaces.plugins.PluginInterface): """ Looks for signs of Skeleton Key malware """ - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) @classmethod def get_requirements(cls): diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 50a69f8fb..d3997c8c8 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -33,7 +33,7 @@ winnt_protections = { class VadInfo(interfaces.plugins.PluginInterface): """Lists process memory ranges.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) _version = (2, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 3954288eb..b71e2f605 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -17,7 +17,7 @@ vollog = logging.getLogger(__name__) class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) _version = (1, 0, 0) @classmethod From 941b5ff9c1aef35d1c06e665e5a119a2a82ba79e Mon Sep 17 00:00:00 2001 From: ikelos Date: Wed, 21 Sep 2022 20:19:42 +0100 Subject: [PATCH 177/181] Update volatility3/framework/constants/__init__.py Yep, quite right Co-authored-by: Donghyun Kim --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 0e661a474..00ae15f4e 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 4 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 4985dcd9a3ddff808004da71d636611f5956385b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Sep 2022 20:53:41 +0100 Subject: [PATCH 178/181] Windows: When constructing a buffer, manually dereference onto the native layer --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index fe32a0322..e290ef52d 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -488,7 +488,7 @@ class UNICODE_STRING(objects.StructType): # We manually construct an object rather than casting a dereferenced pointer in case # the buffer length is 0 and the pointer is a NULL pointer return self._context.object(self.vol.type_name.split(constants.BANG)[0] + constants.BANG + 'string', - layer_name = self.Buffer.vol.layer_name, + layer_name = self.Buffer.vol.native_layer_name, offset = self.Buffer, max_length = self.Length, errors = 'replace', encoding = 'utf16') From e5d4e599d3ea1b71853c530f82662e4d8d6c88bf Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Wed, 21 Sep 2022 14:55:17 -0500 Subject: [PATCH 179/181] refs #713 update API_CHANGES.md --- API_CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/API_CHANGES.md b/API_CHANGES.md index 4d8733286..a541e9619 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,10 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.4.0 +===== +Add a `get_size()` method to Windows VAD structures and fix several off-by-one issues when calculating VAD sizes. + 2.3.0 ===== Add in `child_template` to template class From 7529c7b246734ae02b51bb80dc22bbeddb819078 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Sep 2022 21:15:40 +0100 Subject: [PATCH 180/181] Core: Revert volatility 3.7 bump and associated features --- .github/workflows/test.yaml | 4 ++-- README.md | 2 +- setup.py | 6 ++---- volatility3/framework/__init__.py | 2 +- volatility3/framework/constants/__init__.py | 21 --------------------- 5 files changed, 6 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2d3729981..cf70b66cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.7 + - name: Set up Python 3.6 uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.6' - name: Install dependencies run: | diff --git a/README.md b/README.md index 502e26f10..348121e44 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/setup.py b/setup.py index bce21ca66..f6bb687f2 100644 --- a/setup.py +++ b/setup.py @@ -9,10 +9,9 @@ from volatility3.framework import constants with open("README.md", "r", encoding = "utf-8") as fh: long_description = fh.read() - def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -20,7 +19,6 @@ def get_install_requires(): requirements.append(stripped_line) return requirements - setuptools.setup(name = "volatility3", description = "Memory forensics framework", version = constants.PACKAGE_VERSION, @@ -36,7 +34,7 @@ setuptools.setup(name = "volatility3", "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires = '>=3.7.0', + python_requires = '>=3.6.0', include_package_data = True, exclude_package_data = { '': ['development', 'development.*'], diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 9b11143b2..176eb2242 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 7, 0) +required_python_version = (3, 6, 0) if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): raise RuntimeError( diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 1f646416b..d6fb96e1c 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,7 +9,6 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys -import warnings from typing import Callable, Optional import volatility3.framework.constants.linux @@ -102,23 +101,3 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" - -### -# DEPRECATED VALUES -### - -_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, 'linux_banners.cache') -"""This value is deprecated and is no longer used within volatility""" - -_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, 'mac_banners.cache') -"""This value is deprecated and is no longer used within volatility""" - -_deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) -"""This value is deprecated in favour of CACHE_PATH joined to IDENTIFIER_FILENAME""" - - -def __getattr__(name): - deprecated_tag = '_deprecated_' - if name in [x[len(deprecated_tag):] for x in globals() if x.startswith(deprecated_tag)]: - warnings.warn(f"{name} is deprecated", FutureWarning) - return globals()[f"{deprecated_tag}{name}"] From 949b15a36812d3f4ad33cf55e3e8fb387a0d7cee Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 23 Sep 2022 08:01:11 +0900 Subject: [PATCH 181/181] Fix: unsused module for objects initialize code --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 62e6de553..eedd22bb2 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -9,7 +9,7 @@ import struct from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload from volatility3.framework import constants, interfaces -from volatility3.framework.objects import templates, utility +from volatility3.framework.objects import templates vollog = logging.getLogger(__name__)