From 441fff0d10b2053a1ee52e21dfb15ab0eb06b1a2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 13 Oct 2021 17:00:47 +0100 Subject: [PATCH 001/404] Windows: Reduce DTB false positive rate --- volatility3/framework/automagic/windows.py | 49 ++++++++++++++-------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index eb63a75e5..ce09ca6a4 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -202,31 +202,44 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): for description, tests, sections in cls.test_sets: vollog.debug(description) # There is a very high chance that the DTB will live in these very narrow segments, assuming we couldn't find them previously - hits = context.layers[layer_name].scan(context, - PageMapScanner(tests = tests), - sections = sections, - progress_callback = progress_callback) + hits = base_layer.scan(context, + PageMapScanner(tests = tests), + sections = sections, + progress_callback = progress_callback) # Flatten the generator def sort_by_tests(x): + """Key used to sort by tests""" return tests.index(x[0]), x[1] + def get_max_pointer(page_table, test, ptr_size: int): + """Determines a pointer from a page_table""" + max_ptr = 0 + for index in range(0, len(page_table), ptr_size): + max_ptr = max(max_ptr, + struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] & test.mask) + return max_ptr + hits = sorted(list(hits), key = sort_by_tests) - if hits: - # TODO: Decide which to use if there are multiple options - test, page_map_offset = hits[0] - vollog.debug(f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}") - new_layer_name = context.layers.free_layer_name("IntelLayer") - config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name) - context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name - context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] = page_map_offset - # TODO: Need to determine the layer type (chances are high it's x64, hence this default) - layer = test.layer_type(context, - config_path = config_path, - name = new_layer_name, - metadata = {'os': 'Windows'}) - break + for test, page_map_offset in hits: + # Turn the page tables into integers and find the largest one + page_table = base_layer.read(page_map_offset, 0x1000) + ptr_size = struct.calcsize(test.ptr_struct) + max_pointer = get_max_pointer(page_table, test, ptr_size) + + if max_pointer <= base_layer.maximum_address: + vollog.debug(f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}") + new_layer_name = context.layers.free_layer_name("IntelLayer") + config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name) + context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name + context.config[ + interfaces.configuration.path_join(config_path, "page_map_offset")] = page_map_offset + layer = test.layer_type(context, + config_path = config_path, + name = new_layer_name, + metadata = {'os': 'Windows'}) + break if layer is not None and config_path: vollog.debug("DTB was found at: 0x{:0x}".format(context.config[interfaces.configuration.path_join( From 865cb92527915490cebe35fd00763eb4693ac13d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 17 Oct 2021 00:25:42 +0100 Subject: [PATCH 002/404] Windows: Check only valid page table entries --- volatility3/framework/automagic/windows.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index ce09ca6a4..f90f20b52 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -216,8 +216,9 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): """Determines a pointer from a page_table""" max_ptr = 0 for index in range(0, len(page_table), ptr_size): - max_ptr = max(max_ptr, - struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] & test.mask) + pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] + if pointer & 0x1: + max_ptr = max(max_ptr, pointer & test.mask) return max_ptr hits = sorted(list(hits), key = sort_by_tests) From fb6610ff48d165527ad99b6e127eb09b05194452 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 17 Oct 2021 00:36:16 +0100 Subject: [PATCH 003/404] Windows: Limit pointers to layer maximum address --- volatility3/framework/automagic/windows.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index f90f20b52..6968a1ac2 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -218,7 +218,7 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): for index in range(0, len(page_table), ptr_size): pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] if pointer & 0x1: - max_ptr = max(max_ptr, pointer & test.mask) + max_ptr = max(max_ptr, pointer & test.layer_type.maximum_address) return max_ptr hits = sorted(list(hits), key = sort_by_tests) @@ -241,6 +241,9 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): name = new_layer_name, metadata = {'os': 'Windows'}) break + else: + vollog.debug( + f"Max pointer for hit with test {test.__class__.__name__} not met: {hex(max_pointer)} > {hex(base_layer.maximum_address)}") if layer is not None and config_path: vollog.debug("DTB was found at: 0x{:0x}".format(context.config[interfaces.configuration.path_join( From 49fe653d9849c48d7179774705c768ec1011f680 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 17 Oct 2021 00:53:20 +0100 Subject: [PATCH 004/404] Windows: Max pointers at maximum layer address --- volatility3/framework/automagic/windows.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 6968a1ac2..308ead157 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -217,8 +217,9 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): max_ptr = 0 for index in range(0, len(page_table), ptr_size): pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] - if pointer & 0x1: - max_ptr = max(max_ptr, pointer & test.layer_type.maximum_address) + # Make sure the pointer is valid, ignore large pages which would require more calculation + if pointer & 0x1 and not pointer & 0x80: + max_ptr = max(max_ptr, pointer % test.layer_type.maximum_address) return max_ptr hits = sorted(list(hits), key = sort_by_tests) From 9b4324adaae666e3649c27e8efced45853fc26eb Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 17 Oct 2021 01:53:41 +0100 Subject: [PATCH 005/404] Windows: Stop looking for DTBs when a good one is found --- volatility3/framework/automagic/windows.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 308ead157..beda8d97b 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -219,7 +219,7 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] # Make sure the pointer is valid, ignore large pages which would require more calculation if pointer & 0x1 and not pointer & 0x80: - max_ptr = max(max_ptr, pointer % test.layer_type.maximum_address) + max_ptr = max(max_ptr, (pointer ^ (pointer & 0xfff)) % test.layer_type.maximum_address) return max_ptr hits = sorted(list(hits), key = sort_by_tests) @@ -245,6 +245,8 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): else: vollog.debug( f"Max pointer for hit with test {test.__class__.__name__} not met: {hex(max_pointer)} > {hex(base_layer.maximum_address)}") + if layer is not None and config_path: + break if layer is not None and config_path: vollog.debug("DTB was found at: 0x{:0x}".format(context.config[interfaces.configuration.path_join( From a1cbea27a84cdce776122fcd82a036a62f45c334 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 8 Nov 2021 22:56:06 +0000 Subject: [PATCH 006/404] Layers: Speed up qemu by not bisecting --- volatility3/framework/layers/qemu.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 4ce17bb63..1c5319dfd 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -1,7 +1,6 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import bisect import functools import json import math @@ -218,9 +217,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): of the starting data. It is the responsibility of the layer to turn the provided data chunk into the right portion of data necessary. """ - start_offset, _, start_mapped_offset, _ = self._segments[ - bisect.bisect_right(self._segments, (offset, 0xffffffffffffff,)) - 1] - if start_mapped_offset in self._compressed: + start_offset = offset ^ (offset & 0xfff) + if start_offset in self._compressed: data = (data * 0x1000) result = data[offset - start_offset:output_length + offset - start_offset] return result From ccd4c1bee0d09a8d464dd603330f68de049cc24e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 2 Dec 2021 17:38:31 +1100 Subject: [PATCH 007/404] Fixed exceptions.elf reference --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fbc02399f..0edd60608 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -108,7 +108,7 @@ class module(generic.GenericIntelProcess): "linux", "elf", native_types = None, - class_types = extensions.elf.class_types) + class_types = elf.class_types) syms = self._context.object( self.get_symbol_table().name + constants.BANG + "array", From e8fa6e1e7faac7ac1b70284a0d14e4a5ef30ceba Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 2 Dec 2021 18:36:04 +1100 Subject: [PATCH 008/404] Add the Linux mountinfo module. --- .../framework/plugins/linux/mountinfo.py | 227 ++++++++++++++++++ .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 166 ++++++++++++- 3 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 volatility3/framework/plugins/linux/mountinfo.py diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py new file mode 100644 index 000000000..be423a2f4 --- /dev/null +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -0,0 +1,227 @@ +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +# Author: Gustavo Moreira + +import logging +from collections import namedtuple +from typing import Tuple, List, Iterable, Union + +from volatility3.framework import renderers, interfaces, constants +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + +MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "mnt_root_path", "path_root", + "mnt_opts", "fields", "mnt_type", "devname", "sb_opts")) + +class MountInfo(plugins.PluginInterface): + """Lists mount points in processes mount namespaces""" + + _required_framework_version = (2, 0, 0) + + _version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name="kernel", description="Linux kernel", + architectures=["Intel32", "Intel64"]), + requirements.PluginRequirement(name="pslist", + plugin=pslist.PsList, version=(2, 0, 0)), + requirements.ListRequirement(name="pids", + description="Filter on specific process IDs.", + element_type=int, + optional=True), + requirements.BooleanRequirement(name="all-processes", + description="Shows information about mount points for each process mount " + "namespace. It could take a while depending on the number of processes " + "running. Note that if this argument is not specified it uses the root " + "mount namespace based on pid 1.", + optional=True, + default=False), + requirements.BooleanRequirement(name="mount-format", + description="Shows a brief summary of a process mount points information " + "with similar output format to the older /proc/[pid]/mounts or the " + "user-land command 'mount -l'.", + optional=True, + default=False), + ] + + def _get_symbol_fullname(self, symbol_basename: str) -> str: + """Given a short symbol or type name, it returns its full name""" + return self._vmlinux.symbol_table_name + constants.BANG + symbol_basename + + @classmethod + def _do_get_path(cls, mnt, fs_root) -> Union[None, str]: + """It mimics the Linux kernel prepend_path function.""" + vfsmnt = mnt.mnt + dentry = vfsmnt.get_mnt_root() + + path_reversed = [] + while dentry != fs_root.dentry or vfsmnt.vol.offset != fs_root.mnt: + if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): + parent = mnt.get_mnt_parent().dereference() + # Escaped? + if dentry != vfsmnt.get_mnt_root(): + return None + + # Global root? + if mnt.vol.offset != parent.vol.offset: + dentry = mnt.get_mnt_mountpoint() + mnt = parent + vfsmnt = mnt.mnt + continue + + return None + + parent = dentry.d_parent + dname = dentry.d_name.name_as_str() + path_reversed.append(dname.strip("/")) + dentry = parent + + path = "/" + "/".join(reversed(path_reversed)) + return path + + @classmethod + def get_mountinfo(cls, mnt, task) -> Union[None, Tuple[int, int, str, str, str, List[str], + List[str], str, str, List[str]]]: + """Extract various information about a mount point. + It mimics the Linux kernel show_mountinfo function. + """ + mnt_root = mnt.get_mnt_root() + if not mnt_root: + return None + + mnt_root_path = mnt_root.path() + superblock = mnt.get_mnt_sb() + + mnt_id: int = mnt.mnt_id + parent_id: int = mnt.mnt_parent.mnt_id + + st_dev = f"{superblock.major}:{superblock.minor}" + + path_root = cls._do_get_path(mnt, task.fs.root) + if path_root is None: + return None + + mnt_opts: List[str] = [] + mnt_opts.append(mnt.get_flags_access()) + mnt_opts.extend(mnt.get_flags_opts()) + + # Tagged fields + fields: List[str] = [] + if mnt.is_shared(): + fields.append(f"shared:{mnt.mnt_group_id}") + + if mnt.is_slave(): + master = mnt.mnt_master.mnt_group_id + fields.append(f"master:{master}") + dominating_id = mnt.get_dominating_id(task.fs.root) + if dominating_id and dominating_id != master: + fields.append(f"propagate_from:{dominating_id}") + + if mnt.is_unbindable(): + fields.append("unbindable") + + mnt_type = superblock.get_type() + + devname = mnt.get_devname() + if not devname: + devname = "none" + + sb_opts: List[str] = [] + sb_opts.append(superblock.get_flags_access()) + sb_opts.extend(superblock.get_flags_opts()) + + return MountInfoData(mnt_id, parent_id, st_dev, mnt_root_path, path_root, mnt_opts, fields, + mnt_type, devname, sb_opts) + + def _get_mnt_namespace_mountpoints(self, mnt_namespace): + mnt_type = self._get_symbol_fullname("mount") + if not self.context.symbol_space.has_type(mnt_type): + # Old kernels ~ 2.6 + mnt_type = self._get_symbol_fullname("vfsmount") + + for mount in mnt_namespace.list.to_list(mnt_type, "mnt_list"): + yield mount + + def _get_tasks_mountpoints(self, pids: Iterable[int]): + self._vmlinux = self.context.modules[self.config['kernel']] + + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks(self.context, self.config['kernel'], filter_func=pid_filter) + + seen_namespaces = set() + for task in tasks: + if not (task and task.fs and task.fs.root and task.nsproxy and task.nsproxy.mnt_ns): + # This task doesn't have all the information required + continue + + mnt_namespace = task.nsproxy.mnt_ns + mount_ns_id = mnt_namespace.get_inode() + + if self._show_mountpoints_per_namespace: + if mount_ns_id in seen_namespaces: + continue + else: + seen_namespaces.add(mount_ns_id) + + for mount in self._get_mnt_namespace_mountpoints(mnt_namespace): + yield task, mount, mount_ns_id + + def _generator(self): + pids = self.config.get('pids') + + for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(pids=pids): + mnt_info = self.get_mountinfo(mnt, task) + if mnt_info is None: + continue + + if self.config.get('mount-format'): + all_opts = set() + all_opts.update(mnt_info.mnt_opts) + all_opts.update(mnt_info.sb_opts) + all_opts_str = ",".join(all_opts) + + extra_fields_values = [mnt_info.devname, mnt_info.path_root, mnt_info.mnt_type, all_opts_str] + else: + mnt_opts_str = ",".join(mnt_info.mnt_opts) + fields_str = " ".join(mnt_info.fields) + sb_opts_str = ",".join(mnt_info.sb_opts) + + extra_fields_values = [mnt_info.mnt_id, mnt_info.parent_id, mnt_info.st_dev, mnt_info.mnt_root_path, + mnt_info.path_root, mnt_opts_str, fields_str, mnt_info.mnt_type, + mnt_info.devname, sb_opts_str] + + fields_values = [mnt_ns_id] + if not self._show_mountpoints_per_namespace: + fields_values.append(task.pid) + fields_values.extend(extra_fields_values) + + yield (0, fields_values) + + def run(self): + if self.config.get('all-processes') and self.config.get('pids'): + raise ValueError("Unable to use --all-processes and specified a pid") + + # When no arguments are specified, it displays the mountpoints per namespace + self._show_mountpoints_per_namespace = not any([self.config.get('pids'), self.config.get('all-processes')]) + + columns = [("MNT_NS_ID", int)] + if not self._show_mountpoints_per_namespace: + columns.append(("PID", int)) + + if self.config.get('mount-format'): + extra_columns = [("DEVNAME", str), ("PATH", str), ("FSTYPE", str), ("MNT_OPTS", str)] + else: + # /proc/[pid]/mountinfo output format + extra_columns = [("MOUNT ID", int), ("PARENT_ID", int), ("MAJOR:MINOR", str), ("ROOT", str), + ("MOUNT_POINT", str), ("MOUNT_OPTIONS", str), ("FIELDS", str), ("FSTYPE", str), + ("MOUNT_SRC", str), ("SB_OPTIONS", str)] + + columns.extend(extra_columns) + + return renderers.TreeGrid(columns, self._generator()) \ No newline at end of file diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 36e23a35d..5b0bfcf40 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -29,6 +29,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('files_struct', extensions.files_struct) self.set_type_class('vfsmount', extensions.vfsmount) self.set_type_class('kobject', extensions.kobject) + self.set_type_class('mnt_namespace', extensions.mnt_namespace) if 'module' in self.types: self.set_type_class('module', extensions.module) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0edd60608..3a237d280 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -246,6 +246,29 @@ class super_block(objects.StructType): # include/linux/kdev_t.h MINORBITS = 20 + # Superblock flags + SB_RDONLY = 1 # Mount read-only + SB_NOSUID = 2 # Ignore suid and sgid bits + SB_NODEV = 4 # Disallow access to device special files + SB_NOEXEC = 8 # Disallow program execution + SB_SYNCHRONOUS = 16 # Writes are synced at once + SB_MANDLOCK = 64 # Allow mandatory locks on an FS + SB_DIRSYNC = 128 # Directory modifications are synchronous + SB_NOATIME = 1024 # Do not update access times + SB_NODIRATIME = 2048 # Do not update directory access times + SB_SILENT = 32768 + SB_POSIXACL = (1 << 16) # VFS does not apply the umask + SB_KERNMOUNT = (1 << 22) # this is a kern_mount call + SB_I_VERSION = (1 << 23) # Update inode I_version field + SB_LAZYTIME = (1 << 25) # Update the on-disk [acm]times lazily + + SB_OPTS = { + SB_SYNCHRONOUS: "sync", + SB_DIRSYNC: "dirsync", + SB_MANDLOCK: "mand", + SB_LAZYTIME: "lazytime" + } + @property def major(self) -> int: return self.s_dev >> self.MINORBITS @@ -254,6 +277,20 @@ class super_block(objects.StructType): def minor(self) -> int: return self.s_dev & ((1 << self.MINORBITS) - 1) + def get_flags_access(self) -> str: + return 'ro' if self.s_flags & self.SB_RDONLY else 'rw' + + def get_flags_opts(self) -> Iterable[str]: + sb_opts = [self.SB_OPTS[sb_opt] for sb_opt in self.SB_OPTS if sb_opt & self.s_flags] + return sb_opts + + def get_type(self): + mnt_sb_type = utility.pointer_to_string(self.s_type.name, count=255) + if self.s_subtype: + mnt_sb_subtype = utility.pointer_to_string(self.s_subtype, count=255) + mnt_sb_type += "." + mnt_sb_subtype + return mnt_sb_type + class vm_area_struct(objects.StructType): perm_flags = { @@ -373,7 +410,44 @@ class qstr(objects.StructType): class dentry(objects.StructType): def path(self) -> str: - return self.d_name.name_as_str() + """ Based on __dentry_path Linux kernel function""" + reversed_path = [] + current_dentry = self + while not current_dentry.is_root(): + parent = current_dentry.d_parent + reversed_path.append(current_dentry.d_name.name_as_str()) + current_dentry = parent + return "/" + "/".join(reversed(reversed_path)) + + def is_root(self) -> bool: + return self.vol.offset == self.d_parent + + def is_subdir(self, old_dentry): + """Is this dentry a subdirectory of old_dentry? + + Returns true if this dentry is a subdirectory of the parent (at any depth). + Otherwise, it returns false. + """ + if self.vol.offset == old_dentry: + return True + + return self.d_ancestor(old_dentry) + + def d_ancestor(self, ancestor_dentry): + """Search for an ancestor + + Returns the ancestor dentry which is a child of "ancestor_dentry", + if "ancestor_dentry" is an ancestor of "child_dentry", else None. + """ + + current_dentry = self + while not current_dentry.is_root(): + if current_dentry.d_parent == ancestor_dentry.vol.offset: + return current_dentry + + current_dentry = current_dentry.d_parent + + return None class struct_file(objects.StructType): @@ -468,6 +542,27 @@ class files_struct(objects.StructType): class mount(objects.StructType): + MNT_NOSUID = 0x01 + MNT_NODEV = 0x02 + MNT_NOEXEC = 0x04 + MNT_NOATIME = 0x08 + MNT_NODIRATIME = 0x10 + MNT_RELATIME = 0x20 + MNT_READONLY = 0x40 + MNT_SHRINKABLE = 0x100 + MNT_WRITE_HOLD = 0x200 + MNT_SHARED = 0x1000 + MNT_UNBINDABLE = 0x2000 + + MNT_FLAGS = { + MNT_NOSUID: "nosuid", + MNT_NODEV: "nodev", + MNT_NOEXEC: "noexec", + MNT_NOATIME: "noatime", + MNT_NODIRATIME: "nodiratime", + MNT_RELATIME: "relatime", + } + def get_mnt_sb(self): if self.has_member("mnt"): return self.mnt.mnt_sb @@ -498,6 +593,66 @@ class mount(objects.StructType): def get_mnt_mountpoint(self): return self.mnt_mountpoint + def get_flags_access(self) -> str: + return "ro" if self.get_mnt_flags() & self.MNT_READONLY else "rw" + + def get_flags_opts(self) -> Iterable[str]: + flags = [self.MNT_FLAGS[mntflag] for mntflag in self.MNT_FLAGS if mntflag & self.get_mnt_flags()] + return flags + + def is_shared(self) -> bool: + return self.get_mnt_flags() & self.MNT_SHARED + + def is_unbindable(self) -> bool: + return self.get_mnt_flags() & self.MNT_UNBINDABLE + + def is_slave(self) -> bool: + return self.mnt_master and self.mnt_master.vol.offset != 0 + + def get_devname(self) -> str: + return utility.pointer_to_string(self.mnt_devname, count=255) + + def has_parent(self) -> bool: + return self.vol.offset != self.mnt_parent + + def get_dominating_id(self, root) -> int: + """Get ID of closest dominating peer group having a representative under the given root.""" + current_mnt = self.mnt_master + while current_mnt and current_mnt.vol.offset != 0: + peer = current_mnt.get_peer_under_root(self.mnt_ns, root) + if peer and peer.vol.offset != 0: + return peer.mnt_group_id + + current_mnt = current_mnt.mnt_master + return 0 + + def get_peer_under_root(self, ns, root): + current = self + while True: + if current.mnt_ns == ns and current.is_path_reachable(current.mnt.mnt_root, root): + return current + current = current.next_peer() + if current.vol.offset == self.vol.offset: + break + + return None + + def is_path_reachable(self, current_dentry, root): + """Return true if path is reachable + """ + current_mnt = self + while current_mnt.mnt.vol.offset != root.mnt and current_mnt.has_parent(): + current_dentry = current_mnt.mnt_mountpoint + current_mnt = current_mnt.mnt_parent + + return current_mnt.mnt.vol.offset == root.mnt and current_dentry.is_subdir(root.dentry) + + def next_peer(self): + table_name = self.vol.type_name.split(constants.BANG)[0] + mount_struct = "{0}{1}mount".format(table_name, constants.BANG) + offset = self._context.symbol_space.get_type(mount_struct).relative_child_offset("mnt_share") + + return self._context.object(mount_struct, self.vol.layer_name, offset=self.mnt_share.next.vol.offset - offset) class vfsmount(objects.StructType): @@ -539,3 +694,12 @@ class kobject(objects.StructType): ret = refcnt.refs.counter return ret + +class mnt_namespace(objects.StructType): + def get_inode(self): + if self.has_member("proc_inum"): + return self.proc_inum + elif self.ns.has_member("inum"): + return self.ns.inum + else: + raise AttributeError("Unable to find mnt_namespace inode") From fffb375909ecb7d4116e5d6807b25183c9691da5 Mon Sep 17 00:00:00 2001 From: Jan Date: Thu, 11 Mar 2021 18:21:30 +0100 Subject: [PATCH 009/404] adds netstat symbols to Win8x64 ISF files --- .../windows/netscan/netscan-win8-x64.json | 189 ++++++++++++++++- .../windows/netscan/netscan-win81-x64.json | 191 +++++++++++++++++- 2 files changed, 377 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win8-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win8-x64.json index 993c6a129..0bcf0cc6a 100644 --- a/volatility3/framework/symbols/windows/netscan/netscan-win8-x64.json +++ b/volatility3/framework/symbols/windows/netscan/netscan-win8-x64.json @@ -237,6 +237,16 @@ "kind": "base", "name": "unsigned be short" } + }, + "Next": { + "offset": 136, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } } }, "kind": "struct", @@ -256,7 +266,7 @@ } }, "CreateTime": { - "offset": 32, + "offset": 224, "type": { "kind": "union", "name": "_LARGE_INTEGER" @@ -290,6 +300,16 @@ "kind": "base", "name": "unsigned be short" } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } } }, "kind": "struct", @@ -503,6 +523,173 @@ }, "kind": "union", "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 4096 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 40 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 176, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 160, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 128 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 } }, "enums": { diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win81-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win81-x64.json index 80bea7838..6a4d1aee7 100644 --- a/volatility3/framework/symbols/windows/netscan/netscan-win81-x64.json +++ b/volatility3/framework/symbols/windows/netscan/netscan-win81-x64.json @@ -232,11 +232,21 @@ } }, "Port": { - "offset": 128, + "offset": 120, "type": { "kind": "base", "name": "unsigned be short" } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } } }, "kind": "struct", @@ -256,7 +266,7 @@ } }, "CreateTime": { - "offset": 32, + "offset": 224, "type": { "kind": "union", "name": "_LARGE_INTEGER" @@ -290,6 +300,16 @@ "kind": "base", "name": "unsigned be short" } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } } }, "kind": "struct", @@ -503,6 +523,173 @@ }, "kind": "union", "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 6144 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 40 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 176, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 160, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 128 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 } }, "enums": { From 90ad7b1609bd6017cd44a2d11c214bbe5014d407 Mon Sep 17 00:00:00 2001 From: Jan Date: Thu, 11 Mar 2021 19:05:48 +0100 Subject: [PATCH 010/404] reverts pointless createtime offset change --- .../framework/symbols/windows/netscan/netscan-win8-x64.json | 2 +- .../framework/symbols/windows/netscan/netscan-win81-x64.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win8-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win8-x64.json index 0bcf0cc6a..24873b078 100644 --- a/volatility3/framework/symbols/windows/netscan/netscan-win8-x64.json +++ b/volatility3/framework/symbols/windows/netscan/netscan-win8-x64.json @@ -266,7 +266,7 @@ } }, "CreateTime": { - "offset": 224, + "offset": 64, "type": { "kind": "union", "name": "_LARGE_INTEGER" diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win81-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win81-x64.json index 6a4d1aee7..d6c51d2db 100644 --- a/volatility3/framework/symbols/windows/netscan/netscan-win81-x64.json +++ b/volatility3/framework/symbols/windows/netscan/netscan-win81-x64.json @@ -266,7 +266,7 @@ } }, "CreateTime": { - "offset": 224, + "offset": 64, "type": { "kind": "union", "name": "_LARGE_INTEGER" From ee65793fd85469b6e969cc42f76959dd4f208f27 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 17 Mar 2021 19:24:20 +0100 Subject: [PATCH 011/404] adds new ISF for tcpip.sys 6.3.9600.19935 + distinguisher --- .../framework/plugins/windows/netscan.py | 102 ++- .../windows/netscan-win81-19935-x64.json | 723 ++++++++++++++++++ 2 files changed, 784 insertions(+), 41 deletions(-) create mode 100644 volatility3/framework/symbols/windows/netscan-win81-19935-x64.json diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 33b5a7fbc..a6eadd505 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -14,7 +14,7 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.framework.symbols.windows.extensions import network from volatility3.plugins import timeliner -from volatility3.plugins.windows import info, poolscanner +from volatility3.plugins.windows import info, poolscanner, verinfo vollog = logging.getLogger(__name__) @@ -141,49 +141,55 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # furthermore, it is easy to append new versions. if arch == "x86": version_dict = { - (6, 0, 6000): "netscan-vista-x86", - (6, 0, 6001): "netscan-vista-x86", - (6, 0, 6002): "netscan-vista-x86", - (6, 0, 6003): "netscan-vista-x86", - (6, 1, 7600): "netscan-win7-x86", - (6, 1, 7601): "netscan-win7-x86", - (6, 1, 8400): "netscan-win7-x86", - (6, 2, 9200): "netscan-win8-x86", - (6, 3, 9600): "netscan-win81-x86", - (10, 0, 10240): "netscan-win10-10240-x86", - (10, 0, 10586): "netscan-win10-10586-x86", - (10, 0, 14393): "netscan-win10-14393-x86", - (10, 0, 15063): "netscan-win10-15063-x86", - (10, 0, 16299): "netscan-win10-15063-x86", - (10, 0, 17134): "netscan-win10-17134-x86", - (10, 0, 17763): "netscan-win10-17134-x86", - (10, 0, 18362): "netscan-win10-17134-x86", - (10, 0, 18363): "netscan-win10-17134-x86" + (6, 0, 6000, 0): "netscan-vista-x86", + (6, 0, 6001, 0): "netscan-vista-x86", + (6, 0, 6002, 0): "netscan-vista-x86", + (6, 0, 6003, 0): "netscan-vista-x86", + (6, 1, 7600, 0): "netscan-win7-x86", + (6, 1, 7601, 0): "netscan-win7-x86", + (6, 1, 8400, 0): "netscan-win7-x86", + (6, 2, 9200, 0): "netscan-win8-x86", + (6, 3, 9600, 0): "netscan-win81-x86", + (10, 0, 10240, 0): "netscan-win10-10240-x86", + (10, 0, 10586, 0): "netscan-win10-10586-x86", + (10, 0, 14393, 0): "netscan-win10-14393-x86", + (10, 0, 15063, 0): "netscan-win10-15063-x86", + (10, 0, 16299, 0): "netscan-win10-15063-x86", + (10, 0, 17134, 0): "netscan-win10-17134-x86", + (10, 0, 17763, 0): "netscan-win10-17134-x86", + (10, 0, 18362, 0): "netscan-win10-17134-x86", + (10, 0, 18363, 0): "netscan-win10-17134-x86" } else: version_dict = { - (6, 0, 6000): "netscan-vista-x64", - (6, 0, 6001): "netscan-vista-sp12-x64", - (6, 0, 6002): "netscan-vista-sp12-x64", - (6, 0, 6003): "netscan-vista-sp12-x64", - (6, 1, 7600): "netscan-win7-x64", - (6, 1, 7601): "netscan-win7-x64", - (6, 1, 8400): "netscan-win7-x64", - (6, 2, 9200): "netscan-win8-x64", - (6, 3, 9600): "netscan-win81-x64", - (10, 0, 10240): "netscan-win10-x64", - (10, 0, 10586): "netscan-win10-x64", - (10, 0, 14393): "netscan-win10-x64", - (10, 0, 15063): "netscan-win10-15063-x64", - (10, 0, 16299): "netscan-win10-16299-x64", - (10, 0, 17134): "netscan-win10-17134-x64", - (10, 0, 17763): "netscan-win10-17763-x64", - (10, 0, 18362): "netscan-win10-18362-x64", - (10, 0, 18363): "netscan-win10-18363-x64", - (10, 0, 19041): "netscan-win10-19041-x64" + (6, 0, 6000, 0): "netscan-vista-x64", + (6, 0, 6001, 0): "netscan-vista-sp12-x64", + (6, 0, 6002, 0): "netscan-vista-sp12-x64", + (6, 0, 6003, 0): "netscan-vista-sp12-x64", + (6, 1, 7600, 0): "netscan-win7-x64", + (6, 1, 7601, 0): "netscan-win7-x64", + (6, 1, 8400, 0): "netscan-win7-x64", + (6, 2, 9200, 0): "netscan-win8-x64", + (6, 3, 9600, 0): "netscan-win81-x64", + (6, 3, 9600, 19935): "netscan-win81-19935-x64", + (10, 0, 10240, 0): "netscan-win10-x64", + (10, 0, 10586, 0): "netscan-win10-x64", + (10, 0, 14393, 0): "netscan-win10-x64", + (10, 0, 15063, 0): "netscan-win10-15063-x64", + (10, 0, 16299, 0): "netscan-win10-16299-x64", + (10, 0, 17134, 0): "netscan-win10-17134-x64", + (10, 0, 17763, 0): "netscan-win10-17763-x64", + (10, 0, 18362, 0): "netscan-win10-18362-x64", + (10, 0, 18363, 0): "netscan-win10-18363-x64", + (10, 0, 19041, 0): "netscan-win10-19041-x64" } - # special use case: Win10_18363 is not recognized by windows.info as 18363 + # we do not need to check for tcpip's specific FileVersion in every case + tcpip_mod_version = 0 # keep it 0 as a default + + # special use cases + + # Win10_18363 is not recognized by windows.info as 18363 # because all kernel file headers and debug structures report 18363 as # "10.0.18362.1198" with the last part being incremented. However, we can use # os_distinguisher to differentiate between 18362 and 18363 @@ -191,18 +197,30 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.debug("Detected 18363 data structures: working with 18363 symbol table.") vers_minor_version = 18363 + # Win 8 SP 1 also may have different structures based on specific tcpip.sys version + if (nt_major_version, nt_minor_version, vers_minor_version) == (6, 3, 9600): + vollog.debug("Requiring further version inspection due to OS version by checking tcpip.sys's FileVersion header") + physical_layer_name = context.layers[layer_name].config.get('memory_layer', None) + ver = verinfo.VerInfo.find_version_info(context, physical_layer_name, "tcpip.sys") + if ver: + tcpip_mod_version = ver[3] + vollog.debug("Determined tcpip.sys's FileVersion: {}".format(tcpip_mod_version)) + else: + vollog.debug("Could not determine tcpip.sys's FileVersion.") + # when determining the symbol file we have to consider the following cases: # the determined version's symbol file is found by intermed.create -> proceed # the determined version's symbol file is not found by intermed -> intermed will throw an exc and abort # the determined version has no mapped symbol file -> if win10 use latest, otherwise throw exc # windows version cannot be determined -> throw exc - filename = version_dict.get((nt_major_version, nt_minor_version, vers_minor_version)) + + filename = version_dict.get((nt_major_version, nt_minor_version, vers_minor_version, tcpip_mod_version)) if not filename: # no match on filename means that we possibly have a version newer than those listed here. # try to grab the latest supported version of the current image NT version. If that symbol # version does not work, support has to be added manually. current_versions = [ - key for key in list(version_dict.keys()) if key[0] == nt_major_version and key[1] == nt_minor_version + (nt_maj, nt_min, vers_min, tcpip_ver) for nt_maj, nt_min, vers_min, tcpip_ver in version_dict if nt_maj == nt_major_version and nt_min == nt_minor_version and tcpip_ver <= tcpip_mod_version ] current_versions.sort() @@ -210,7 +228,9 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): latest_version = current_versions[-1] filename = version_dict.get(latest_version) + vollog.debug(f"Unable to find exact matching symbol file, going with latest: {filename}") + else: raise NotImplementedError("This version of Windows is not supported: {}.{} {}.{}!".format( nt_major_version, nt_minor_version, vers.MajorVersion, vers_minor_version)) diff --git a/volatility3/framework/symbols/windows/netscan-win81-19935-x64.json b/volatility3/framework/symbols/windows/netscan-win81-19935-x64.json new file mode 100644 index 000000000..f75428613 --- /dev/null +++ b/volatility3/framework/symbols/windows/netscan-win81-19935-x64.json @@ -0,0 +1,723 @@ +{ + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned be short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "big" + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "symbols": {}, + "user_types": { + "_TCP_SYN_ENDPOINT": { + "fields": { + "Owner": { + "offset": 64, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SYN_OWNER" + } + } + }, + "CreateTime": { + "offset": 0, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 16, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "InetAF": { + "offset": 48, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "LocalPort": { + "offset": 100, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 102, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "LocalAddr": { + "offset": 56, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "RemoteAddress": { + "offset": 80, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 104 + }, + "_TCP_TIMEWAIT_ENDPOINT": { + "fields": { + "CreateTime": { + "offset": 0, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 0, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "InetAF": { + "offset": 24, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + } + }, + "LocalPort": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 50, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "LocalAddr": { + "offset": 56, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "RemoteAddress": { + "offset": 64, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 72 + }, + "_UDP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 88, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 96, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "InetAF": { + "offset": 32, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Port": { + "offset": 120, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } + } + }, + "kind": "struct", + "size": 130 + }, + "_TCP_LISTENER": { + "fields": { + "Owner": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 64, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 88, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + + } + }, + "InetAF": { + "offset": 96, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Port": { + "offset": 106, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } + } + }, + "kind": "struct", + "size": 108 + }, + "_TCP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 608, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 624, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 40, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "AddrInfo": { + "offset": 24, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ADDRINFO" + } + } + }, + "InetAF": { + "offset": 16, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "LocalPort": { + "offset": 112, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "State": { + "offset": 108, + "type": { + "kind": "enum", + "name": "TCPStateEnum" + } + } + }, + "kind": "struct", + "size": 608 + }, + "_LOCAL_ADDRESS": { + "fields": { + "pData": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_ADDRINFO": { + "fields": { + "Local": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "Remote": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_IN_ADDR": { + "fields": { + "addr4": { + "offset": 0, + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + }, + "addr6": { + "offset": 0, + "type": { + "count": 16, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + } + }, + "kind": "struct", + "size": 6 + }, + "_INETAF": { + "fields": { + "AddressFamily": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 26 + }, + "_SYN_OWNER": { + "fields": { + "Process": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 6144 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 216, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 200, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 128 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 + } + }, + "enums": { + "TCPStateEnum": { + "base": "long", + "constants": { + "CLOSED": 0, + "LISTENING": 1, + "SYN_SENT": 2, + "SYN_RCVD": 3, + "ESTABLISHED": 4, + "FIN_WAIT1": 5, + "FIN_WAIT2": 6, + "CLOSE_WAIT": 7, + "CLOSING": 8, + "LAST_ACK": 9, + "TIME_WAIT": 12, + "DELETE_TCB": 13 + }, + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "japhlange-by-hand", + "datetime": "2020-06-12T14:00:00" + }, + "format": "6.0.0" + } +} From 3aac7b228e5c3cf11a0ee17f11e07cda8f365d75 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 17 Mar 2021 21:57:41 +0100 Subject: [PATCH 012/404] adds a few clarifying comments and handles errors better --- volatility3/framework/plugins/windows/netscan.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index a6eadd505..88f2235dd 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -198,15 +198,19 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vers_minor_version = 18363 # Win 8 SP 1 also may have different structures based on specific tcpip.sys version + # the following is IntelLayer specific and might need to be adapted to other architectures. if (nt_major_version, nt_minor_version, vers_minor_version) == (6, 3, 9600): vollog.debug("Requiring further version inspection due to OS version by checking tcpip.sys's FileVersion header") physical_layer_name = context.layers[layer_name].config.get('memory_layer', None) - ver = verinfo.VerInfo.find_version_info(context, physical_layer_name, "tcpip.sys") - if ver: - tcpip_mod_version = ver[3] - vollog.debug("Determined tcpip.sys's FileVersion: {}".format(tcpip_mod_version)) + if physical_layer_name: + ver = verinfo.VerInfo.find_version_info(context, physical_layer_name, "tcpip.sys") + if ver: + tcpip_mod_version = ver[3] + vollog.debug("Determined tcpip.sys's FileVersion: {}".format(tcpip_mod_version)) + else: + vollog.debug("Could not determine tcpip.sys's FileVersion.") else: - vollog.debug("Could not determine tcpip.sys's FileVersion.") + vollog.debug("Unable to retrieve physical memory layer, skipping FileVersion check.") # when determining the symbol file we have to consider the following cases: # the determined version's symbol file is found by intermed.create -> proceed From 6cd2d940f3b5c8cddca25b6a080623932cadc646 Mon Sep 17 00:00:00 2001 From: Jan Date: Thu, 18 Mar 2021 14:21:48 +0100 Subject: [PATCH 013/404] adds proper version requirement for verinfo --- .../framework/plugins/windows/netscan.py | 1 + .../framework/plugins/windows/netstat.py | 4 +- .../framework/plugins/windows/verinfo.py | 204 ------------------ 3 files changed, 4 insertions(+), 205 deletions(-) delete mode 100644 volatility3/framework/plugins/windows/verinfo.py diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 88f2235dd..ea66a6b64 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -34,6 +34,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): component = poolscanner.PoolScanner, version = (1, 0, 0)), requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), + requirements.VersionRequirement(name = 'verinfo', component = verinfo.VerInfo, version = (1, 0, 0)), requirements.BooleanRequirement( name = 'include-corrupt', description = diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index e71683c10..486957565 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -12,7 +12,7 @@ from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.windows import pdbutil from volatility3.framework.symbols.windows.extensions import network from volatility3.plugins import timeliner -from volatility3.plugins.windows import netscan, modules +from volatility3.plugins.windows import netscan, modules, info, verinfo vollog = logging.getLogger(__name__) @@ -31,6 +31,8 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement(name = 'netscan', component = netscan.NetScan, version = (1, 0, 0)), requirements.VersionRequirement(name = 'modules', component = modules.Modules, version = (1, 0, 0)), requirements.VersionRequirement(name = 'pdbutil', component = pdbutil.PDBUtility, version = (1, 0, 0)), + requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), + requirements.VersionRequirement(name = 'verinfo', component = verinfo.VerInfo, version = (1, 0, 0)), requirements.BooleanRequirement( name = 'include-corrupt', description = diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py deleted file mode 100644 index 762832e39..000000000 --- a/volatility3/framework/plugins/windows/verinfo.py +++ /dev/null @@ -1,204 +0,0 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -import io -import logging -import struct -from typing import Generator, List, Tuple, Optional - -from volatility3.framework import exceptions, renderers, constants, interfaces -from volatility3.framework.configuration import requirements -from volatility3.framework.layers import scanners -from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, modules, dlllist - -vollog = logging.getLogger(__name__) - -try: - import pefile -except ImportError: - vollog.info("Python pefile module not found, plugin (and dependent plugins) not available") - raise - - -class VerInfo(interfaces.plugins.PluginInterface): - """Lists version information from PE files.""" - - _version = (1, 0, 0) - _required_framework_version = (2, 0, 0) - - @classmethod - def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - ## TODO: we might add a regex option on the name later, but otherwise we're good - ## TODO: and we don't want any CLI options from pslist, modules, or moddump - return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.PluginRequirement(name = 'modules', plugin = modules.Modules, version = (1, 0, 0)), - requirements.VersionRequirement(name = 'dlllist', component = dlllist.DllList, version = (2, 0, 0)), - requirements.BooleanRequirement(name = "extensive", - description = "Search physical layer for version information", - optional = True, - default = False), - ] - - @classmethod - def find_version_info(cls, context: interfaces.context.ContextInterface, layer_name: str, - filename: str) -> Optional[Tuple[int, int, int, int]]: - """Searches for an original filename, then tracks back to find the VS_VERSION_INFO and read the fixed - version information structure""" - premable_max_distance = 0x500 - filename = "OriginalFilename\x00" + filename - iterator = context.layers[layer_name].scan(context = context, - scanner = scanners.BytesScanner(bytes(filename, 'utf-16be'))) - for offset in iterator: - data = context.layers[layer_name].read(offset - premable_max_distance, premable_max_distance) - vs_ver_info = b"\xbd\x04\xef\xfe" - verinfo_offset = data.find(vs_ver_info) + len(vs_ver_info) - if verinfo_offset >= 0: - structure = ' Tuple[int, int, int, int]: - """Get File and Product version information from PE files. - - Args: - context: volatility context on which to operate - pe_table_name: name of the PE table - layer_name: name of the layer containing the PE file - base_address: base address of the PE (where MZ is found) - """ - - if layer_name is None: - raise TypeError("Layer must be a string not None") - - pe_data = io.BytesIO() - - dos_header = context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset = base_address, - layer_name = layer_name) - - for offset, data in dos_header.reconstruct(): - pe_data.seek(offset) - pe_data.write(data) - - pe = pefile.PE(data = pe_data.getvalue(), fast_load = True) - pe.parse_data_directories([pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]]) - - if isinstance(pe.VS_FIXEDFILEINFO, list): - # pefile >= 2018.8.8 (estimated) - version_struct = pe.VS_FIXEDFILEINFO[0] - else: - # pefile <= 2017.11.5 (estimated) - version_struct = pe.VS_FIXEDFILEINFO - - major = version_struct.ProductVersionMS >> 16 - minor = version_struct.ProductVersionMS & 0xFFFF - product = version_struct.ProductVersionLS >> 16 - build = version_struct.ProductVersionLS & 0xFFFF - - pe_data.close() - - return major, minor, product, build - - def _generator(self, procs: Generator[interfaces.objects.ObjectInterface, None, None], - mods: Generator[interfaces.objects.ObjectInterface, None, None], session_layers: Generator[str, None, - None]): - """Generates a list of PE file version info for processes, dlls, and - modules. - - Args: - procs: of processes - mods: of modules - session_layers: of layers in the session to be checked - """ - kernel = self.context.modules[self.config['kernel']] - - pe_table_name = intermed.IntermediateSymbolTable.create(self.context, - self.config_path, - "windows", - "pe", - class_types = pe.class_types) - - # TODO: Fix this so it works with more than just intel layers - physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) - - for mod in mods: - try: - BaseDllName = mod.BaseDllName.get_string() - except exceptions.InvalidAddressException: - BaseDllName = renderers.UnreadableValue() - - session_layer_name = modules.Modules.find_session_layer(self.context, session_layers, mod.DllBase) - try: - (major, minor, product, build) = self.get_version_information(self._context, pe_table_name, - session_layer_name, mod.DllBase) - except (exceptions.InvalidAddressException, TypeError, AttributeError): - (major, minor, product, build) = [renderers.UnreadableValue()] * 4 - if (not isinstance(BaseDllName, renderers.UnreadableValue) and physical_layer_name is not None - and self.config['extensive']): - result = self.find_version_info(self._context, physical_layer_name, BaseDllName) - if result is not None: - (major, minor, product, build) = result - - # the pid and process are not applicable for kernel modules - yield (0, (renderers.NotApplicableValue(), renderers.NotApplicableValue(), format_hints.Hex(mod.DllBase), - BaseDllName, major, minor, product, build)) - - # now go through the process and dll lists - for proc in procs: - proc_id = "Unknown" - try: - proc_id = proc.UniqueProcessId - proc_layer_name = proc.add_process_layer() - except exceptions.InvalidAddressException as excp: - vollog.debug("Process {}: invalid address {} in layer {}".format(proc_id, excp.invalid_address, - excp.layer_name)) - continue - - for entry in proc.load_order_modules(): - - try: - BaseDllName = entry.BaseDllName.get_string() - except exceptions.InvalidAddressException: - BaseDllName = renderers.UnreadableValue() - - try: - DllBase = format_hints.Hex(entry.DllBase) - except exceptions.InvalidAddressException: - DllBase = renderers.UnreadableValue() - - try: - (major, minor, product, build) = self.get_version_information(self._context, pe_table_name, - proc_layer_name, entry.DllBase) - except (exceptions.InvalidAddressException, ValueError, AttributeError): - (major, minor, product, build) = [renderers.UnreadableValue()] * 4 - - yield (0, (proc.UniqueProcessId, - proc.ImageFileName.cast("string", - max_length = proc.ImageFileName.vol.count, - errors = "replace"), DllBase, BaseDllName, major, minor, product, - build)) - - def run(self): - kernel = self.context.modules[self.config['kernel']] - - procs = pslist.PsList.list_processes(self.context, kernel.layer_name, kernel.symbol_table_name) - - mods = modules.Modules.list_modules(self.context, kernel.layer_name, kernel.symbol_table_name) - - # populate the session layers for kernel modules - session_layers = modules.Modules.get_session_layers(self.context, kernel.layer_name, kernel.symbol_table_name) - - return renderers.TreeGrid([("PID", int), ("Process", str), ("Base", format_hints.Hex), ("Name", str), - ("Major", int), ("Minor", int), ("Product", int), ("Build", int)], - self._generator(procs, mods, session_layers)) From be4d014f1a9ca0293784da542bcfef6c7de18161 Mon Sep 17 00:00:00 2001 From: Jan Date: Thu, 18 Mar 2021 18:35:06 +0100 Subject: [PATCH 014/404] improves decision making when to inspect driver pe header based on version_dict --- volatility3/framework/plugins/windows/netscan.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index ea66a6b64..16301bdd8 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -198,10 +198,12 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.debug("Detected 18363 data structures: working with 18363 symbol table.") vers_minor_version = 18363 - # Win 8 SP 1 also may have different structures based on specific tcpip.sys version - # the following is IntelLayer specific and might need to be adapted to other architectures. - if (nt_major_version, nt_minor_version, vers_minor_version) == (6, 3, 9600): + # we need to define additional version numbers (which are then found via tcpip.sys's FileVersion header) in case there is + # ambiguity _within_ an OS version. If such a version number (last number of the tuple) is defined for the current OS + # we need to inspect tcpip.sys's headers to see if we can grab the precise version + if [ (a,b,c,d) for a, b, c, d in version_dict if (a,b,c) == (nt_major_version, nt_minor_version, vers_minor_version) and d != 0]: vollog.debug("Requiring further version inspection due to OS version by checking tcpip.sys's FileVersion header") + # the following is IntelLayer specific and might need to be adapted to other architectures. physical_layer_name = context.layers[layer_name].config.get('memory_layer', None) if physical_layer_name: ver = verinfo.VerInfo.find_version_info(context, physical_layer_name, "tcpip.sys") From 125658ee0da64716d7e4f1d0b24bbac237f78ef7 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 6 Dec 2021 21:09:39 +0100 Subject: [PATCH 015/404] re-adds accidentally deleted file --- .../framework/plugins/windows/verinfo.py | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 volatility3/framework/plugins/windows/verinfo.py diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py new file mode 100644 index 000000000..70d473d2e --- /dev/null +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -0,0 +1,205 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import io +import logging +import struct +from typing import Generator, List, Tuple, Optional + +from volatility3.framework import exceptions, renderers, constants, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, modules, dlllist + +vollog = logging.getLogger(__name__) + +try: + import pefile +except ImportError: + vollog.info("Python pefile module not found, plugin (and dependent plugins) not available") + raise + + +class VerInfo(interfaces.plugins.PluginInterface): + """Lists version information from PE files.""" + + _version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + ## TODO: we might add a regex option on the name later, but otherwise we're good + ## TODO: and we don't want any CLI options from pslist, modules, or moddump + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.PluginRequirement(name = 'modules', plugin = modules.Modules, version = (1, 0, 0)), + requirements.VersionRequirement(name = 'dlllist', component = dlllist.DllList, version = (2, 0, 0)), + requirements.BooleanRequirement(name = "extensive", + description = "Search physical layer for version information", + optional = True, + default = False), + ] + + @classmethod + def find_version_info(cls, context: interfaces.context.ContextInterface, layer_name: str, + filename: str) -> Optional[Tuple[int, int, int, int]]: + """Searches for an original filename, then tracks back to find the VS_VERSION_INFO and read the fixed + version information structure""" + premable_max_distance = 0x500 + filename = "OriginalFilename\x00" + filename + iterator = context.layers[layer_name].scan(context = context, + scanner = scanners.BytesScanner(bytes(filename, 'utf-16be'))) + for offset in iterator: + data = context.layers[layer_name].read(offset - premable_max_distance, premable_max_distance) + vs_ver_info = b"\xbd\x04\xef\xfe" + verinfo_offset = data.find(vs_ver_info) + len(vs_ver_info) + if verinfo_offset >= 0: + structure = ' Tuple[int, int, int, int]: + """Get File and Product version information from PE files. + + Args: + context: volatility context on which to operate + pe_table_name: name of the PE table + layer_name: name of the layer containing the PE file + base_address: base address of the PE (where MZ is found) + """ + + if layer_name is None: + raise TypeError("Layer must be a string not None") + + pe_data = io.BytesIO() + + dos_header = context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset = base_address, + layer_name = layer_name) + + for offset, data in dos_header.reconstruct(): + pe_data.seek(offset) + pe_data.write(data) + + pe = pefile.PE(data = pe_data.getvalue(), fast_load = True) + pe.parse_data_directories([pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]]) + + if isinstance(pe.VS_FIXEDFILEINFO, list): + # pefile >= 2018.8.8 (estimated) + version_struct = pe.VS_FIXEDFILEINFO[0] + else: + # pefile <= 2017.11.5 (estimated) + version_struct = pe.VS_FIXEDFILEINFO + + major = version_struct.ProductVersionMS >> 16 + minor = version_struct.ProductVersionMS & 0xFFFF + product = version_struct.ProductVersionLS >> 16 + build = version_struct.ProductVersionLS & 0xFFFF + + pe_data.close() + + return major, minor, product, build + + def _generator(self, procs: Generator[interfaces.objects.ObjectInterface, None, None], + mods: Generator[interfaces.objects.ObjectInterface, None, None], session_layers: Generator[str, None, + None]): + """Generates a list of PE file version info for processes, dlls, and + modules. + + Args: + procs: of processes + mods: of modules + session_layers: of layers in the session to be checked + """ + kernel = self.context.modules[self.config['kernel']] + + pe_table_name = intermed.IntermediateSymbolTable.create(self.context, + self.config_path, + "windows", + "pe", + class_types = pe.class_types) + + # TODO: Fix this so it works with more than just intel layers + physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) + + for mod in mods: + try: + BaseDllName = mod.BaseDllName.get_string() + except exceptions.InvalidAddressException: + BaseDllName = renderers.UnreadableValue() + + session_layer_name = modules.Modules.find_session_layer(self.context, session_layers, mod.DllBase) + try: + (major, minor, product, build) = self.get_version_information(self._context, pe_table_name, + session_layer_name, mod.DllBase) + except (exceptions.InvalidAddressException, TypeError, AttributeError): + (major, minor, product, build) = [renderers.UnreadableValue()] * 4 + if (not isinstance(BaseDllName, renderers.UnreadableValue) and physical_layer_name is not None + and self.config['extensive']): + result = self.find_version_info(self._context, physical_layer_name, BaseDllName) + if result is not None: + (major, minor, product, build) = result + + # the pid and process are not applicable for kernel modules + yield (0, (renderers.NotApplicableValue(), renderers.NotApplicableValue(), format_hints.Hex(mod.DllBase), + BaseDllName, major, minor, product, build)) + + # now go through the process and dll lists + for proc in procs: + proc_id = "Unknown" + try: + proc_id = proc.UniqueProcessId + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException as excp: + vollog.debug("Process {}: invalid address {} in layer {}".format(proc_id, excp.invalid_address, + excp.layer_name)) + continue + + for entry in proc.load_order_modules(): + + try: + BaseDllName = entry.BaseDllName.get_string() + except exceptions.InvalidAddressException: + BaseDllName = renderers.UnreadableValue() + + try: + DllBase = format_hints.Hex(entry.DllBase) + except exceptions.InvalidAddressException: + DllBase = renderers.UnreadableValue() + + try: + (major, minor, product, build) = self.get_version_information(self._context, pe_table_name, + proc_layer_name, entry.DllBase) + except (exceptions.InvalidAddressException, ValueError, AttributeError): + (major, minor, product, build) = [renderers.UnreadableValue()] * 4 + + yield (0, (proc.UniqueProcessId, + proc.ImageFileName.cast("string", + max_length = proc.ImageFileName.vol.count, + errors = "replace"), DllBase, BaseDllName, major, minor, product, + build)) + + def run(self): + kernel = self.context.modules[self.config['kernel']] + + procs = pslist.PsList.list_processes(self.context, kernel.layer_name, kernel.symbol_table_name) + + mods = modules.Modules.list_modules(self.context, kernel.layer_name, kernel.symbol_table_name) + + # populate the session layers for kernel modules + session_layers = modules.Modules.get_session_layers(self.context, kernel.layer_name, kernel.symbol_table_name) + + return renderers.TreeGrid([("PID", int), ("Process", str), ("Base", format_hints.Hex), ("Name", str), + ("Major", int), ("Minor", int), ("Product", int), ("Build", int)], + self._generator(procs, mods, session_layers)) + From 581251cb0547b118ef01687aed81aab0a2f75f6d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 8 Dec 2021 23:34:23 +0000 Subject: [PATCH 016/404] Linux: Fix long standing typo (thanks to @gcmoreira) --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fbc02399f..0edd60608 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -108,7 +108,7 @@ class module(generic.GenericIntelProcess): "linux", "elf", native_types = None, - class_types = extensions.elf.class_types) + class_types = elf.class_types) syms = self._context.object( self.get_symbol_table().name + constants.BANG + "array", From 17e0b04cb35f44612ecfc07ef7d154eec013c5be Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 12:20:51 +1100 Subject: [PATCH 017/404] undefined glob module --- volatility3/cli/volshell/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index f2375c774..94f735ba0 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -7,6 +7,7 @@ import json import logging import os import sys +import glob import volatility3.plugins import volatility3.symbols From 36ac92c11c868490891a53b285b4a9f9093a18d6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 12:27:53 +1100 Subject: [PATCH 018/404] undefined `layers` module --- volatility3/framework/automagic/symbol_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 72dc071a5..143abd02e 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -5,7 +5,7 @@ import logging from typing import Any, Iterable, List, Tuple, Type, Optional, Callable -from volatility3.framework import interfaces, constants +from volatility3.framework import interfaces, constants, layers from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners From 8e7aff4ad8e14b77d6b792fbc5581499ee89640b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 13:09:41 +1100 Subject: [PATCH 019/404] `not in` please --- development/stock-linux-json.py | 2 +- volatility3/cli/__init__.py | 2 +- volatility3/framework/layers/resources.py | 2 +- volatility3/framework/plugins/linux/check_creds.py | 2 +- volatility3/framework/plugins/mac/list_files.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/development/stock-linux-json.py b/development/stock-linux-json.py index c863d41e4..877f78e1c 100644 --- a/development/stock-linux-json.py +++ b/development/stock-linux-json.py @@ -87,7 +87,7 @@ class Downloader: output_filename = 'unknown-kernel.json' for named_file in named_files: prefix = '--system-map' - if not 'System' in named_files[named_file]: + if 'System' not in named_files[named_file]: prefix = '--elf' output_filename = './' + '-'.join((named_file.split('/')[-1]).split('-')[2:])[:-4] + '.json.xz' args += [prefix, named_files[named_file]] diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 608fdf79c..2c5e13211 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -543,7 +543,7 @@ class CommandLine: self._file = io.open(fd, mode = 'w+b') CLIFileHandler.__init__(self, filename) for item in dir(self._file): - if not item.startswith('_') and not item in ['closed', 'close', 'mode', 'name']: + if not item.startswith('_') and item not in ('closed', 'close', 'mode', 'name'): setattr(self, item, getattr(self._file, item)) def __getattr__(self, item): diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index b6ef1b6ba..7ace25290 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -82,7 +82,7 @@ class ResourceAccessor(object): """Determines whether a URLs contents should be cached""" parsed_url = urllib.parse.urlparse(url) - return self._enable_cache and not parsed_url.scheme in self._non_cached_schemes() + return self._enable_cache and parsed_url.scheme not in self._non_cached_schemes() @staticmethod def _non_cached_schemes() -> List[str]: diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 613469eed..9bc1a067d 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -44,7 +44,7 @@ class Check_creds(interfaces.plugins.PluginInterface): cred_addr = task.cred.dereference().vol.offset - if not cred_addr in creds: + if cred_addr not in creds: creds[cred_addr] = [] creds[cred_addr].append(task.pid) diff --git a/volatility3/framework/plugins/mac/list_files.py b/volatility3/framework/plugins/mac/list_files.py index 19f28b18f..8bae986b7 100644 --- a/volatility3/framework/plugins/mac/list_files.py +++ b/volatility3/framework/plugins/mac/list_files.py @@ -72,7 +72,7 @@ class List_Files(plugins.PluginInterface): key = vnode.vol.offset added = False - if not key in loop_vnodes: + if key not in loop_vnodes: # We can't do anything with a no-name vnode v_name = cls._vnode_name(vnode) if v_name is None: @@ -108,7 +108,7 @@ class List_Files(plugins.PluginInterface): added = True parent = cls._get_parent(context, vnode) - while parent and not parent in loop_vnodes: + while parent and parent not in loop_vnodes: if not cls._walk_vnode(context, parent, loop_vnodes): break From e1942976bf95d0952d020dccd504c43751dee8a9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 12:24:22 +1100 Subject: [PATCH 020/404] wrong comparison with None --- volatility3/framework/plugins/linux/check_syscall.py | 2 +- volatility3/framework/symbols/windows/extensions/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 729a0bec6..87d252cd5 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -152,7 +152,7 @@ class Check_syscall(plugins.PluginInterface): except exceptions.SymbolError: ia32_symbol = None - if ia32_symbol != None: + if ia32_symbol is not None: ia32_info = self._get_table_info(vmlinux, "ia32_sys_call_table", ptr_sz) tables.append(("32bit", ia32_info)) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ae7c45d04..55f237581 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -84,7 +84,7 @@ class MMVAD_SHORT(objects.StructType): if tag in ["VadS", "VadF"]: target = "_MMVAD_SHORT" - elif tag != None and tag.startswith("Vad"): + elif tag is not None and tag.startswith("Vad"): target = "_MMVAD" elif depth == 0: # the root node at depth 0 is allowed to not have a tag @@ -651,7 +651,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): except AttributeError: return False - return value != 0 and value != None + return not value def get_vad_root(self): From d2ad867d1e422579e3ef024d342bfeb0658a054f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 13:20:43 +1100 Subject: [PATCH 021/404] my mistake, it should be negated twice to get True when is valid --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 55f237581..7c931f148 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -651,7 +651,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): except AttributeError: return False - return not value + return not not value def get_vad_root(self): From c5c1f355ef7dd20a024a5358db7f7ae758187af7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Dec 2021 09:51:25 +1100 Subject: [PATCH 022/404] Changing `not not` for a more explicit if statement --- volatility3/framework/symbols/windows/extensions/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 7c931f148..84c47e733 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -651,7 +651,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): except AttributeError: return False - return not not value + if value: + return True + + return False def get_vad_root(self): From a1d145c34f60db7f68c3949ba642466502403cd2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Dec 2021 22:10:52 +1100 Subject: [PATCH 023/404] Cleaning space issues --- .../framework/plugins/linux/mountinfo.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index be423a2f4..10869a45b 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -14,14 +14,14 @@ from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) -MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "mnt_root_path", "path_root", +MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "mnt_root_path", "path_root", "mnt_opts", "fields", "mnt_type", "devname", "sb_opts")) class MountInfo(plugins.PluginInterface): """Lists mount points in processes mount namespaces""" _required_framework_version = (2, 0, 0) - + _version = (2, 0, 0) @classmethod @@ -86,7 +86,7 @@ class MountInfo(plugins.PluginInterface): return path @classmethod - def get_mountinfo(cls, mnt, task) -> Union[None, Tuple[int, int, str, str, str, List[str], + def get_mountinfo(cls, mnt, task) -> Union[None, Tuple[int, int, str, str, str, List[str], List[str], str, str, List[str]]]: """Extract various information about a mount point. It mimics the Linux kernel show_mountinfo function. @@ -136,7 +136,7 @@ class MountInfo(plugins.PluginInterface): sb_opts.append(superblock.get_flags_access()) sb_opts.extend(superblock.get_flags_opts()) - return MountInfoData(mnt_id, parent_id, st_dev, mnt_root_path, path_root, mnt_opts, fields, + return MountInfoData(mnt_id, parent_id, st_dev, mnt_root_path, path_root, mnt_opts, fields, mnt_type, devname, sb_opts) def _get_mnt_namespace_mountpoints(self, mnt_namespace): @@ -147,13 +147,13 @@ class MountInfo(plugins.PluginInterface): for mount in mnt_namespace.list.to_list(mnt_type, "mnt_list"): yield mount - + def _get_tasks_mountpoints(self, pids: Iterable[int]): self._vmlinux = self.context.modules[self.config['kernel']] - + pid_filter = pslist.PsList.create_pid_filter(pids) tasks = pslist.PsList.list_tasks(self.context, self.config['kernel'], filter_func=pid_filter) - + seen_namespaces = set() for task in tasks: if not (task and task.fs and task.fs.root and task.nsproxy and task.nsproxy.mnt_ns): @@ -162,13 +162,13 @@ class MountInfo(plugins.PluginInterface): mnt_namespace = task.nsproxy.mnt_ns mount_ns_id = mnt_namespace.get_inode() - + if self._show_mountpoints_per_namespace: if mount_ns_id in seen_namespaces: continue else: seen_namespaces.add(mount_ns_id) - + for mount in self._get_mnt_namespace_mountpoints(mnt_namespace): yield task, mount, mount_ns_id @@ -186,34 +186,34 @@ class MountInfo(plugins.PluginInterface): all_opts.update(mnt_info.sb_opts) all_opts_str = ",".join(all_opts) - extra_fields_values = [mnt_info.devname, mnt_info.path_root, mnt_info.mnt_type, all_opts_str] + extra_fields_values = [mnt_info.devname, mnt_info.path_root, mnt_info.mnt_type, all_opts_str] else: mnt_opts_str = ",".join(mnt_info.mnt_opts) fields_str = " ".join(mnt_info.fields) sb_opts_str = ",".join(mnt_info.sb_opts) - extra_fields_values = [mnt_info.mnt_id, mnt_info.parent_id, mnt_info.st_dev, mnt_info.mnt_root_path, - mnt_info.path_root, mnt_opts_str, fields_str, mnt_info.mnt_type, + extra_fields_values = [mnt_info.mnt_id, mnt_info.parent_id, mnt_info.st_dev, mnt_info.mnt_root_path, + mnt_info.path_root, mnt_opts_str, fields_str, mnt_info.mnt_type, mnt_info.devname, sb_opts_str] - + fields_values = [mnt_ns_id] if not self._show_mountpoints_per_namespace: fields_values.append(task.pid) fields_values.extend(extra_fields_values) - + yield (0, fields_values) def run(self): if self.config.get('all-processes') and self.config.get('pids'): raise ValueError("Unable to use --all-processes and specified a pid") - + # When no arguments are specified, it displays the mountpoints per namespace self._show_mountpoints_per_namespace = not any([self.config.get('pids'), self.config.get('all-processes')]) columns = [("MNT_NS_ID", int)] if not self._show_mountpoints_per_namespace: columns.append(("PID", int)) - + if self.config.get('mount-format'): extra_columns = [("DEVNAME", str), ("PATH", str), ("FSTYPE", str), ("MNT_OPTS", str)] else: @@ -221,7 +221,7 @@ class MountInfo(plugins.PluginInterface): extra_columns = [("MOUNT ID", int), ("PARENT_ID", int), ("MAJOR:MINOR", str), ("ROOT", str), ("MOUNT_POINT", str), ("MOUNT_OPTIONS", str), ("FIELDS", str), ("FSTYPE", str), ("MOUNT_SRC", str), ("SB_OPTIONS", str)] - + columns.extend(extra_columns) - return renderers.TreeGrid(columns, self._generator()) \ No newline at end of file + return renderers.TreeGrid(columns, self._generator()) From fcf9983d5127370b67b350e72fb7e8363ff3b283 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Dec 2021 13:48:46 +1100 Subject: [PATCH 024/404] Removed --all-process and added --mntns. --- .../framework/plugins/linux/mountinfo.py | 87 +++++++++---------- .../symbols/linux/extensions/__init__.py | 20 +++-- 2 files changed, 55 insertions(+), 52 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 10869a45b..a3006d47e 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -7,7 +7,7 @@ import logging from collections import namedtuple from typing import Tuple, List, Iterable, Union -from volatility3.framework import renderers, interfaces, constants +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.plugins.linux import pslist @@ -18,11 +18,11 @@ MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "m "mnt_opts", "fields", "mnt_type", "devname", "sb_opts")) class MountInfo(plugins.PluginInterface): - """Lists mount points in processes mount namespaces""" + """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -35,25 +35,19 @@ class MountInfo(plugins.PluginInterface): description="Filter on specific process IDs.", element_type=int, optional=True), - requirements.BooleanRequirement(name="all-processes", - description="Shows information about mount points for each process mount " - "namespace. It could take a while depending on the number of processes " - "running. Note that if this argument is not specified it uses the root " - "mount namespace based on pid 1.", - optional=True, - default=False), + requirements.ListRequirement(name="mntns", + description="Filter results by mount namespace. " + "Otherwise, all of them are shown.", + element_type=int, + optional=True), requirements.BooleanRequirement(name="mount-format", - description="Shows a brief summary of a process mount points information " + description="Shows a brief summary of the mount points information " "with similar output format to the older /proc/[pid]/mounts or the " "user-land command 'mount -l'.", optional=True, default=False), ] - def _get_symbol_fullname(self, symbol_basename: str) -> str: - """Given a short symbol or type name, it returns its full name""" - return self._vmlinux.symbol_table_name + constants.BANG + symbol_basename - @classmethod def _do_get_path(cls, mnt, fs_root) -> Union[None, str]: """It mimics the Linux kernel prepend_path function.""" @@ -139,21 +133,7 @@ class MountInfo(plugins.PluginInterface): return MountInfoData(mnt_id, parent_id, st_dev, mnt_root_path, path_root, mnt_opts, fields, mnt_type, devname, sb_opts) - def _get_mnt_namespace_mountpoints(self, mnt_namespace): - mnt_type = self._get_symbol_fullname("mount") - if not self.context.symbol_space.has_type(mnt_type): - # Old kernels ~ 2.6 - mnt_type = self._get_symbol_fullname("vfsmount") - - for mount in mnt_namespace.list.to_list(mnt_type, "mnt_list"): - yield mount - - def _get_tasks_mountpoints(self, pids: Iterable[int]): - self._vmlinux = self.context.modules[self.config['kernel']] - - pid_filter = pslist.PsList.create_pid_filter(pids) - tasks = pslist.PsList.list_tasks(self.context, self.config['kernel'], filter_func=pid_filter) - + def _get_tasks_mountpoints(self, tasks: Iterable[interfaces.objects.ObjectInterface], per_namespace: bool): seen_namespaces = set() for task in tasks: if not (task and task.fs and task.fs.root and task.nsproxy and task.nsproxy.mnt_ns): @@ -161,26 +141,33 @@ class MountInfo(plugins.PluginInterface): continue mnt_namespace = task.nsproxy.mnt_ns - mount_ns_id = mnt_namespace.get_inode() + mnt_ns_id = mnt_namespace.get_inode() - if self._show_mountpoints_per_namespace: - if mount_ns_id in seen_namespaces: + if per_namespace: + if mnt_ns_id in seen_namespaces: continue else: - seen_namespaces.add(mount_ns_id) + seen_namespaces.add(mnt_ns_id) - for mount in self._get_mnt_namespace_mountpoints(mnt_namespace): - yield task, mount, mount_ns_id + for mount in mnt_namespace.get_mount_points(): + yield task, mount, mnt_ns_id - def _generator(self): - pids = self.config.get('pids') + def _generator( + self, + tasks: Iterable[interfaces.objects.ObjectInterface], + mnt_ns_ids: List[int], + mount_format: bool, + per_namespace: bool) -> Iterable[Tuple[int, Tuple]]: + + for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, per_namespace): + if mnt_ns_ids and mnt_ns_id not in mnt_ns_ids: + continue - for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(pids=pids): mnt_info = self.get_mountinfo(mnt, task) if mnt_info is None: continue - if self.config.get('mount-format'): + if mount_format: all_opts = set() all_opts.update(mnt_info.mnt_opts) all_opts.update(mnt_info.sb_opts) @@ -197,22 +184,28 @@ class MountInfo(plugins.PluginInterface): mnt_info.devname, sb_opts_str] fields_values = [mnt_ns_id] - if not self._show_mountpoints_per_namespace: + if not per_namespace: fields_values.append(task.pid) fields_values.extend(extra_fields_values) yield (0, fields_values) def run(self): - if self.config.get('all-processes') and self.config.get('pids'): - raise ValueError("Unable to use --all-processes and specified a pid") + pids = self.config.get('pids') + mount_ns_ids = self.config.get('mntns') + mount_format = self.config.get('mount-format') - # When no arguments are specified, it displays the mountpoints per namespace - self._show_mountpoints_per_namespace = not any([self.config.get('pids'), self.config.get('all-processes')]) + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks(self.context, self.config['kernel'], filter_func=pid_filter) columns = [("MNT_NS_ID", int)] - if not self._show_mountpoints_per_namespace: + # The PID column does not make sense when a PID filter is not specified. In that case, the default behavior is + # to displays the mountpoints per namespace. + if pids: columns.append(("PID", int)) + per_namespace = False + else: + per_namespace = True if self.config.get('mount-format'): extra_columns = [("DEVNAME", str), ("PATH", str), ("FSTYPE", str), ("MNT_OPTS", str)] @@ -224,4 +217,4 @@ class MountInfo(plugins.PluginInterface): columns.extend(extra_columns) - return renderers.TreeGrid(columns, self._generator()) + return renderers.TreeGrid(columns, self._generator(tasks, mount_ns_ids, mount_format, per_namespace)) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3a237d280..23967dd25 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -555,12 +555,12 @@ class mount(objects.StructType): MNT_UNBINDABLE = 0x2000 MNT_FLAGS = { - MNT_NOSUID: "nosuid", - MNT_NODEV: "nodev", - MNT_NOEXEC: "noexec", - MNT_NOATIME: "noatime", + MNT_NOSUID: "nosuid", + MNT_NODEV: "nodev", + MNT_NOEXEC: "noexec", + MNT_NOATIME: "noatime", MNT_NODIRATIME: "nodiratime", - MNT_RELATIME: "relatime", + MNT_RELATIME: "relatime", } def get_mnt_sb(self): @@ -703,3 +703,13 @@ class mnt_namespace(objects.StructType): return self.ns.inum else: raise AttributeError("Unable to find mnt_namespace inode") + + def get_mount_points(self): + table_name = self.vol.type_name.split(constants.BANG)[0] + mnt_type = table_name + constants.BANG + "mount" + if not self._context.symbol_space.has_type(mnt_type): + # Old kernels ~ 2.6 + mnt_type = table_name + constants.BANG + "vfsmount" + + for mount in self.list.to_list(mnt_type, "mnt_list"): + yield mount From f23b288ffcc15234eb25e88ce82df86f7285e0ea Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Dec 2021 17:52:07 +1100 Subject: [PATCH 025/404] Bump framework and plugin required minor version --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/linux/mountinfo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 23598837b..09a2e4820 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 0 # Number of changes that only add to the interface +VERSION_MINOR = 1 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index a3006d47e..6c0f8bcc3 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -20,7 +20,7 @@ MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "m class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 1, 0) _version = (1, 0, 0) From 16cb449a77555fa7514b968352771f882589f37c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Dec 2021 14:23:14 +1100 Subject: [PATCH 026/404] Added changes to the Linux pslist and pstree plugins to be able to show user threads. --- .../framework/constants/linux/__init__.py | 2 + volatility3/framework/plugins/linux/pslist.py | 125 +++++++++++++++--- volatility3/framework/plugins/linux/pstree.py | 79 +++++++---- .../framework/symbols/linux/__init__.py | 9 ++ 4 files changed, 171 insertions(+), 44 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index c25ea0e2f..276c6015f 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -11,3 +11,5 @@ KERNEL_NAME = "__kernel__" # arch/x86/include/asm/page_types.h PAGE_SHIFT = 12 """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" + +PF_KTHREAD = 0x00200000 # I'm a kernel thread diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 5672bb56e..516c74305 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -1,11 +1,12 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Callable, Iterable, List, Any +from typing import Callable, Iterable, List, Any, Tuple -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, constants from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility +from volatility3.framework.symbols import linux class PsList(interfaces.plugins.PluginInterface): @@ -13,7 +14,7 @@ class PsList(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -23,7 +24,15 @@ class PsList(interfaces.plugins.PluginInterface): requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, - optional = True) + optional = True), + requirements.BooleanRequirement(name="threads", + description="Include user threads", + optional=True, + default=False), + requirements.BooleanRequirement(name="decorate_comm", + description="Show `user threads` comm in curly brackets, and `kernel threads` comm in square brackets", + optional=True, + default=False), ] @classmethod @@ -48,31 +57,88 @@ class PsList(interfaces.plugins.PluginInterface): else: return lambda _: False - def _generator(self): + @staticmethod + def task_is_kernel_thread(task: interfaces.objects.ObjectInterface) -> bool: + return (task.flags & constants.PF_KTHREAD) != 0 + + @staticmethod + def task_is_thread_group_leader(task: interfaces.objects.ObjectInterface) -> bool: + return task.tgid == task.pid + + @staticmethod + def task_is_user_thread(task: interfaces.objects.ObjectInterface) -> bool: + return task.tgid != task.pid + + def _get_task_fields( + self, + task: interfaces.objects.ObjectInterface, + decorate_comm: bool = False) -> Tuple[int, int, int, str]: + """Extract the fields needed for the final output + + Args: + task: A task object from where to get the fields. + decorate_comm: If True, it decorates the comm string of + - User threads: in curly brackets, + - Kernel threads: in square brackets + Defaults to False. + Returns: + A tuple with the fields to show in the plugin output. + """ + pid = task.tgid + tid = task.pid + ppid = task.parent.tgid if task.parent else 0 + name = utility.array_to_string(task.comm) + if decorate_comm: + if self.task_is_kernel_thread(task): + name = f"[{name}]" + elif self.task_is_user_thread(task): + name = f"{{{name}}}" + + task_fields = (pid, tid, ppid, name) + return task_fields + + def _generator( + self, + pid_filter: Callable[[Any], bool], + include_threads: bool = False, + decorate_comm: bool = False): + """Generates the tasks list. + + Args: + pid_filter: A function which takes a process object and returns True if the process should be ignored/filtered + include_threads: If True, the output will also show the user threads + If False, only the thread group leaders will be shown + Defaults to False. + decorate_comm: If True, it decorates the comm string of + - User threads: in curly brackets, + - Kernel threads: in square brackets + Defaults to False. + Yields: + Each rows + """ for task in self.list_tasks(self.context, self.config['kernel'], - filter_func = self.create_pid_filter(self.config.get('pid', None))): - pid = task.pid - ppid = 0 - if task.parent: - ppid = task.parent.pid - name = utility.array_to_string(task.comm) - yield (0, (pid, ppid, name)) + pid_filter, + include_threads): + row = self._get_task_fields(task, decorate_comm) + yield (0, row) @classmethod def list_tasks( cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False) -> Iterable[interfaces.objects.ObjectInterface]: + filter_func: Callable[[int], bool] = lambda _: False, + include_threads: bool = False) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer. Args: context: The context to retrieve required elements (layers, symbol tables) from vmlinux_module_name: The name of the kernel module on which to operate - + filter_func: A function which takes a process object and returns True if the process should be ignored/filtered + include_threads: If True, it will also return user threads. Yields: - Process objects + Task objects """ vmlinux = context.modules[vmlinux_module_name] @@ -80,8 +146,29 @@ class PsList(interfaces.plugins.PluginInterface): # Note that the init_task itself is not yielded, since "ps" also never shows it. for task in init_task.tasks: - if not filter_func(task): - yield task + if filter_func(task): + continue + + task_threads = [] + current_task = None + next_task = task.thread_group.next + while current_task is None or current_task.vol.offset != task.vol.offset: + current_task = linux.LinuxUtilities.container_of(next_task, "task_struct", "thread_group", vmlinux) + if cls.task_is_thread_group_leader(current_task): + # Making sure the first task yielded is the Task Group Leader + yield current_task + elif include_threads: + task_threads.append(current_task) + next_task = current_task.thread_group.next + + # yield the other task threads + yield from task_threads def run(self): - return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str)], self._generator()) + pids = self.config.get('pid') + include_threads = self.config.get('threads') + decorate_comm = self.config.get('decorate_comm') + filter_func = self.create_pid_filter(pids) + + columns = [("PID", int), ("TID", int), ("PPID", int), ("COMM", str)] + return renderers.TreeGrid(columns, self._generator(filter_func, include_threads, decorate_comm)) diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 3b95a344c..174e94ec4 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -1,8 +1,7 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework.objects import utility from volatility3.plugins.linux import pslist @@ -12,44 +11,74 @@ class PsTree(pslist.PsList): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._processes = {} + self._tasks = {} self._levels = {} self._children = {} - def find_level(self, pid): - """Finds how deep the pid is in the processes list.""" - seen = set([]) - seen.add(pid) - level = 0 - proc = self._processes.get(pid, None) - while proc is not None and proc.parent != 0 and proc.parent.pid not in seen: - ppid = int(proc.parent.pid) + def find_level(self, pid: int) -> None: + """Finds how deep the PID is in the tasks hierarchy. - child_list = self._children.get(ppid, set([])) + Args: + pid: PID to find the level in the hierachy + """ + seen = set([pid]) + level = 0 + proc = self._tasks.get(pid) + while proc and proc.parent and proc.parent.pid not in seen: + if self.task_is_thread_group_leader(proc): + parent_pid = proc.parent.pid + else: + parent_pid = proc.tgid + + child_list = self._children.setdefault(parent_pid, set()) child_list.add(proc.pid) - self._children[ppid] = child_list - proc = self._processes.get(ppid, None) + + proc = self._tasks.get(parent_pid) level += 1 + self._levels[pid] = level - def _generator(self): - """Generates the.""" + def _generator( + self, + pid_filter, + include_threads: bool = False, + decorate_com: bool = False): + """Generates the tasks hierarchy tree. + + Args: + pid_filter: A function which takes a process object and returns True if the process should be ignored/filtered + include_threads: If True, the output will also show the user threads + If False, only the thread group leaders will be shown + Defaults to False. + decorate_comm: If True, it decorates the comm string of + - User threads: in curly brackets, + - Kernel threads: in square brackets + Defaults to False. + Yields: + Each rows + """ vmlinux = self.context.modules[self.config['kernel']] - for proc in self.list_tasks(self.context, vmlinux.name): - self._processes[proc.pid] = proc + for proc in self.list_tasks(self.context, + vmlinux.name, + filter_func=pid_filter, + include_threads=include_threads): + self._tasks[proc.pid] = proc # Build the child/level maps - for pid in self._processes: + for pid in self._tasks: self.find_level(pid) def yield_processes(pid): - proc = self._processes[pid] - row = (proc.pid, proc.parent.pid, utility.array_to_string(proc.comm)) + task = self._tasks[pid] - yield (self._levels[pid] - 1, row) - for child_pid in self._children.get(pid, []): + row = self._get_task_fields(task, decorate_com) + + tid = task.pid + yield (self._levels[tid] - 1, row) + + for child_pid in sorted(self._children.get(tid, [])): yield from yield_processes(child_pid) - for pid in self._levels: - if self._levels[pid] == 1: + for pid, level in self._levels.items(): + if level == 1: yield from yield_processes(pid) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 36e23a35d..4df97954d 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -267,3 +267,12 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): list_struct = vmlinux.object(object_type = struct_name, offset = list_start.vol.offset) yield list_struct list_start = getattr(list_struct, list_member) + + @classmethod + def container_of(cls, addr, type_name, member_name, vmlinux): + if not addr: + return + type_dec = vmlinux.get_type(type_name) + member_offset = type_dec.relative_child_offset(member_name) + container_addr = addr - member_offset + return vmlinux.object(object_type=type_name, offset=container_addr, absolute=True) \ No newline at end of file From 86d3785bea15f4337555c39b7008510d66032943 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Dec 2021 14:42:23 +1100 Subject: [PATCH 027/404] Fixing constant module and reference comment --- volatility3/framework/constants/linux/__init__.py | 1 + volatility3/framework/plugins/linux/pslist.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 276c6015f..c0f85593f 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -12,4 +12,5 @@ KERNEL_NAME = "__kernel__" PAGE_SHIFT = 12 """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" +# include/linux/sched.h PF_KTHREAD = 0x00200000 # I'm a kernel thread diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 516c74305..b42453d3c 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -59,7 +59,7 @@ class PsList(interfaces.plugins.PluginInterface): @staticmethod def task_is_kernel_thread(task: interfaces.objects.ObjectInterface) -> bool: - return (task.flags & constants.PF_KTHREAD) != 0 + return (task.flags & constants.linux.PF_KTHREAD) != 0 @staticmethod def task_is_thread_group_leader(task: interfaces.objects.ObjectInterface) -> bool: From 997b8572465ef811677c3e2775655e731cdb27fd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 19 Dec 2021 22:16:53 +0000 Subject: [PATCH 028/404] CLI: Add in None renderer to avoid text output --- volatility3/cli/text_renderer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 35b2468e8..e62f705ce 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -182,6 +182,17 @@ class QuickTextRenderer(CLIRenderer): outfd.write("\n") +class NoneRenderer(CLIRenderer): + """Outputs no results""" + name = "none" + + def get_render_options(self): + pass + + def render(self, grid: interfaces.renderers.TreeGrid) -> None: + if not grid.populated: + grid.populate(lambda x, y: True, True) + class CSVRenderer(CLIRenderer): _type_renderers = { format_hints.Bin: quoted_optional(lambda x: f"0b{x:b}"), From 39c97b8e9795892f0fbf6851154f66f62bb1871b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 20 Dec 2021 15:26:45 +1100 Subject: [PATCH 029/404] Moving thread type check methods to the task object --- volatility3/framework/plugins/linux/pslist.py | 20 +++----------- .../symbols/linux/extensions/__init__.py | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index b42453d3c..4c55f853c 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -3,7 +3,7 @@ # from typing import Callable, Iterable, List, Any, Tuple -from volatility3.framework import renderers, interfaces, constants +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.symbols import linux @@ -57,18 +57,6 @@ class PsList(interfaces.plugins.PluginInterface): else: return lambda _: False - @staticmethod - def task_is_kernel_thread(task: interfaces.objects.ObjectInterface) -> bool: - return (task.flags & constants.linux.PF_KTHREAD) != 0 - - @staticmethod - def task_is_thread_group_leader(task: interfaces.objects.ObjectInterface) -> bool: - return task.tgid == task.pid - - @staticmethod - def task_is_user_thread(task: interfaces.objects.ObjectInterface) -> bool: - return task.tgid != task.pid - def _get_task_fields( self, task: interfaces.objects.ObjectInterface, @@ -89,9 +77,9 @@ class PsList(interfaces.plugins.PluginInterface): ppid = task.parent.tgid if task.parent else 0 name = utility.array_to_string(task.comm) if decorate_comm: - if self.task_is_kernel_thread(task): + if task.is_kernel_thread: name = f"[{name}]" - elif self.task_is_user_thread(task): + elif task.is_user_thread: name = f"{{{name}}}" task_fields = (pid, tid, ppid, name) @@ -154,7 +142,7 @@ class PsList(interfaces.plugins.PluginInterface): next_task = task.thread_group.next while current_task is None or current_task.vol.offset != task.vol.offset: current_task = linux.LinuxUtilities.container_of(next_task, "task_struct", "thread_group", vmlinux) - if cls.task_is_thread_group_leader(current_task): + if current_task.is_thread_group_leader: # Making sure the first task yielded is the Task Group Leader yield current_task elif include_threads: diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0edd60608..f28705617 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -201,6 +201,33 @@ class task_struct(generic.GenericIntelProcess): yield (start, end - start) + @property + def is_kernel_thread(self) -> bool: + """Checks if this task is a kernel thread. + + Returns: + bool: True, if this task is a kernel thread. Otherwise, False. + """ + return (self.flags & constants.linux.PF_KTHREAD) != 0 + + @property + def is_thread_group_leader(self) -> bool: + """Checks if this task is a thread group leader. + + Returns: + bool: True, if this task is a thread group leader. Otherwise, False. + """ + return self.tgid == self.pid + + @property + def is_user_thread(self) -> bool: + """Checks if this task is a user thread. + + Returns: + bool: True, if this task is a user thread. Otherwise, False. + """ + return not self.is_kernel_thread and self.tgid != self.pid + class fs_struct(objects.StructType): From 3bc90b82009c8cb4774c635714a5c5d7009d2a5e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 20 Dec 2021 20:38:03 +0000 Subject: [PATCH 030/404] Plugins: Add more information to layerwriter --list --- volatility3/framework/plugins/layerwriter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index b2a02116e..0068ec224 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -110,9 +110,9 @@ class LayerWriter(plugins.PluginInterface): def _generate_layers(self): """List layer names from this run""" for name in self.context.layers: - yield (0, (name, )) + yield (0, (name, self.context.layers[name].__class__.__name__)) def run(self): if self.config['list']: - return renderers.TreeGrid([("Layer name", str)], self._generate_layers()) + return renderers.TreeGrid([("Layer name", str), ('Layer type', str)], self._generate_layers()) return renderers.TreeGrid([("Status", str)], self._generator()) From a96f5de6a0d635e381717b84f5bf643d7bbc6a3b Mon Sep 17 00:00:00 2001 From: cstation Date: Tue, 28 Dec 2021 18:45:08 +0100 Subject: [PATCH 031/404] QEMU: Add 'dirty-bitmap' and 'pbs-state', handle page_size more consistently --- volatility3/framework/layers/qemu.py | 31 +++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 1c5319dfd..df383a04a 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -54,7 +54,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, name: str) -> Any: """Reads the JSON configuration from the end of the file""" - chunk_size = 0x4096 + chunk_size = 4096 data = b'' for i in range(base_layer.maximum_address, base_layer.minimum_address, -chunk_size): if i != base_layer.maximum_address: @@ -65,6 +65,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if start_of_json >= 0: data = data[start_of_json:] return json.loads(data) + # No JSON configuration found at the end of the file, return empty dict return dict() raise exceptions.LayerException(name, "Invalid JSON configuration at the end of the file") @@ -79,9 +80,11 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): addr = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', offset = index, layer_name = self._base_layer) + # Flags are stored in the n least significant bits, where n equals the bit-length of pagesize flags = addr & (page_size - 1) - page_size_bits = int(math.log(page_size, 2)) - addr = (addr >> page_size_bits) << page_size_bits + # addr equals the highest multiple of pagesize <= offset + # (We assume that page_size is a power of 2) + addr = addr ^ (addr & (page_size - 1)) index += 8 if flags & self.SEGMENT_FLAG_MEM_SIZE: @@ -126,6 +129,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._configuration = self._read_configuration(base_layer, self.name) section_byte = -1 index = 8 + section_info = dict() current_section_id = -1 version_id = -1 name = None @@ -162,6 +166,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): offset = index, layer_name = self._base_layer) index += 4 + # Store section info for handling QEVM_SECTION_PARTs later on + section_info[current_section_id] = {'name': name, 'version_id': version_id} # Read additional data index = self.extract_data(index, name, version_id) elif section_byte == self.QEVM_SECTION_PART or section_byte == self.QEVM_SECTION_END: @@ -171,7 +177,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): current_section_id = section_id index += 4 # Read additional data - index = self.extract_data(index, name, version_id) + index = self.extract_data(index, section_info[current_section_id]['name'], + section_info[current_section_id]['version_id']) elif section_byte == self.QEVM_SECTION_FOOTER: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', offset = index, @@ -189,7 +196,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if name == 'ram': if version_id != 4: raise exceptions.LayerException(f"QEMU unknown RAM version_id {version_id}") - new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', None) or 4096) + new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', 4096)) self._segments += new_segments elif name == 'spapr/htab': if version_id != 1: @@ -208,6 +215,13 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): layer_name = self._base_layer) htab_index, htab_n_valid, htab_n_invalid = htab index += 8 + (htab_n_valid * self.HASH_PTE_SIZE_64) + elif name == 'dirty-bitmap': + index += 1 + elif name == 'pbs-state': + section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', + offset = index, + layer_name = self._base_layer) + index += 8 + section_len return index def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes: @@ -217,9 +231,12 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): of the starting data. It is the responsibility of the layer to turn the provided data chunk into the right portion of data necessary. """ - start_offset = offset ^ (offset & 0xfff) + page_size = self._configuration.get('page_size', 4096) + # start_offset equals the highest multiple of pagesize <= offset + # (We assume that page_size is a power of 2) + start_offset = offset ^ (offset & (page_size - 1)) if start_offset in self._compressed: - data = (data * 0x1000) + data = (data * page_size) result = data[offset - start_offset:output_length + offset - start_offset] return result From 7b0a90afa69f268db1a8b835f37d630f050c2fc5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Dec 2021 22:15:08 +0000 Subject: [PATCH 032/404] Automagic: Warn when multiple symbol files match a banner --- volatility3/framework/automagic/linux.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f9fa22c07..154f01749 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -45,6 +45,12 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): symbol_files = linux_banners.get(banner, None) if symbol_files: + if len(symbol_files) > 1: + using = "*" + vollog.warning(f"Multiple symbol files identified (using {using}):") + for symbol_file in symbol_files: + vollog.warning(f" {using} {symbol_file}") + using = " " isf_path = symbol_files[0] table_name = context.symbol_space.free_table_name('LintelStacker') table = linux.LinuxKernelIntermedSymbols(context, From c8dd8d08bda450d29cec91b797eb4094fbfed0e0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Dec 2021 22:26:28 +0000 Subject: [PATCH 033/404] Volshell: Synchronize with the standard CLI cache clearing --- volatility3/cli/volshell/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 94f735ba0..409123457 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -138,8 +138,7 @@ class VolShell(cli.CommandLine): console.setLevel(10 - (partial_args.verbosity - 2)) if partial_args.clear_cache: - for cache_filename in glob.glob(os.path.join(constants.CACHE_PATH, '*.cache')): - os.unlink(cache_filename) + framework.clear_cache() # Do the initialization ctx = contexts.Context() # Construct a blank context From f38fc22002714eb87734b1049cb3858e222d2fc0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Dec 2021 23:51:28 +0000 Subject: [PATCH 034/404] CLI: Support multi-line fields and tabstops in pretty renderer --- volatility3/cli/text_renderer.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index e62f705ce..8663bb995 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -283,9 +283,10 @@ class PrettyTextRenderer(CLIRenderer): column = grid.columns[column_index] renderer = self._type_renderers.get(column.type, self._type_renderers['default']) data = renderer(node.values[column_index]) + field_width = max([len(self.tab_stop(x)) for x in f"{data}".split("\n")]) max_column_widths[column.name] = max(max_column_widths.get(column.name, len(column.name)), - len(f"{data}")) - line[column] = data + field_width) + line[column] = data.split("\n") accumulator.append((node.path_depth, line)) return accumulator @@ -307,7 +308,25 @@ class PrettyTextRenderer(CLIRenderer): column_titles = [""] + [column.name for column in grid.columns] outfd.write(format_string.format(*column_titles)) for (depth, line) in final_output: - outfd.write(format_string.format("*" * depth, *[line[column] for column in grid.columns])) + nums_line = max([len(line[column]) for column in line]) + for column in line: + line[column] = line[column] + ([""] * (nums_line - len(line[column]))) + for index in range(nums_line): + if index == 0: + outfd.write(format_string.format("*" * depth, *[self.tab_stop(line[column][index]) for column in grid.columns])) + else: + outfd.write(format_string.format(" " * depth, *[self.tab_stop(line[column][index]) for column in grid.columns])) + + def tab_stop(self, line: str) -> str: + tab_width = 8 + while line.find('\t') >= 0: + i = line.find('\t') + if (tab_width > 0): + pad = " " * (tab_width - (i % tab_width)) + else: + pad = "" + line = line.replace("\t", pad, 1) + return line class JsonRenderer(CLIRenderer): From c13b49262d5ee5049d34d2781d319fbc9ac578d8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 30 Dec 2021 01:47:45 +0000 Subject: [PATCH 035/404] Renderers: Make appending rows massively more efficient Previously we were generating the list of children (for most likely the root node) for every single append statement, during which we were recalculating the length of the list, twice. In plugins that output a lot of rows this would add an enourmous overhead (that likely grew as the length of the output grew). Without this overhead, the time taken for the TreeGrid._append method went from 1230.0s to 2.4s. --- volatility3/framework/renderers/__init__.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index de1b14c91..5773861d9 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -272,20 +272,26 @@ class TreeGrid(interfaces.renderers.TreeGrid): def _append(self, parent: Optional[interfaces.renderers.TreeNode], values: Any) -> TreeNode: """Adds a new node at the top level if parent is None, or under the parent node otherwise, after all other children.""" - children = self.children(parent) - return self._insert(parent, len(children), values) + return self._insert(parent, None, values) - def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: int, values: Any) -> TreeNode: + def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: Optional[int], values: Any) -> TreeNode: """Inserts an element into the tree at a specific position.""" parent_path = "" children = self._find_children(parent) if parent is not None: parent_path = parent.path + self.path_sep - newpath = parent_path + str(position) + if position is None: + newpath = parent_path + str(len(children)) + else: + newpath = parent_path + str(position) + for node, _ in children[position:]: + self.visit(node, lambda child, _: child.path_changed(newpath, True), None) + tree_item = TreeNode(newpath, self, parent, values) - for node, _ in children[position:]: - self.visit(node, lambda child, _: child.path_changed(newpath, True), None) - children.insert(position, (tree_item, [])) + if position is None: + children.append((tree_item, [])) + else: + children.insert(position, (tree_item, [])) return tree_item def is_ancestor(self, node, descendant): From 571ab8f6590ec5306af8c6e391e3a481c2580560 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 31 Dec 2021 00:51:21 +0000 Subject: [PATCH 036/404] Volshell: Update the linux pslist requirement --- volatility3/cli/volshell/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 850e3111c..4338ae06f 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -17,7 +17,7 @@ class Volshell(generic.Volshell): def get_requirements(cls): return (super().get_requirements() + [ requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) From ac7127d05a5dc9a81870492d851b0c74c8eb8a43 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 31 Dec 2021 00:55:23 +0000 Subject: [PATCH 037/404] Volshell: Sync errors with the CLI concerning unsatisfied requirements Fixes #607 --- volatility3/cli/volshell/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 409123457..7b8a759a6 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -238,9 +238,14 @@ class VolShell(cli.CommandLine): vollog.debug("Writing out configuration data to config.json") with open("config.json", "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + except exceptions.UnsatisfiedException as excp: + self.process_unsatisfied_exceptions(excp) + parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") + try: # Construct and run the plugin - constructed.run() + if constructed: + constructed.run() except exceptions.VolatilityException as excp: self.process_exceptions(excp) parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") From 7aed488721153c7e786de7775a83f1056f3d8d16 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jan 2022 01:21:37 +0000 Subject: [PATCH 038/404] Automagic: Allow automagic to exclude unsupported OSes --- doc/source/using-as-a-library.rst | 3 ++- doc/source/vol2to3.rst | 4 ++++ volatility3/framework/automagic/__init__.py | 21 ++++++------------- volatility3/framework/automagic/linux.py | 2 ++ volatility3/framework/automagic/mac.py | 2 ++ volatility3/framework/constants/__init__.py | 1 + volatility3/framework/interfaces/automagic.py | 3 +++ 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index 95d2b1080..c63adcfc3 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -131,7 +131,8 @@ A suitable list of automagics for a particular plugin (based on operating system automagics = automagic.choose_automagic(available_automagics, plugin) This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just -the automagics which apply to the operating system. +the automagics which apply to the operating system. Each automagic can exclude itself from being used for specific +operating systems, so that an automagic designed for linux is not used for windows or mac plugins. These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that the plugin will be constructed on ('plugins' by default) and a progress_callback. This is a callable which takes diff --git a/doc/source/vol2to3.rst b/doc/source/vol2to3.rst index eb33b6618..e768df0c2 100644 --- a/doc/source/vol2to3.rst +++ b/doc/source/vol2to3.rst @@ -62,6 +62,10 @@ automagic processes are clearly defined and can be enabled or disabled as necess included a stacker automagic to emulate the most common feature of Volatility 2, automatically stacking address spaces (now translation layers) on top of each other. +By default the automagic chosen to be run are determined based on the plugin requested, so that linux plugins get linux +specific automagic and windows plugins get windows specific automagic. This should reduce unnecessarily searching for +linux kernels in a windows image, for example. At the moment this is not user configurableS. + Searching and Scanning ---------------------- Scanning is very similar to scanning in Volatility 2, a scanner object (such as a diff --git a/volatility3/framework/automagic/__init__.py b/volatility3/framework/automagic/__init__.py index e4d422c99..7567f206d 100644 --- a/volatility3/framework/automagic/__init__.py +++ b/volatility3/framework/automagic/__init__.py @@ -21,14 +21,6 @@ from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) -windows_automagic = [ - 'ConstructionMagic', 'LayerStacker', 'KernelPDBScanner', 'WinSwapLayers', 'KernelModule' -] - -linux_automagic = ['ConstructionMagic', 'LayerStacker', 'LinuxBannerCache', 'LinuxSymbolFinder', 'KernelModule'] - -mac_automagic = ['ConstructionMagic', 'LayerStacker', 'MacBannerCache', 'MacSymbolFinder', 'KernelModule'] - def available(context: interfaces.context.ContextInterface) -> List[interfaces.automagic.AutomagicInterface]: """Returns an ordered list of all subclasses of @@ -58,10 +50,7 @@ def choose_automagic( plugin_category = "None" plugin_categories = plugin.__module__.split('.') lowest_index = len(plugin_categories) - - automagic_categories = {'windows': windows_automagic, 'linux': linux_automagic, 'mac': mac_automagic} - - for os in automagic_categories: + for os in constants.OS_CATEGORIES: try: if plugin_categories.index(os) < lowest_index: lowest_index = plugin_categories.index(os) @@ -70,14 +59,16 @@ def choose_automagic( # The value wasn't found, try the next one pass - if plugin_category not in automagic_categories: + if plugin_category not in constants.OS_CATEGORIES: vollog.info("No plugin category detected") return automagics - vollog.info(f"Detected a {plugin_category} category plugin") + output = [] for amagic in automagics: - if amagic.__class__.__name__ in automagic_categories[plugin_category]: + if plugin_category not in amagic.exclusion_list: + # Only include uncategorized automagic, or platform specific automagic + # (This allows user defined/uncategorized automagic to be included) output += [amagic] return output diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f9fa22c07..a6577e322 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -147,6 +147,7 @@ class LinuxBannerCache(symbol_cache.SymbolBannerCache): os = "linux" symbol_name = "linux_banner" banner_path = constants.LINUX_BANNERS_PATH + exclusion_list = ['mac', 'windows'] class LinuxSymbolFinder(symbol_finder.SymbolFinder): @@ -156,3 +157,4 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder): banner_cache = LinuxBannerCache symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] + exclusion_list = ['mac', 'windows'] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index fb725a234..c37aef463 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -202,6 +202,7 @@ class MacBannerCache(symbol_cache.SymbolBannerCache): os = "mac" symbol_name = "version" banner_path = constants.MAC_BANNERS_PATH + exclusion_list = ['windows', 'linux'] class MacSymbolFinder(symbol_finder.SymbolFinder): @@ -211,3 +212,4 @@ class MacSymbolFinder(symbol_finder.SymbolFinder): banner_cache = MacBannerCache find_aslr = MacIntelStacker.find_aslr symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols" + exclusion_list = ['windows', 'linux'] diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 23598837b..82ebd4936 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -78,6 +78,7 @@ BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" ProgressCallback = Optional[Callable[[float, str], None]] """Type information for ProgressCallback objects""" +OS_CATEGORIES = ['windows', 'mac', 'linux'] class Parallelism(enum.IntEnum): """An enumeration listing the different types of parallelism applied to diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index c310f5b4a..c96c9bdbe 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -40,6 +40,9 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla priority = 10 """An ordering to indicate how soon this automagic should be run""" + exclusion_list = [] + """A list of plugin categories (typically operating systems) which the plugin will not operate on""" + def __init__(self, context: interfaces.context.ContextInterface, config_path: str, *args, **kwargs) -> None: super().__init__(context, config_path) for requirement in self.get_requirements(): From 2c949dfa50e2b530a3df9cc480933e583a1ca4e7 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Thu, 6 Jan 2022 22:53:24 +0000 Subject: [PATCH 039/404] Create MFTScanner plugin --- .../framework/plugins/windows/mftscan.py | 305 ++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 volatility3/framework/plugins/windows/mftscan.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py new file mode 100644 index 000000000..1d4ca954e --- /dev/null +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -0,0 +1,305 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from struct import unpack +from typing import Iterable + +from volatility3.framework import constants, renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework.renderers import conversion, format_hints +from volatility3.framework.symbols import intermed +from volatility3.plugins import yarascan + +vollog = logging.getLogger(__name__) + +try: + import yara +except ImportError: + vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") + raise + +signatures = { + 'mft_objects': """rule mft_headers + { + strings: + $header1 = "FILE0" + $header2 = "FILE*" + $header3 = "BAAD" + condition: + any of them + }""" +} + +# https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee9338b881c0fa25937/volatility/plugins/mftparser.py#L60 +ATTRIBUTE_TYPE_ID = { + 0x10:"STANDARD_INFORMATION", + 0x20:"ATTRIBUTE_LIST", + 0x30:"FILE_NAME", + 0x40:"OBJECT_ID", + 0x50:"SECURITY_DESCRIPTOR", + 0x60:"VOLUME_NAME", + 0x70:"VOLUME_INFORMATION", + 0x80:"DATA", + 0x90:"INDEX_ROOT", + 0xa0:"INDEX_ALLOCATION", + 0xb0:"BITMAP", + 0xc0:"REPARSE_POINT", + 0xd0:"EA_INFORMATION", #Extended Attribute + 0xe0:"EA", + 0xf0:"PROPERTY_SET", + 0x100:"LOGGED_UTILITY_STREAM", +} + +VERBOSE_STANDARD_INFO_FLAGS = { + 0x1:"Read Only", + 0x2:"Hidden", + 0x4:"System", + 0x20:"Archive", + 0x40:"Device", + 0x80:"Normal", + 0x100:"Temporary", + 0x200:"Sparse File", + 0x400:"Reparse Point", + 0x800:"Compressed", + 0x1000:"Offline", + 0x2000:"Content not indexed", + 0x4000:"Encrypted", + 0x10000000:"Directory", + 0x20000000:"Index view", +} + +FILE_NAME_NAMESPACE = { + 0x0:"POSIX", # Case sensitive, allows all Unicode chars except '/' and NULL + 0x1:"Win32", # Case insensitive, allows most Unicide except specials ('/', '\', ';', '>', '<', '?') + 0x2:"DOS", # Case insensitive, upper case, no special chars, name is 8 or fewer chars in name and 3 or less extension + 0x3:"Win32 & DOS", # Used when original name fits in DOS namespace and 2 names are not needed +} + +MFT_FLAGS = { + 0x0: "Removed", + 0x1: "File", # "In Use", + 0x2: "Directory", # if flag & 0x0002 == 0 this is a regular file + 0x3: "Directory" +} + +INDEX_ENTRY_FLAGS = { + 0x1:"Child Node Exists", + 0x2:"Last entry in list", +} + + +class MFTScan(interfaces.plugins.PluginInterface): + """Scans for MFT FILE objects present in a particular windows memory image.""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, + version = (2, 0, 0)), + ] + + # https://docs.python.org/3/library/struct.html + @classmethod + def unpack_data(self, mft_record, offset, data_type): + """Helper to unpack values from the raw mft_record""" + + if data_type == 'unsigned long': + return unpack(' 1000: + continue + + # attr_header + attr_type = self.unpack_data(mft_record, attr_offset, 'int') + attr_len = self.unpack_data(mft_record, attr_offset+4, 'int') + + # As we look for strucutres of header + 1K we can not unpack non resident structures + nr_flag = self.unpack_data(mft_record, attr_offset+8, 'unsigned char') + + # Skip headers + attr_data = attr_offset+24 # Len of Common and Resident Headers + + if attr_type in ATTRIBUTE_TYPE_ID: + vollog.debug(f'Found Attribute {ATTRIBUTE_TYPE_ID[attr_type]}') + + if ATTRIBUTE_TYPE_ID[attr_type] == 'STANDARD_INFORMATION': + creation_time_win = self.unpack_data(mft_record, attr_data, 'unsigned long long') + modified_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') + altered_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') + access_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') + flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') + permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') + + + mft_entry['attributes']['SI'] = { + "creation_time": self.human_date(creation_time_win), + "modified_time": self.human_date(modified_time_win), + "updated_time": self.human_date(altered_time_win), + "accessed_time": self.human_date(access_time_win), + "flags": permissions + } + + if ATTRIBUTE_TYPE_ID[attr_type] == 'FILE_NAME': + parent_record = self.unpack_data(mft_record, attr_data, 'unsigned long long') + creation_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') + modified_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') + altered_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') + access_time_win = self.unpack_data(mft_record, attr_data+32, 'unsigned long long') + + name_len = self.unpack_data(mft_record, attr_data+64, 'unsigned char') + name_space = self.unpack_data(mft_record, attr_data+65, 'unsigned char') + + # Unicode and partially corruprted records can break us here. + file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] + try: + file_name = file_name.replace(b'\x00', b'').decode() + except: + file_name = str(file_name.replace(b'\x00', b'')) + + flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') + permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') + + mft_entry['attributes']['FN'].append( + { + "creation_time": self.human_date(creation_time_win), + "modified_time": self.human_date(modified_time_win), + "updated_time": self.human_date(altered_time_win), + "accessed_time": self.human_date(access_time_win), + "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), + "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), + "flags": permissions, + "file_name": file_name, + "name_space": name_space + }) + + # Update Offset for next Attribute + attr_offset += attr_len + + return mft_entry + + def _generator(self): + rules = yara.compile(sources = signatures) + + layer = self.context.layers[self.config['primary']] + for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + + try: + mft_record = layer.read(offset, 1024, False) + mft_entry = self.parse_mft_record(mft_record) + except PagedInvalidAddressException: + mft_entry = None + except Exception as err: + vollog.error(err) + mft_entry = None + + if mft_entry: + vollog.debug(mft_entry) + + # Tree Grid is large and variable + si = mft_entry['attributes']['SI'] + fn = mft_entry['attributes']['FN'] + + signature = mft_entry.get('signature', 0) + record_number = mft_entry.get('record_number', 0) + link_count = mft_entry.get('link_count', 0) + permissions = mft_entry.get('flags', '') + + si_creation_time = si.get('creation_time', '') + si_modified_time = si.get('modified_time', '') + si_updated_time = si.get('updated_time', '') + si_accessed_time = si.get('accessed_time', '') + + yield 0, ( + format_hints.Hex(offset), + signature, + record_number, + link_count, + permissions, + 'Standard Information', + 'N/A', + si_creation_time, + si_modified_time, + si_updated_time, + si_accessed_time) + + for entry in fn: + # As this is variable and may or may not exist + # And could have 0-6 entries lets do it per row. + yield 0, ( + format_hints.Hex(offset), + signature, + record_number, + link_count, + permissions, + 'FileName', + entry.get('file_name', ''), + entry.get('creation_time', ''), + entry.get('modified_time', ''), + entry.get('updated_time', ''), + entry.get('accessed_time', '')) + + def run(self): + return renderers.TreeGrid([ + ('Offset', format_hints.Hex), + ('Record Type', str), + ('Record Number', int), + ('Link Count', int), + ('Permissions', str), + ('Attribute Type', str), + ('Filename', str), + ('Created', str), + ('Modified', str), + ('Updated', str), + ('Accessed', str) + ],self._generator()) From 899ec09ce3b620f75c9ecb74e117d8a3a918be3b Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Thu, 6 Jan 2022 23:09:50 +0000 Subject: [PATCH 040/404] Doc Strings --- .../framework/plugins/windows/mftscan.py | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 1d4ca954e..b75756a1d 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,13 @@ import logging from struct import unpack -from typing import Iterable +from typing import Dict -from volatility3.framework import constants, renderers, interfaces, exceptions +from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols import intermed from volatility3.plugins import yarascan vollog = logging.getLogger(__name__) @@ -109,8 +109,17 @@ class MFTScan(interfaces.plugins.PluginInterface): # https://docs.python.org/3/library/struct.html @classmethod - def unpack_data(self, mft_record, offset, data_type): - """Helper to unpack values from the raw mft_record""" + def unpack_data(self, mft_record: bytes, offset: int, data_type: str) -> bytes: + """Helper to unpack values from the raw mft_record + + Args: + mft_record: 1024 bytes starting from header value as returned by layer read + offset: how far in to the record to read + data_type: what is the data type to unpack + + Returns: + bytes: the unpacked data + """ if data_type == 'unsigned long': return unpack(' str: + """Converts a windows epoch to a date time string with a fixed format + + Args: + datetime_object: windows epoch time + + Returns: + str: strftime of the windows epoch in UTC + + """ dtg = conversion.wintime_to_datetime(datetime_object) return dtg.strftime('%Y-%m-%d %H:%M:%S %z') @classmethod - def parse_mft_record(self, mft_record): - """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes""" + def parse_mft_record(self, mft_record: bytes) -> Dict: + """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes + + Args: + mft_record: 1024 bytes starting from header value as returned by layer read + + Returns: + Dict: a Dictionary that contains the Parse MFT Record + """ # https://github.com/Invoke-IR/ForensicPosters flags = self.unpack_data(mft_record, 22, 'unsigned short') @@ -202,10 +226,11 @@ class MFTScan(interfaces.plugins.PluginInterface): # Unicode and partially corruprted records can break us here. file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - try: - file_name = file_name.replace(b'\x00', b'').decode() - except: - file_name = str(file_name.replace(b'\x00', b'')) + file_name = utility.array_to_string(file_name) + #try: + # # file_name = file_name.replace(b'\x00', b'').decode() + #except: + # file_name = str(file_name.replace(b'\x00', b'')) flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') @@ -234,6 +259,7 @@ class MFTScan(interfaces.plugins.PluginInterface): layer = self.context.layers[self.config['primary']] for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + # For each matching rule try to read 1024 bytes (size of an MFT record) at the offset. try: mft_record = layer.read(offset, 1024, False) mft_entry = self.parse_mft_record(mft_record) From 20a5868ff3df16710695b497069aba9ff0387842 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 00:23:31 +0000 Subject: [PATCH 041/404] Change return types for MFT Records DTGs --- .../framework/plugins/windows/mftscan.py | 101 ++++++++---------- 1 file changed, 44 insertions(+), 57 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index b75756a1d..ddc179fe1 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -2,14 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime import logging +import struct -from struct import unpack from typing import Dict from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework import exceptions from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints from volatility3.plugins import yarascan @@ -122,29 +123,15 @@ class MFTScan(interfaces.plugins.PluginInterface): """ if data_type == 'unsigned long': - return unpack(' str: - """Converts a windows epoch to a date time string with a fixed format - - Args: - datetime_object: windows epoch time - - Returns: - str: strftime of the windows epoch in UTC - - """ - dtg = conversion.wintime_to_datetime(datetime_object) - return dtg.strftime('%Y-%m-%d %H:%M:%S %z') + return struct.unpack(' Dict: @@ -207,10 +194,10 @@ class MFTScan(interfaces.plugins.PluginInterface): mft_entry['attributes']['SI'] = { - "creation_time": self.human_date(creation_time_win), - "modified_time": self.human_date(modified_time_win), - "updated_time": self.human_date(altered_time_win), - "accessed_time": self.human_date(access_time_win), + "creation_time": conversion.wintime_to_datetime(creation_time_win), + "modified_time": conversion.wintime_to_datetime(modified_time_win), + "updated_time": conversion.wintime_to_datetime(altered_time_win), + "accessed_time": conversion.wintime_to_datetime(access_time_win), "flags": permissions } @@ -226,21 +213,21 @@ class MFTScan(interfaces.plugins.PluginInterface): # Unicode and partially corruprted records can break us here. file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - file_name = utility.array_to_string(file_name) - #try: - # # file_name = file_name.replace(b'\x00', b'').decode() - #except: - # file_name = str(file_name.replace(b'\x00', b'')) + #file_name = utility.array_to_string(file_name) + try: + file_name = file_name.replace(b'\x00', b'').decode() + except: + file_name = str(file_name.replace(b'\x00', b'')) flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') mft_entry['attributes']['FN'].append( { - "creation_time": self.human_date(creation_time_win), - "modified_time": self.human_date(modified_time_win), - "updated_time": self.human_date(altered_time_win), - "accessed_time": self.human_date(access_time_win), + "creation_time": conversion.wintime_to_datetime(creation_time_win), + "modified_time": conversion.wintime_to_datetime(modified_time_win), + "updated_time": conversion.wintime_to_datetime(altered_time_win), + "accessed_time": conversion.wintime_to_datetime(access_time_win), "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), "flags": permissions, @@ -263,11 +250,11 @@ class MFTScan(interfaces.plugins.PluginInterface): try: mft_record = layer.read(offset, 1024, False) mft_entry = self.parse_mft_record(mft_record) - except PagedInvalidAddressException: - mft_entry = None - except Exception as err: - vollog.error(err) + except exceptions.PagedInvalidAddressException: mft_entry = None + #except Exception as err: + # vollog.error(err) + # mft_entry = None if mft_entry: vollog.debug(mft_entry) @@ -276,15 +263,15 @@ class MFTScan(interfaces.plugins.PluginInterface): si = mft_entry['attributes']['SI'] fn = mft_entry['attributes']['FN'] - signature = mft_entry.get('signature', 0) - record_number = mft_entry.get('record_number', 0) - link_count = mft_entry.get('link_count', 0) - permissions = mft_entry.get('flags', '') + signature = mft_entry.get('signature', renderers.NotAvailableValue()) + record_number = mft_entry.get('record_number', renderers.NotAvailableValue()) + link_count = mft_entry.get('link_count', renderers.NotAvailableValue()) + permissions = mft_entry.get('flags', renderers.NotAvailableValue()) - si_creation_time = si.get('creation_time', '') - si_modified_time = si.get('modified_time', '') - si_updated_time = si.get('updated_time', '') - si_accessed_time = si.get('accessed_time', '') + si_creation_time = si.get('creation_time', renderers.NotAvailableValue()) + si_modified_time = si.get('modified_time', renderers.NotAvailableValue()) + si_updated_time = si.get('updated_time', renderers.NotAvailableValue()) + si_accessed_time = si.get('accessed_time', renderers.NotAvailableValue()) yield 0, ( format_hints.Hex(offset), @@ -293,7 +280,7 @@ class MFTScan(interfaces.plugins.PluginInterface): link_count, permissions, 'Standard Information', - 'N/A', + renderers.NotApplicableValue(), si_creation_time, si_modified_time, si_updated_time, @@ -302,18 +289,18 @@ class MFTScan(interfaces.plugins.PluginInterface): for entry in fn: # As this is variable and may or may not exist # And could have 0-6 entries lets do it per row. - yield 0, ( + yield 1, ( format_hints.Hex(offset), signature, record_number, link_count, permissions, 'FileName', - entry.get('file_name', ''), - entry.get('creation_time', ''), - entry.get('modified_time', ''), - entry.get('updated_time', ''), - entry.get('accessed_time', '')) + entry.get('file_name',''), + entry.get('creation_time', renderers.NotAvailableValue()), + entry.get('modified_time', renderers.NotAvailableValue()), + entry.get('updated_time', renderers.NotAvailableValue()), + entry.get('accessed_time', renderers.NotAvailableValue())) def run(self): return renderers.TreeGrid([ @@ -324,8 +311,8 @@ class MFTScan(interfaces.plugins.PluginInterface): ('Permissions', str), ('Attribute Type', str), ('Filename', str), - ('Created', str), - ('Modified', str), - ('Updated', str), - ('Accessed', str) + ('Created', datetime.datetime), + ('Modified', datetime.datetime), + ('Updated', datetime.datetime), + ('Accessed', datetime.datetime) ],self._generator()) From 295fb453f5e73f65d39912336ef6517268e26d17 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 00:55:34 +0000 Subject: [PATCH 042/404] Add MFT Filename N/A type --- volatility3/framework/plugins/windows/mftscan.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index ddc179fe1..10db6f112 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -192,7 +192,6 @@ class MFTScan(interfaces.plugins.PluginInterface): flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - mft_entry['attributes']['SI'] = { "creation_time": conversion.wintime_to_datetime(creation_time_win), "modified_time": conversion.wintime_to_datetime(modified_time_win), @@ -296,7 +295,7 @@ class MFTScan(interfaces.plugins.PluginInterface): link_count, permissions, 'FileName', - entry.get('file_name',''), + entry.get('file_name',renderers.NotAvailableValue()), entry.get('creation_time', renderers.NotAvailableValue()), entry.get('modified_time', renderers.NotAvailableValue()), entry.get('updated_time', renderers.NotAvailableValue()), From 114a8d7c8d0195a83bfc0c0654fb24bd63202302 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 9 Jan 2022 20:32:34 +0000 Subject: [PATCH 043/404] Plugins: Update yarascan options Add in a yara_source option in the process method. Unfortunately yara_rules is still poorly named, but would require a major version bump, so to avoid major disruption, we're just adding the yara_source option instead. --- volatility3/framework/plugins/yarascan.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 94b3cba45..4d2ba88ee 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -3,7 +3,7 @@ # import logging -from typing import Iterable, Tuple, List, Dict, Any +from typing import Any, Dict, Iterable, List, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -40,7 +40,7 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -83,6 +83,8 @@ class YaraScan(plugins.PluginInterface): if config.get('wide', False): rule += " wide ascii" rules = yara.compile(sources = {'n': f'rule r1 {{strings: $a = {rule} condition: $a}}'}) + elif config.get('yara_source', None) is not None: + rules = yara.compile(source = config['yara_source']) elif config.get('yara_file', None) is not None: rules = yara.compile(file = resources.ResourceAccessor().open(config['yara_file'], "rb")) elif config.get('yara_compiled_file', None) is not None: From d34030e9ea44e81287c4836d9851280b3190d096 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 9 Jan 2022 20:52:49 +0000 Subject: [PATCH 044/404] Plugins: Add note to improve yarascan in the future --- volatility3/framework/plugins/yarascan.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 4d2ba88ee..e51669b2c 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -42,6 +42,9 @@ class YaraScan(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 1, 0) + # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string + # or something that makes more sense + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ From 9d86599e7f391c41f2fb7e06f6f0f811bd97a7b7 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 22:04:13 +0000 Subject: [PATCH 045/404] MFT Plugin use ISF instead of Struct --- .../framework/plugins/windows/mftscan.py | 352 ++++------------- .../symbols/windows/extensions/mft.py | 104 +++++ .../framework/symbols/windows/mft.json | 371 ++++++++++++++++++ volatility3/framework/symbols/windows/mft.py | 15 + 4 files changed, 577 insertions(+), 265 deletions(-) create mode 100644 volatility3/framework/symbols/windows/extensions/mft.py create mode 100644 volatility3/framework/symbols/windows/mft.json create mode 100644 volatility3/framework/symbols/windows/mft.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 10db6f112..484fcdb9a 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -4,95 +4,20 @@ import datetime import logging -import struct from typing import Dict from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions -from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints +from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags +from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols + from volatility3.plugins import yarascan vollog = logging.getLogger(__name__) -try: - import yara -except ImportError: - vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") - raise - -signatures = { - 'mft_objects': """rule mft_headers - { - strings: - $header1 = "FILE0" - $header2 = "FILE*" - $header3 = "BAAD" - condition: - any of them - }""" -} - -# https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee9338b881c0fa25937/volatility/plugins/mftparser.py#L60 -ATTRIBUTE_TYPE_ID = { - 0x10:"STANDARD_INFORMATION", - 0x20:"ATTRIBUTE_LIST", - 0x30:"FILE_NAME", - 0x40:"OBJECT_ID", - 0x50:"SECURITY_DESCRIPTOR", - 0x60:"VOLUME_NAME", - 0x70:"VOLUME_INFORMATION", - 0x80:"DATA", - 0x90:"INDEX_ROOT", - 0xa0:"INDEX_ALLOCATION", - 0xb0:"BITMAP", - 0xc0:"REPARSE_POINT", - 0xd0:"EA_INFORMATION", #Extended Attribute - 0xe0:"EA", - 0xf0:"PROPERTY_SET", - 0x100:"LOGGED_UTILITY_STREAM", -} - -VERBOSE_STANDARD_INFO_FLAGS = { - 0x1:"Read Only", - 0x2:"Hidden", - 0x4:"System", - 0x20:"Archive", - 0x40:"Device", - 0x80:"Normal", - 0x100:"Temporary", - 0x200:"Sparse File", - 0x400:"Reparse Point", - 0x800:"Compressed", - 0x1000:"Offline", - 0x2000:"Content not indexed", - 0x4000:"Encrypted", - 0x10000000:"Directory", - 0x20000000:"Index view", -} - -FILE_NAME_NAMESPACE = { - 0x0:"POSIX", # Case sensitive, allows all Unicode chars except '/' and NULL - 0x1:"Win32", # Case insensitive, allows most Unicide except specials ('/', '\', ';', '>', '<', '?') - 0x2:"DOS", # Case insensitive, upper case, no special chars, name is 8 or fewer chars in name and 3 or less extension - 0x3:"Win32 & DOS", # Used when original name fits in DOS namespace and 2 names are not needed -} - -MFT_FLAGS = { - 0x0: "Removed", - 0x1: "File", # "In Use", - 0x2: "Directory", # if flag & 0x0002 == 0 this is a regular file - 0x3: "Directory" -} - -INDEX_ENTRY_FLAGS = { - 0x1:"Child Node Exists", - 0x2:"Last entry in list", -} - - class MFTScan(interfaces.plugins.PluginInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" @@ -108,198 +33,94 @@ class MFTScan(interfaces.plugins.PluginInterface): version = (2, 0, 0)), ] - # https://docs.python.org/3/library/struct.html - @classmethod - def unpack_data(self, mft_record: bytes, offset: int, data_type: str) -> bytes: - """Helper to unpack values from the raw mft_record - - Args: - mft_record: 1024 bytes starting from header value as returned by layer read - offset: how far in to the record to read - data_type: what is the data type to unpack - - Returns: - bytes: the unpacked data - """ - - if data_type == 'unsigned long': - return struct.unpack(' Dict: - """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes - - Args: - mft_record: 1024 bytes starting from header value as returned by layer read - - Returns: - Dict: a Dictionary that contains the Parse MFT Record - """ - # https://github.com/Invoke-IR/ForensicPosters - - flags = self.unpack_data(mft_record, 22, 'unsigned short') - file_type = MFT_FLAGS.get(flags, 'Unknown') - - mft_entry = { - "signature": mft_record[:4].decode(), - "FixupArrayOffset": self.unpack_data(mft_record, 4, 'unsigned short'), - "NumFixupEntries": self.unpack_data(mft_record, 6, 'unsigned short'), - "LSN": self.unpack_data(mft_record, 8, 'unsigned long long'), - "SequenceValue": self.unpack_data(mft_record, 16, 'unsigned short'), - "link_count": self.unpack_data(mft_record, 18, 'unsigned short'), - "FirstAttrOffset": self.unpack_data(mft_record, 20, 'unsigned short'), - "flags": file_type, - "record_number": self.unpack_data(mft_record, 44, 'unsigned long'), - "attributes": { - "SI": {}, - "FN": [] - } - } - - attr_offset = mft_entry['FirstAttrOffset'] - # Check at most for 6 entries - for i in range(6): - # If we attempt to overread the entry continue out - if attr_offset > 1000: - continue - - # attr_header - attr_type = self.unpack_data(mft_record, attr_offset, 'int') - attr_len = self.unpack_data(mft_record, attr_offset+4, 'int') - - # As we look for strucutres of header + 1K we can not unpack non resident structures - nr_flag = self.unpack_data(mft_record, attr_offset+8, 'unsigned char') - - # Skip headers - attr_data = attr_offset+24 # Len of Common and Resident Headers - - if attr_type in ATTRIBUTE_TYPE_ID: - vollog.debug(f'Found Attribute {ATTRIBUTE_TYPE_ID[attr_type]}') - - if ATTRIBUTE_TYPE_ID[attr_type] == 'STANDARD_INFORMATION': - creation_time_win = self.unpack_data(mft_record, attr_data, 'unsigned long long') - modified_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') - altered_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') - access_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') - flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') - permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - - mft_entry['attributes']['SI'] = { - "creation_time": conversion.wintime_to_datetime(creation_time_win), - "modified_time": conversion.wintime_to_datetime(modified_time_win), - "updated_time": conversion.wintime_to_datetime(altered_time_win), - "accessed_time": conversion.wintime_to_datetime(access_time_win), - "flags": permissions - } - - if ATTRIBUTE_TYPE_ID[attr_type] == 'FILE_NAME': - parent_record = self.unpack_data(mft_record, attr_data, 'unsigned long long') - creation_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') - modified_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') - altered_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') - access_time_win = self.unpack_data(mft_record, attr_data+32, 'unsigned long long') - - name_len = self.unpack_data(mft_record, attr_data+64, 'unsigned char') - name_space = self.unpack_data(mft_record, attr_data+65, 'unsigned char') - - # Unicode and partially corruprted records can break us here. - file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - #file_name = utility.array_to_string(file_name) - try: - file_name = file_name.replace(b'\x00', b'').decode() - except: - file_name = str(file_name.replace(b'\x00', b'')) - - flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') - permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - - mft_entry['attributes']['FN'].append( - { - "creation_time": conversion.wintime_to_datetime(creation_time_win), - "modified_time": conversion.wintime_to_datetime(modified_time_win), - "updated_time": conversion.wintime_to_datetime(altered_time_win), - "accessed_time": conversion.wintime_to_datetime(access_time_win), - "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), - "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), - "flags": permissions, - "file_name": file_name, - "name_space": name_space - }) - - # Update Offset for next Attribute - attr_offset += attr_len - - return mft_entry def _generator(self): - rules = yara.compile(sources = signatures) - layer = self.context.layers[self.config['primary']] + + # Yara Rule to scan for MFT Header Signatures + rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) + + # Read in the Symbol File + symbol_table = MFTIntermedSymbols.create( + self.context, + self.config_path, + "windows", + "mft" + ) + + # get each of the individual Field Sets + mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + header_object = symbol_table + constants.BANG + "ATTR_HEADER" + si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Scan the layer for Raw MFT records and parse the fields for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): - - # For each matching rule try to read 1024 bytes (size of an MFT record) at the offset. try: - mft_record = layer.read(offset, 1024, False) - mft_entry = self.parse_mft_record(mft_record) + mft_record = self.context.object(mft_object, offset=offset, layer_name=layer.name) + # We will update this on each pass in the next loop and use it as the new offset. + attr_base_offset = mft_record.FirstAttrOffset + + # There is no field that has a count of Attributes + # Keep Attempting to read attributes until we get an invalid attr_header.AttrType + while True: + attr_header = self.context.object(header_object, offset=offset+attr_base_offset, layer_name=layer.name) + attr_resident_header = self.context.object(header_object, offset=offset+attr_base_offset+16, layer_name=layer.name) + + vollog.debug(f"Attr Type: {attr_header.AttrType}") + + # If this is not a valid type then exit the loop + if not AttributeTypes(attr_header.AttrType).value: + break + + # Offset past the headers to the attribute data + attr_data_offset = offset+attr_base_offset+24 + + # Standard Information Attribute + if attr_header.AttrType == 0x10: + attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) + + yield 0, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + MFTFlags(mft_record.Flags).name, + renderers.NotApplicableValue(), + AttributeTypes(attr_header.AttrType).name, + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + renderers.NotApplicableValue(), + ) + + # File Name Attribute + if attr_header.AttrType == 0x30: + attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) + file_name = attr_data.get_full_name() + + yield 1, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + MFTFlags(mft_record.Flags).name, + PermissionFlags(attr_data.Flags).name, + AttributeTypes(attr_header.AttrType).name, + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + file_name + ) + + # Update the base offset to point to the next attribute + attr_base_offset += attr_header.Length + except exceptions.PagedInvalidAddressException: - mft_entry = None - #except Exception as err: - # vollog.error(err) - # mft_entry = None + pass - if mft_entry: - vollog.debug(mft_entry) - - # Tree Grid is large and variable - si = mft_entry['attributes']['SI'] - fn = mft_entry['attributes']['FN'] - - signature = mft_entry.get('signature', renderers.NotAvailableValue()) - record_number = mft_entry.get('record_number', renderers.NotAvailableValue()) - link_count = mft_entry.get('link_count', renderers.NotAvailableValue()) - permissions = mft_entry.get('flags', renderers.NotAvailableValue()) - - si_creation_time = si.get('creation_time', renderers.NotAvailableValue()) - si_modified_time = si.get('modified_time', renderers.NotAvailableValue()) - si_updated_time = si.get('updated_time', renderers.NotAvailableValue()) - si_accessed_time = si.get('accessed_time', renderers.NotAvailableValue()) - - yield 0, ( - format_hints.Hex(offset), - signature, - record_number, - link_count, - permissions, - 'Standard Information', - renderers.NotApplicableValue(), - si_creation_time, - si_modified_time, - si_updated_time, - si_accessed_time) - - for entry in fn: - # As this is variable and may or may not exist - # And could have 0-6 entries lets do it per row. - yield 1, ( - format_hints.Hex(offset), - signature, - record_number, - link_count, - permissions, - 'FileName', - entry.get('file_name',renderers.NotAvailableValue()), - entry.get('creation_time', renderers.NotAvailableValue()), - entry.get('modified_time', renderers.NotAvailableValue()), - entry.get('updated_time', renderers.NotAvailableValue()), - entry.get('accessed_time', renderers.NotAvailableValue())) def run(self): return renderers.TreeGrid([ @@ -307,11 +128,12 @@ class MFTScan(interfaces.plugins.PluginInterface): ('Record Type', str), ('Record Number', int), ('Link Count', int), + ('MFT Type', str), ('Permissions', str), ('Attribute Type', str), - ('Filename', str), ('Created', datetime.datetime), ('Modified', datetime.datetime), ('Updated', datetime.datetime), - ('Accessed', datetime.datetime) + ('Accessed', datetime.datetime), + ('Filename', str), ],self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py new file mode 100644 index 000000000..09f6346cc --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -0,0 +1,104 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import enum + +from volatility3.framework import exceptions, objects, renderers +from volatility3.framework.objects import utility + + +class AttributeTypes(enum.Enum): + STANDARD_INFORMATION = 0x10 + ATTRIBUTE_LIST = 0x20 + FILE_NAME = 0x30 + OBJECT_ID = 0x40 + SECURITY_DESCRIPTOR = 0x50 + VOLUME_NAME = 0x60 + VOLUME_INFORMATION = 0x70 + DATA = 0x80 + INDEX_ROOT = 0x90 + INDEX_ALLOCATION = 0xa0 + BITMAP = 0xb0 + REPARSE_POINT = 0xc0 + EA_INFORMATION = 0xd0 + EA = 0xe0 + PROPERTY_SET = 0xf0 + LOGGED_UTILITY_STREAM = 0x100 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(AttributeTypes.Unknown) + +class NameSpace(enum.Enum): + POSIX = 0x0 + Win32 = 0x1 + DOS = 0x2 + Win32DOS = 0x3 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(NameSpace.Unknown) + + +class MFTFlags(enum.Enum): + Removed = 0x00 + File = 0x1 + Directory = 0x2 + DirInUse = 0x3 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(MFTFlags.Unknown) + + +class PermissionFlags(enum.Enum): + ReadOnly = 0x1 + Hidden = 0x2 + System = 0x4 + Archive = 0x20 + ArchiveHidden = 0x22 + ArchiveSystem = 0x24 + ArchiveHiddenSystem = 0x26 + Device = 0x40 + Normal = 0x80 + Temporary = 0x100 + TempArchive = 0x120 + SparseFile = 0x200 + ReparsePoint = 0x400 + Compressed = 0x800 + Offline = 0x1000 + NotIndexed = 0x2000 + Encrypted = 0x4000 + Directory = 0x10000000 + IndexView = 0x20000000 + unknown = None + + @classmethod + def _missing_(cls, value): + return cls(PermissionFlags.unknown) + + +class MFTEntry(objects.StructType): + """This represents the base MFT Record""" + + def get_signature(self) -> str: + signature = self.Signature.cast('string', max_length = 4, encoding = 'latin-1') + return signature + + +class MFTFileName(objects.StructType): + """This represents an MFT $FILE_NAME Attribute""" + + def get_full_name(self) -> str: + output = self.Name.cast("string", + encoding = "utf16", + max_length = self.NameLength*2, + errors = "replace") + return output + + def get_file_namespace(self) -> str: + pass diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json new file mode 100644 index 000000000..e045ed0fc --- /dev/null +++ b/volatility3/framework/symbols/windows/mft.json @@ -0,0 +1,371 @@ +{ + "metadata": { + "producer": { + "version": "0.0.1", + "name": "kevthehermit-by-hand", + "comment": "Using structures defined in File System Forensic Analysis pg 353+", + "datetime": "2022-01-03T13:37:00" + }, + "format": "6.1.0" + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": true, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "int", + "size": 1, + "signed": false, + "endian": "little" + }, + "wchar": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + } + }, + "symbols": {}, + "enums": {}, + "user_types": { + "MFT_ENTRY": { + "fields": { + "Signature": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "UpdateSequenceOffset": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "NumFixupEntries": { + "offset": 6, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "LSN": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "SequenceValue": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "LinkCount": { + "offset": 18, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "FirstAttrOffset": { + "offset": 20, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "Flags": { + "offset": 22, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "RealSize": { + "offset": 24, + "type":{ + "kind": "base", + "name": "unsigned int" + } + }, + "AlocatedSize": { + "offset": 28, + "type":{ + "kind": "base", + "name": "unsigned int" + } + }, + "BaseReference": { + "offset": 32, + "type":{ + "kind": "base", + "name": "unsigned long long" + } + }, + "NextAttrID": { + "offset": 40, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "RecordNumber": { + "offset": 44, + "type":{ + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 1024 + },"ATTR_HEADER": { + "fields": { + "AttrType": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned int" + } + },"Length": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "NonResidentFlag": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned char" } + }, + "NameLength": { + "offset": 9, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "NameOffset": { + "offset": 10, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "Flags": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "AttributeID": { + "offset": 14, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 16 + },"RESIDENT_HEADER": { + "fields": { + "AttrSize": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned int" + } + },"AttrOffset": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "IndexFlag": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned short" } + } + }, + "kind": "struct", + "size": 8 + }, + "STANDARD_INFORMATION_ENTRY": { + "fields": { + "CreationTime": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ModifiedTime": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "UpdatedTime": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AccessedTime": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "flags": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 1024 + }, + "FILE_NAME_ENTRY": { + "fields": { + "ParentDirectory": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "CreationTime": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ModifiedTime": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "UpdatedTime": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AccessedTime": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AllocatedFileSize": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "RealFileSize": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Flags": { + "offset": 56, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ReparseValue": { + "offset": 60, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "NameLength": { + "offset": 64, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "NameSpace": { + "offset": 65, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "Name": { + "offset": 66, + "type": { + "count": 10, + "kind": "array", + "subtype": { + "kind": "base", + "name": "wchar" + } + } + } + }, + "kind": "struct", + "size": 1024 + } + } +} \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/mft.py b/volatility3/framework/symbols/windows/mft.py new file mode 100644 index 000000000..921d75cd8 --- /dev/null +++ b/volatility3/framework/symbols/windows/mft.py @@ -0,0 +1,15 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mft + + +class MFTIntermedSymbols(intermed.IntermediateSymbolTable): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.set_type_class('FILE_NAME_ENTRY', mft.MFTFileName) + self.set_type_class('MFT_ENTRY', mft.MFTEntry) From 793d08faf487c1d440bae016d8ca1e87766da7cc Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 22:27:32 +0000 Subject: [PATCH 046/404] Add TimeLiner interface to MFTScan plugin --- volatility3/framework/plugins/windows/mftscan.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 484fcdb9a..616c0d738 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -14,11 +14,11 @@ from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols -from volatility3.plugins import yarascan +from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) -class MFTScan(interfaces.plugins.PluginInterface): +class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) @@ -121,6 +121,18 @@ class MFTScan(interfaces.plugins.PluginInterface): except exceptions.PagedInvalidAddressException: pass + def generate_timeline(self): + for row in self._generator(): + if row[-1] != 'N/A': + filename = row[-1] + created = f'File {row[-1]} Created' + updated = f'File {row[-1]} Updated' + modified = f'File {row[-1]} Modified' + accessed = f'File {row[-1]} Accessed' + yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) + yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) + yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) + yield (f'File {filename} accessed', timeliner.TimeLinerType.ACCESSED, row[10]) def run(self): return renderers.TreeGrid([ From a0b66f33f90968b24e1cefab9bd7d7efd0b12638 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jan 2022 21:06:20 +0000 Subject: [PATCH 047/404] Documentation: Update README.md before release --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 75c3c1c23..f7d326c33 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ technical and performance challenges associated with the original code base that became apparent over the previous 10 years. Another benefit of the rewrite is that Volatility 3 could be released under a custom license that was more aligned with the goals of the Volatility community, -the Volatility Software License (VSL). See the [LICENSE](LICENSE.txt) file for more details. +the Volatility Software License (VSL). See the +[LICENSE](https://www.volatilityfoundation.org/license/vsl-v1.0) file for +more details. ## Requirements @@ -102,7 +104,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 12 Jan 2022 21:13:05 +0000 Subject: [PATCH 048/404] Documentation: Update master branch to stable branch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7d326c33..9f9c1bbb7 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ pip3 install -r requirements.txt ## Downloading Volatility -The latest stable version of Volatility will always be the master branch of the GitHub repository. You can get the latest version of the code using the following command: +The latest stable version of Volatility will always be the stable branch of the GitHub repository. You can get the latest version of the code using the following command: ```shell git clone https://github.com/volatilityfoundation/volatility3.git From f67f1e242d7f5cf1571864d052a545161849c8e2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jan 2022 22:11:25 +0000 Subject: [PATCH 049/404] Documentation: Ensure the doc reqs are included in s_dist builds --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 504c7d89a..1cec729f6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ prune development include * .* -include doc/make.bat doc/Makefile +include doc/make.bat doc/Makefile doc/requirements.txt recursive-include doc/source * recursive-include volatility3 *.json recursive-exclude doc/source volatility3.*.rst From d91a6f94fbe015a4b837efcbd68407a93a31b682 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 13 Jan 2022 01:01:20 +0000 Subject: [PATCH 050/404] Automagic: Ensure linxu/mac are excluded from windows automagic --- volatility3/framework/automagic/pdbscan.py | 1 + volatility3/framework/automagic/windows.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 93d4337da..8179339c8 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -44,6 +44,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): """ priority = 30 max_pdb_size = 0x400000 + exclusion_list = ['linux', 'mac'] def find_virtual_layers_from_req(self, context: interfaces.context.ContextInterface, config_path: str, requirement: interfaces.configuration.RequirementInterface) -> List[str]: diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index eb63a75e5..71548ca40 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -238,6 +238,8 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): """Class to read swap_layers filenames from single-swap-layers, create the layers and populate the single-layers swap_layers.""" + exclusion_list = ['linux', 'mac'] + def __call__(self, context: interfaces.context.ContextInterface, config_path: str, From 1c6cd0fb528b02e8f35ac65f1173241ff84dfe26 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 15:53:35 +0000 Subject: [PATCH 051/404] Move mftscan enums to ISF file. --- .../framework/plugins/windows/mftscan.py | 33 +++++--- .../symbols/windows/extensions/mft.py | 77 ------------------- .../framework/symbols/windows/mft.json | 70 ++++++++++++++++- 3 files changed, 93 insertions(+), 87 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 616c0d738..991191d5b 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -11,7 +11,6 @@ from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols from volatility3.plugins import timeliner, yarascan @@ -53,6 +52,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Get the Enums + attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") + namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") + mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") + permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): @@ -70,14 +75,20 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.debug(f"Attr Type: {attr_header.AttrType}") # If this is not a valid type then exit the loop - if not AttributeTypes(attr_header.AttrType).value: + if attr_header.AttrType not in attr_types.choices.values(): break # Offset past the headers to the attribute data attr_data_offset = offset+attr_base_offset+24 + + # MFT Flags determine the file type or dir + if mft_record.Flags in mft_flags.choices.values(): + mft_flag = mft_flags.lookup(mft_record.Flags) + else: + mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType == 0x10: + if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) yield 0, ( @@ -85,9 +96,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, - MFTFlags(mft_record.Flags).name, + mft_flag, renderers.NotApplicableValue(), - AttributeTypes(attr_header.AttrType).name, + attr_types.lookup(attr_header.AttrType), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -96,18 +107,22 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType == 0x30: + if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) file_name = attr_data.get_full_name() + if attr_data.Flags in permission_flags.choices.values(): + permissions = permission_flags.lookup(attr_data.Flags) + else: + permissions = hex(attr_data.Flags) yield 1, ( format_hints.Hex(attr_data_offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, - MFTFlags(mft_record.Flags).name, - PermissionFlags(attr_data.Flags).name, - AttributeTypes(attr_header.AttrType).name, + mft_flag, + permissions, + attr_types.lookup(attr_header.AttrType), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 09f6346cc..0713c969a 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -8,80 +8,6 @@ from volatility3.framework import exceptions, objects, renderers from volatility3.framework.objects import utility -class AttributeTypes(enum.Enum): - STANDARD_INFORMATION = 0x10 - ATTRIBUTE_LIST = 0x20 - FILE_NAME = 0x30 - OBJECT_ID = 0x40 - SECURITY_DESCRIPTOR = 0x50 - VOLUME_NAME = 0x60 - VOLUME_INFORMATION = 0x70 - DATA = 0x80 - INDEX_ROOT = 0x90 - INDEX_ALLOCATION = 0xa0 - BITMAP = 0xb0 - REPARSE_POINT = 0xc0 - EA_INFORMATION = 0xd0 - EA = 0xe0 - PROPERTY_SET = 0xf0 - LOGGED_UTILITY_STREAM = 0x100 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(AttributeTypes.Unknown) - -class NameSpace(enum.Enum): - POSIX = 0x0 - Win32 = 0x1 - DOS = 0x2 - Win32DOS = 0x3 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(NameSpace.Unknown) - - -class MFTFlags(enum.Enum): - Removed = 0x00 - File = 0x1 - Directory = 0x2 - DirInUse = 0x3 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(MFTFlags.Unknown) - - -class PermissionFlags(enum.Enum): - ReadOnly = 0x1 - Hidden = 0x2 - System = 0x4 - Archive = 0x20 - ArchiveHidden = 0x22 - ArchiveSystem = 0x24 - ArchiveHiddenSystem = 0x26 - Device = 0x40 - Normal = 0x80 - Temporary = 0x100 - TempArchive = 0x120 - SparseFile = 0x200 - ReparsePoint = 0x400 - Compressed = 0x800 - Offline = 0x1000 - NotIndexed = 0x2000 - Encrypted = 0x4000 - Directory = 0x10000000 - IndexView = 0x20000000 - unknown = None - - @classmethod - def _missing_(cls, value): - return cls(PermissionFlags.unknown) - - class MFTEntry(objects.StructType): """This represents the base MFT Record""" @@ -99,6 +25,3 @@ class MFTFileName(objects.StructType): max_length = self.NameLength*2, errors = "replace") return output - - def get_file_namespace(self) -> str: - pass diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index e045ed0fc..a99b82e9e 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -53,7 +53,75 @@ } }, "symbols": {}, - "enums": {}, + "enums": { + "AttrTypeEnum": { + "base": "unsigned char", + "constants": { + "STANDARD_INFORMATION": 16, + "ATTRIBUTE_LIST": 32, + "FILE_NAME": 48, + "OBJECT_ID": 64, + "SECURITY_DESCRIPTOR": 80, + "VOLUME_NAME": 96, + "VOLUME_INFORMATION": 112, + "DATA": 128, + "INDEX_ROOT": 114, + "INDEX_ALLOCATION": 160, + "BITMAP": 176, + "REPARSE_POINT": 192, + "EA_INFORMATION": 208, + "EA": 224, + "PROPERTY_SET": 240, + "LOGGED_UTILITY_STREAM": 256 + }, + "size": 1 + }, + "NameSpaceEnum": { + "base":"unsigned char", + "constants": { + "POSIX": 0, + "Win32": 1, + "DOS": 2, + "Win32 DOS": 3 + }, + "size": 1 + }, + "MFTFlagsEnum": { + "base":"unsigned char", + "constants": { + "Removed": 0, + "File": 1, + "Directory": 2, + "DirInUse": 3 + }, + "size": 1 + }, + "PermissionFlagEnum": { + "base":"unsigned char", + "constants": { + "ReadOnly": 1, + "Hidden": 2, + "System": 4, + "Archive": 32, + "ArchiveHidden": 34, + "ArchiveSystem": 36, + "ArchiveHiddenSystem": 38, + "Device": 60, + "Normal": 128, + "Temporary": 256, + "TempArchive": 288, + "SparseFile": 512, + "ReparsePoint": 1024, + "Compressed": 2048, + "Offline": 4096, + "NotIndexed": 8192, + "Encrypted": 16384, + "Directory": 268435456, + "IndexView": 536870912 + }, + "size": 1 + } + }, "user_types": { "MFT_ENTRY": { "fields": { From a8f5b0381664b963a565161d88c64fc1b4053102 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 15:57:23 +0000 Subject: [PATCH 052/404] Apply yapf to mftscan plugin --- .../framework/plugins/windows/mftscan.py | 84 +++++++++---------- .../symbols/windows/extensions/mft.py | 10 +-- 2 files changed, 41 insertions(+), 53 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 991191d5b..070061b17 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -17,6 +17,7 @@ from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) + class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" @@ -32,7 +33,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): version = (2, 0, 0)), ] - def _generator(self): layer = self.context.layers[self.config['primary']] @@ -40,12 +40,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) # Read in the Symbol File - symbol_table = MFTIntermedSymbols.create( - self.context, - self.config_path, - "windows", - "mft" - ) + symbol_table = MFTIntermedSymbols.create(self.context, self.config_path, "windows", "mft") # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" @@ -57,20 +52,26 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") - permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") - + permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + + "PermissionFlagEnum") + # Scan the layer for Raw MFT records and parse the fields - for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + for offset, rule_name, name, value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): try: - mft_record = self.context.object(mft_object, offset=offset, layer_name=layer.name) + mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while True: - attr_header = self.context.object(header_object, offset=offset+attr_base_offset, layer_name=layer.name) - attr_resident_header = self.context.object(header_object, offset=offset+attr_base_offset+16, layer_name=layer.name) + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) + attr_resident_header = self.context.object(header_object, + offset = offset + attr_base_offset + 16, + layer_name = layer.name) vollog.debug(f"Attr Type: {attr_header.AttrType}") @@ -79,17 +80,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset+attr_base_offset+24 + attr_data_offset = offset + attr_base_offset + 24 # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): mft_flag = mft_flags.lookup(mft_record.Flags) else: mft_flag = hex(mft_record.Flags) - + # Standard Information Attribute if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): - attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) + attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name) yield 0, ( format_hints.Hex(attr_data_offset), @@ -108,28 +109,21 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # File Name Attribute if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): - attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) + attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name) file_name = attr_data.get_full_name() if attr_data.Flags in permission_flags.choices.values(): permissions = permission_flags.lookup(attr_data.Flags) else: permissions = hex(attr_data.Flags) - yield 1, ( - format_hints.Hex(attr_data_offset), - mft_record.get_signature(), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - permissions, - attr_types.lookup(attr_header.AttrType), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - file_name - ) - + yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(), + mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, + attr_types.lookup(attr_header.AttrType), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), file_name) + # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length @@ -151,16 +145,16 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): def run(self): return renderers.TreeGrid([ - ('Offset', format_hints.Hex), - ('Record Type', str), - ('Record Number', int), - ('Link Count', int), - ('MFT Type', str), - ('Permissions', str), - ('Attribute Type', str), - ('Created', datetime.datetime), - ('Modified', datetime.datetime), - ('Updated', datetime.datetime), - ('Accessed', datetime.datetime), - ('Filename', str), - ],self._generator()) + ('Offset', format_hints.Hex), + ('Record Type', str), + ('Record Number', int), + ('Link Count', int), + ('MFT Type', str), + ('Permissions', str), + ('Attribute Type', str), + ('Created', datetime.datetime), + ('Modified', datetime.datetime), + ('Updated', datetime.datetime), + ('Accessed', datetime.datetime), + ('Filename', str), + ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 0713c969a..ba79b7c8b 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,10 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import enum - -from volatility3.framework import exceptions, objects, renderers -from volatility3.framework.objects import utility +from volatility3.framework import objects class MFTEntry(objects.StructType): @@ -20,8 +17,5 @@ class MFTFileName(objects.StructType): """This represents an MFT $FILE_NAME Attribute""" def get_full_name(self) -> str: - output = self.Name.cast("string", - encoding = "utf16", - max_length = self.NameLength*2, - errors = "replace") + output = self.Name.cast("string", encoding = "utf16", max_length = self.NameLength * 2, errors = "replace") return output From c5987a45d2362562329ca0e977ec87cf76babca9 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 20:54:22 +0000 Subject: [PATCH 053/404] Relative Offset MFT Header --- .../framework/plugins/windows/mftscan.py | 13 +++----- .../framework/symbols/windows/mft.json | 30 ++++++++++++++++++- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 070061b17..3d8d96221 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -44,13 +44,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Get the Enums attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") - namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") + namespace_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") @@ -69,9 +70,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): attr_header = self.context.object(header_object, offset = offset + attr_base_offset, layer_name = layer.name) - attr_resident_header = self.context.object(header_object, - offset = offset + attr_base_offset + 16, - layer_name = layer.name) vollog.debug(f"Attr Type: {attr_header.AttrType}") @@ -80,7 +78,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + 24 + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( + attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -134,10 +133,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for row in self._generator(): if row[-1] != 'N/A': filename = row[-1] - created = f'File {row[-1]} Created' - updated = f'File {row[-1]} Updated' - modified = f'File {row[-1]} Modified' - accessed = f'File {row[-1]} Accessed' yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index a99b82e9e..2470dcbd5 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -223,7 +223,35 @@ }, "kind": "struct", "size": 1024 - },"ATTR_HEADER": { + }, + "ATTRIBUTE": { + "fields":{ + "Attr_Header": { + "offset": 0, + "type": { + "kind": "struct", + "name": "mft!ATTR_HEADER" + } + }, + "Resident_Header": { + "offset": 16, + "type": { + "kind": "struct", + "name": "mft!RESIDENT_HEADER" + } + }, + "Attr_Data": { + "offset": 24, + "type": { + "kind": "struct", + "name": "mft!ATTR_HEADER" + } + } + }, + "kind": "struct", + "size": 96 + }, + "ATTR_HEADER": { "fields": { "AttrType": { "offset": 0, From 1f7acf2779047f9b5bf39e2de2ecf214c0c678a7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 21:02:20 +0000 Subject: [PATCH 054/404] Documentation: Update sphinx requirement to 4.0.0 --- doc/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index d646d22ce..93d6ea70a 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,4 @@ # These packages are required for building the documentation. -sphinx>=1.8.2 +sphinx>=4.0.0 sphinx_autodoc_typehints>=1.4.0 -sphinx-rtd-theme>=0.4.3 \ No newline at end of file +sphinx-rtd-theme>=0.4.3 From c93e20ab36ae6a988d9690d9ac44b165368b76dd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 21:14:52 +0000 Subject: [PATCH 055/404] Plugins: linux.kmsg update documentation and reformat --- volatility3/framework/plugins/linux/kmsg.py | 66 ++++++++++++--------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index a729c1695..8f4540766 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -4,9 +4,9 @@ import logging from abc import ABC, abstractmethod from enum import Enum -from typing import List, Iterator, Tuple, Generator +from typing import Generator, Iterator, List, Tuple -from volatility3.framework import renderers, interfaces, constants, contexts, class_subclasses +from volatility3.framework import class_subclasses, constants, contexts, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -15,39 +15,39 @@ vollog = logging.getLogger(__name__) class DescStateEnum(Enum): - desc_miss = -1 # ID mismatch (pseudo state) - desc_reserved = 0x0 # reserved, in use by writer - desc_committed = 0x1 # committed by writer, could get reopened - desc_finalized = 0x2 # committed, no further modification allowed - desc_reusable = 0x3 # free, not yet used by any writer + desc_miss = -1 # ID mismatch (pseudo state) + desc_reserved = 0x0 # reserved, in use by writer + desc_committed = 0x1 # committed by writer, could get reopened + desc_finalized = 0x2 # committed, no further modification allowed + desc_reusable = 0x3 # free, not yet used by any writer class ABCKmsg(ABC): """Kernel log buffer reader""" LEVELS = ( - "emerg", # system is unusable - "alert", # action must be taken immediately - "crit", # critical conditions - "err", # error conditions - "warn", # warning conditions - "notice", # normal but significant condition - "info", # informational - "debug", # debug-level messages + "emerg", # system is unusable + "alert", # action must be taken immediately + "crit", # critical conditions + "err", # error conditions + "warn", # warning conditions + "notice", # normal but significant condition + "info", # informational + "debug", # debug-level messages ) FACILITIES = ( - "kern", # kernel messages - "user", # random user-level messages - "mail", # mail system - "daemon", # system daemons - "auth", # security/authorization messages - "syslog", # messages generated internally by syslogd - "lpr", # line printer subsystem - "news", # network news subsystem - "uucp", # UUCP subsystem - "cron", # clock daemon + "kern", # kernel messages + "user", # random user-level messages + "mail", # mail system + "daemon", # system daemons + "auth", # security/authorization messages + "syslog", # messages generated internally by syslogd + "lpr", # line printer subsystem + "news", # network news subsystem + "uucp", # UUCP subsystem + "cron", # clock daemon "authpriv", # security/authorization messages (private) - "ftp" # FTP daemon + "ftp" # FTP daemon ) def __init__( @@ -247,12 +247,20 @@ class KmsgFiveTen(ABCKmsg): The data block ring 'text_data_ring' contains the records' text strings. A pointer to the high level structure is kept in the prb pointer which is initialized to a static ringbuffer. + + .. code-block:: c + static struct printk_ringbuffer *prb = &printk_rb_static; + In SMP systems with more than 64 CPUs this ringbuffer size is dynamically allocated according the number of CPUs based on the value of CONFIG_LOG_CPU_MAX_BUF_SHIFT. The prb pointer is updated consequently to this dynamic ringbuffer in setup_log_buf(). + + .. code-block:: c + prb = &printk_rb_dynamic; + Behind scenes, log_buf is still used as external buffer. When the static printk_ringbuffer struct is initialized, _DEFINE_PRINTKRB sets text_data_ring.data pointer to the address in log_buf which points to @@ -262,12 +270,14 @@ class KmsgFiveTen(ABCKmsg): buffer via the prb_init function. In that case, the original external static buffer in __log_buf and printk_rb_static are unused. - ... + + .. code-block:: c + new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN); prb_init(&printk_rb_dynamic, new_log_buf, ...); log_buf = new_log_buf; prb = &printk_rb_dynamic; - ... + See printk.c and printk_ringbuffer.c in kernel/printk/ folder for more details. """ From 8791631db5168b5f85ee72304f7d8906f915586b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 21:40:25 +0000 Subject: [PATCH 056/404] Documentation: More minor fixes --- doc/source/conf.py | 9 +++++++++ doc/source/volshell.rst | 20 ++++++++++---------- volatility3/framework/objects/__init__.py | 4 ++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index eab5c2c94..54c5da5ed 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -84,6 +84,15 @@ def setup(app): for line in submodule_lines: contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins')) + # Clear up the framework.plugins page + with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "rb") as contents: + real_lines = contents.readlines() + + with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "wb") as contents: + for line in real_lines: + if b'volatility3.framework.plugins.' not in line: + contents.write(line) + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 1b6846e6b..3d4cad890 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -22,7 +22,7 @@ be run. When volshell starts, it will show the version of volshell, a brief message indicating how to get more help, the current operating system mode for volshell, and the current layer available for use. -.. code-block:: python +:: Volshell (Volatility 3 Framework) 1.0.1 Readline imported successfully PDB scanning finished @@ -53,7 +53,7 @@ run our examples against. We'll start by creating a process variable, and putting the first result from `ps()` in it. Since the shell is a python environment, we can do the following: -.. code-block:: python +:: (primary) >>> proc = ps()[0] (primary) >>> proc @@ -68,7 +68,7 @@ built-in mechanism for providing more information about a structure, called `dis either a type name (which if not prefixed with symbol table name, will use the kernel symbol table identified by the automagic). -.. code-block:: python +:: (primary) >>> dt('_EPROCESS') nt_symbols1!_EPROCESS (2624 bytes) @@ -80,7 +80,7 @@ automagic). It can also be provided with an object and will interpret the data for each in the process: -.. code-block:: python +:: (primary) >>> dt(proc) nt_symbols1!_EPROCESS (2624 bytes) @@ -92,7 +92,7 @@ It can also be provided with an object and will interpret the data for each in t These values can be accessed directory as attributes -.. code-block:: python +:: (primary) >>> proc.UniqueProcessId 356 @@ -100,7 +100,7 @@ These values can be accessed directory as attributes Pointer structures contain the value they point to, but attributes accessed are forwarded to the object they point to. This means that pointers do not need to be explicitly dereferenced to access underling objects. -.. code-block:: python +:: (primary) >>> proc.Pcb.DirectoryTableBase 4355817472 @@ -112,7 +112,7 @@ It's possible to run any plugin by importing it appropriately and passing it to method. In the following example we'll provide no additional parameters. Volatility will show us which parameters were required: -.. code-block:: python +:: (primary) >>> from volatility3.plugins.windows import pslist (primary) >>> display_plugin_output(pslist.PsList) @@ -124,14 +124,14 @@ was fulfilled. We can see all the options that the plugin can accept by access the `get_requirements()` method of the plugin. This is a classmethod, so can be called on an uninstantiated copy of the plugin. -.. code-block:: python +:: (primary) >>> pslist.PsList.get_requirements() [, , , , ] We can provide arguments via the `dpo` method call: -.. code-block:: python +:: (primary) >>> display_plugin_output(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) @@ -149,7 +149,7 @@ by the `dpo` method is always `context`. Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by using the `generate_treegrid` or `gt` command. -.. code-block:: python +:: (primary) >>> treegrid = gt(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) (primary) >>> treegrid.populate() diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 6ee24407f..b5a7db286 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -6,9 +6,9 @@ import collections import collections.abc import logging import struct -from typing import Any, ClassVar, Dict, List, Iterable, Optional, Tuple, Type, Union as TUnion, overload +from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.objects import templates, utility vollog = logging.getLogger(__name__) From 4aaba89d024a06a00760978b2f8bb7f099087c72 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 21:47:37 +0000 Subject: [PATCH 057/404] Unity timeliner output for mftscan --- .../framework/plugins/windows/mftscan.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 3d8d96221..03f269735 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -78,8 +78,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( - attribute_object).relative_child_offset("Attr_Data") + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type(attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -130,13 +129,18 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pass def generate_timeline(self): + print("MFT Timeline") for row in self._generator(): - if row[-1] != 'N/A': - filename = row[-1] - yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) - yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) - yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) - yield (f'File {filename} accessed', timeliner.TimeLinerType.ACCESSED, row[10]) + _depth, row_data = row + + # Only Output FN Records + if row_data[6] == 'FILE_NAME': + filename = row_data[-1] + description = f"MFT FILE_NAME entry for {filename}" + yield (description, timeliner.TimeLinerType.CREATED, row_data[7]) + yield (description, timeliner.TimeLinerType.MODIFIED, row_data[8]) + yield (description, timeliner.TimeLinerType.CHANGED, row_data[9]) + yield (description, timeliner.TimeLinerType.ACCESSED, row_data[10]) def run(self): return renderers.TreeGrid([ From a9e5589260710a22f47ea2612f5a27ae8ffe87fc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:02:02 +0000 Subject: [PATCH 058/404] Documentation: Add summary table for linux/mac ISF creation --- doc/source/symbol-tables.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 31329b9eb..245dd9c67 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -76,3 +76,21 @@ The banners available for volatility to use can be found using the `isfinfo` plu long time to run depending on the number of JSON files available. This will list all the JSON (ISF) files that volatility3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON file, the banners must match exactly (down to the compilation date). + +.. note:: + + Steps for constructing a new kernel ISF JSON file: + + * Run the `banners` plugin on the image to determine the necessary kernel + * Locate a copy of the debug kernel that matches the identified banner + + * Clone or update the dwarf2json repo: :code:`git clone https://github.com/volatilityfoundation/dwarf2json` + * Run :code:`go build` in the directory if the source has changed + + * Run :code:`dwarf2json linux --elf [path to debug kernel] > [kernel name].json` + + * For Mac change `linux` to `mac` + + * Copy the `.json` file to the symbols directory into `[symbols directory]/linux` + + * For Mac change `linux` to `mac` \ No newline at end of file From 32abab8733d697ed86e3a6208d9c03ce08233117 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 22:38:05 +0000 Subject: [PATCH 059/404] Remove debug print from mftscan --- volatility3/framework/plugins/windows/mftscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 03f269735..7cc57f111 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -129,7 +129,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pass def generate_timeline(self): - print("MFT Timeline") for row in self._generator(): _depth, row_data = row From dab746aff07bc7afe021bd620822640d4f026ce6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:44:24 +0000 Subject: [PATCH 060/404] Plugins: yara python module check improvement Fixes #616 --- volatility3/framework/plugins/yarascan.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index e51669b2c..0ef55ff4b 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -15,8 +15,11 @@ vollog = logging.getLogger(__name__) try: import yara + + if tuple([int(x) for x in yara.__version__.split('.')]) < (3, 8): + raise ImportError except ImportError: - vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") + vollog.info("Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available") raise From d9ba3e6bd9b71f0c9bc1973ada12c7bab104e729 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:50:21 +0000 Subject: [PATCH 061/404] Plugins: Timeliner improve support for body files --- volatility3/framework/plugins/timeliner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 6bf592504..0ae87e274 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -12,7 +12,7 @@ import traceback from typing import Generator, Iterable, List, Optional, Tuple, Type from volatility3 import framework -from volatility3.framework import renderers, automagic, interfaces, plugins, exceptions +from volatility3.framework import automagic, exceptions, interfaces, plugins, renderers from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -145,7 +145,7 @@ class Timeliner(interfaces.plugins.PluginInterface): # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime if self._any_time_present(times): - fp.write("|{} - {}||||||{}|{}|{}|{}\n".format( + fp.write("|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format( plugin_name, self._sanitize_body_format(item), self._text_format(times.get(TimeLinerType.ACCESSED, "")), self._text_format(times.get(TimeLinerType.MODIFIED, "")), @@ -202,7 +202,7 @@ class Timeliner(interfaces.plugins.PluginInterface): if isinstance(plugin, TimeLinerInterface): if not len(filter_list) or any( - [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]): + [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]): plugins_to_run.append(plugin) except exceptions.UnsatisfiedException as excp: # Remove the failed plugin from the list and continue From 742d46786bc0b48781343584204bd51a6c316da9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:56:36 +0000 Subject: [PATCH 062/404] Plugins: Don't forget missing values --- volatility3/framework/plugins/timeliner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 0ae87e274..faa99ff67 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -164,7 +164,7 @@ class Timeliner(interfaces.plugins.PluginInterface): def _text_format(self, value): """Formats a value as text, in case it is an AbsentValue""" if isinstance(value, interfaces.renderers.BaseAbsentValue): - return "" + return "0" if isinstance(value, datetime.datetime): return int(value.timestamp()) return value From f31bc853c4d7eb05041c854f26704dea21656d0d Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 23:30:39 +0000 Subject: [PATCH 063/404] remove MFTIntermedSymbols --- .../framework/plugins/windows/mftscan.py | 43 +++++++++++-------- volatility3/framework/symbols/windows/mft.py | 15 ------- 2 files changed, 25 insertions(+), 33 deletions(-) delete mode 100644 volatility3/framework/symbols/windows/mft.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 7cc57f111..6487ffe97 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,12 @@ import datetime import logging -from typing import Dict - from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mft from volatility3.plugins import timeliner, yarascan @@ -40,7 +39,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) # Read in the Symbol File - symbol_table = MFTIntermedSymbols.create(self.context, self.config_path, "windows", "mft") + symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, + config_path = self.config_path, + sub_path = "windows", + filename = "mft", + class_types = { + 'FILE_NAME_ENTRY': mft.MFTFileName, + 'MFT_ENTRY': mft.MFTEntry + }) # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" @@ -57,28 +63,25 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields - for offset, rule_name, name, value in layer.scan(context = self.context, - scanner = yarascan.YaraScanner(rules = rules)): + for offset, _rule_name, _name, _value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): try: mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) + # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while True: - attr_header = self.context.object(header_object, - offset = offset + attr_base_offset, - layer_name = layer.name) - + while attr_header.AttrType in attr_types.choices.values(): vollog.debug(f"Attr Type: {attr_header.AttrType}") - # If this is not a valid type then exit the loop - if attr_header.AttrType not in attr_types.choices.values(): - break - # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type(attribute_object).relative_child_offset("Attr_Data") + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( + attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -124,9 +127,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length + # Get the next attribute + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) - except exceptions.PagedInvalidAddressException: - pass + except Exception as err: + vollog.debug(f'Error Parsing MFT Record: {err}') def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/symbols/windows/mft.py b/volatility3/framework/symbols/windows/mft.py deleted file mode 100644 index 921d75cd8..000000000 --- a/volatility3/framework/symbols/windows/mft.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows.extensions import mft - - -class MFTIntermedSymbols(intermed.IntermediateSymbolTable): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.set_type_class('FILE_NAME_ENTRY', mft.MFTFileName) - self.set_type_class('MFT_ENTRY', mft.MFTEntry) From a31f846b14ffb4ffe529f1aae93256ba2c8e2d1e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Jan 2022 00:47:40 +0000 Subject: [PATCH 064/404] Documentation: Reorganize and consolidate pages --- doc/source/development.rst | 8 ++++++++ doc/source/index.rst | 6 ++---- 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 doc/source/development.rst diff --git a/doc/source/development.rst b/doc/source/development.rst new file mode 100644 index 000000000..ade068322 --- /dev/null +++ b/doc/source/development.rst @@ -0,0 +1,8 @@ +Writing Plugins +=============== + +.. toctree:: + + simple-plugin + complex-plugin + using-as-a-library diff --git a/doc/source/index.rst b/doc/source/index.rst index 0dc3b5025..3b5a5d2a8 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -12,11 +12,9 @@ Here are some guidelines for using Volatility 3 effectively: .. toctree:: basics - simple-plugin - vol2to3 - complex-plugin - using-as-a-library + development symbol-tables + vol2to3 volshell glossary From e3a7ac566840ee052ffbefe3fb370e4142704706 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 16 Jan 2022 02:20:55 +0000 Subject: [PATCH 065/404] Use lookups on mft enums instead of choices --- .../framework/plugins/windows/mftscan.py | 39 +++++++++---------- .../framework/symbols/windows/mft.json | 22 +++++------ 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 6487ffe97..16f0c9c95 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -55,12 +55,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" - # Get the Enums - attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") - namespace_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") - mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") - permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + - "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan(context = self.context, @@ -74,23 +68,26 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) + # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr_header.AttrType in attr_types.choices.values(): - vollog.debug(f"Attr Type: {attr_header.AttrType}") + + while attr_header.AttrType.is_valid_choice: + vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") # Offset past the headers to the attribute data attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir - if mft_record.Flags in mft_flags.choices.values(): - mft_flag = mft_flags.lookup(mft_record.Flags) - else: + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + mft_flag = mft_record.Flags.lookup() + except ValueError: mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): + if attr_header.AttrType.lookup() == 'STANDARD_INFORMATION': attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name) yield 0, ( @@ -100,7 +97,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_record.LinkCount, mft_flag, renderers.NotApplicableValue(), - attr_types.lookup(attr_header.AttrType), + attr_header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -109,17 +106,19 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): + if attr_header.AttrType.lookup() == 'FILE_NAME': attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name) file_name = attr_data.get_full_name() - if attr_data.Flags in permission_flags.choices.values(): - permissions = permission_flags.lookup(attr_data.Flags) - else: + + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + permissions = attr_data.Flags.lookup() + except ValueError: permissions = hex(attr_data.Flags) yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, - attr_types.lookup(attr_header.AttrType), + attr_header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -132,8 +131,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) - except Exception as err: - vollog.debug(f'Error Parsing MFT Record: {err}') + except exceptions.PagedInvalidAddressException: + pass def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index 2470dcbd5..b71be6444 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -181,8 +181,8 @@ "Flags": { "offset": 22, "type":{ - "kind": "base", - "name": "unsigned short" + "kind": "enum", + "name": "MFTFlagsEnum" } }, "RealSize": { @@ -256,8 +256,8 @@ "AttrType": { "offset": 0, "type": { - "kind": "base", - "name": "unsigned int" + "kind": "enum", + "name": "AttrTypeEnum" } },"Length": { "offset": 4, @@ -289,9 +289,9 @@ "Flags": { "offset": 12, "type": { - "kind": "base", - "name": "unsigned short" - } + "kind": "enum", + "name": "MFTFlagsEnum" + } }, "AttributeID": { "offset": 14, @@ -361,8 +361,8 @@ "flags": { "offset": 32, "type": { - "kind": "base", - "name": "unsigned short" + "kind": "enum", + "name": "PermissionFlagEnum" } } }, @@ -423,8 +423,8 @@ "Flags": { "offset": 56, "type": { - "kind": "base", - "name": "unsigned int" + "kind": "enum", + "name": "PermissionFlagEnum" } }, "ReparseValue": { From 57c9470f66f25726eaa2e6c5f87ac8a29e8dab68 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Jan 2022 22:09:02 +0000 Subject: [PATCH 066/404] Documentation: Fix building from different directories --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 54c5da5ed..6030c6308 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -85,10 +85,10 @@ def setup(app): contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins')) # Clear up the framework.plugins page - with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "rb") as contents: + with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "rb") as contents: real_lines = contents.readlines() - with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "wb") as contents: + with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "wb") as contents: for line in real_lines: if b'volatility3.framework.plugins.' not in line: contents.write(line) From 64acff1b59d7f85a1a187bf45875a8790029407e Mon Sep 17 00:00:00 2001 From: "Nick L. Petroni, Jr" Date: Sun, 16 Jan 2022 15:34:03 -0500 Subject: [PATCH 067/404] use doc/requirements.txt when building with readthedocs --- .readthedocs.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 764bb5a1a..4d21d9b40 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -16,7 +16,4 @@ formats: all python: version: 3.7 install: - - method: pip - path: . - extra_requirements: - - doc + - requirements: doc/requirements.txt From fa2a608cd296e925d9e9847c9ac55cff34bb5f6a Mon Sep 17 00:00:00 2001 From: "Nick L. Petroni, Jr" Date: Sun, 16 Jan 2022 17:32:02 -0500 Subject: [PATCH 068/404] update doc copyright --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 6030c6308..731a73d56 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # # @@ -135,7 +135,7 @@ master_doc = 'index' # General information about the project. project = 'Volatility 3' -copyright = '2012-2019, Volatility Foundation' +copyright = '2012-2022, Volatility Foundation' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 0f23089cb483484ce1eed54ca87d5e78825d02e3 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Mon, 17 Jan 2022 00:28:59 +0000 Subject: [PATCH 069/404] Create Sessions Plugin --- .../framework/plugins/windows/sessions.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 volatility3/framework/plugins/windows/sessions.py diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py new file mode 100644 index 000000000..cd02668ae --- /dev/null +++ b/volatility3/framework/plugins/windows/sessions.py @@ -0,0 +1,96 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import datetime +import logging + +from volatility3.framework import renderers, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + + +class Sessions(interfaces.plugins.PluginInterface): + """lists Processes with Session information extracted from Environmental Variables""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True) + ] + + def _generator(self, procs): + + # Collect all the values as we will want to group them later + sessions = {} + + for proc in procs: + + session_id = proc.get_session_id() + + # Detect RDP, Console or set default value + session_type = renderers.NotAvailableValue() + + # Construct Username from Process Env + user_domain = '' + user_name = '' + + for var, val in proc.environment_variables(): + if var.lower() == 'username': + user_name = val + elif var.lower() == 'userdomain': + user_domain = val + if var.lower() == 'sessionname': + session_type = val + + # Concat Domain and User + full_user = f'{user_domain}/{user_name}' + if full_user == '/': + full_user = renderers.NotAvailableValue() + + # Collect all the values in to a row we can yield after sorting. + row = { + "session_id": session_id, + "process_id": proc.UniqueProcessId, + "process_name": utility.array_to_string(proc.ImageFileName), + "user_name": full_user, + "process_start": proc.get_create_time(), + "session_type": session_type + } + + # Add row to correct session so we can sort it later + if session_id in sessions: + sessions[session_id].append(row) + else: + sessions[session_id] = [row] + + # Group and yield each row + for rows in sessions.values(): + for row in rows: + yield 0, (row.get('session_id'), row.get('session_type'), row.get('process_id'), + row.get('process_name'), row.get('user_name'), row.get('process_start')) + + def run(self): + + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + + return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), + ("User Name", str), ("Create Time", datetime.datetime)], + self._generator( + pslist.PsList.list_processes(self.context, + self.config['primary'], + self.config['nt_symbols'], + filter_func = filter_func))) From be0dd49d314bc6d4b2aa907928f0fb7123cb5f43 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Tue, 18 Jan 2022 19:34:06 +0000 Subject: [PATCH 070/404] Sessions Plugin use ModuleRequirement --- .../framework/plugins/windows/sessions.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py index cd02668ae..66a54c3f0 100644 --- a/volatility3/framework/plugins/windows/sessions.py +++ b/volatility3/framework/plugins/windows/sessions.py @@ -21,10 +21,9 @@ class Sessions(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', element_type = int, @@ -32,12 +31,17 @@ class Sessions(interfaces.plugins.PluginInterface): optional = True) ] - def _generator(self, procs): + def _generator(self): + kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) # Collect all the values as we will want to group them later sessions = {} - for proc in procs: + for proc in pslist.PsList.list_processes(self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func = filter_func): session_id = proc.get_session_id() @@ -56,7 +60,7 @@ class Sessions(interfaces.plugins.PluginInterface): if var.lower() == 'sessionname': session_type = val - # Concat Domain and User + # Concat Domain and User full_user = f'{user_domain}/{user_name}' if full_user == '/': full_user = renderers.NotAvailableValue() @@ -85,12 +89,5 @@ class Sessions(interfaces.plugins.PluginInterface): def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), - ("User Name", str), ("Create Time", datetime.datetime)], - self._generator( - pslist.PsList.list_processes(self.context, - self.config['primary'], - self.config['nt_symbols'], - filter_func = filter_func))) + ("User Name", str), ("Create Time", datetime.datetime)], self._generator()) From ac7cecf231e98c7f104d4c2b8fb569cafb977409 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Tue, 18 Jan 2022 20:42:38 +0000 Subject: [PATCH 071/404] add timeliner output to windows.sessions --- volatility3/framework/plugins/windows/sessions.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py index 66a54c3f0..6745e95ec 100644 --- a/volatility3/framework/plugins/windows/sessions.py +++ b/volatility3/framework/plugins/windows/sessions.py @@ -9,11 +9,12 @@ from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.plugins.windows import pslist +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class Sessions(interfaces.plugins.PluginInterface): +class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """lists Processes with Session information extracted from Environmental Variables""" _required_framework_version = (2, 0, 0) @@ -87,6 +88,15 @@ class Sessions(interfaces.plugins.PluginInterface): yield 0, (row.get('session_id'), row.get('session_type'), row.get('process_id'), row.get('process_name'), row.get('user_name'), row.get('process_start')) + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + # Only add to timeline if we have the username + # Without the user context PSList output is identical + if isinstance(row_data[4], str): + description = f"Process: {row_data[2]} {row_data[3]} started by user {row_data[4]}" + yield (description, timeliner.TimeLinerType.CREATED, row_data[5]) + def run(self): return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), From 0f90f7f6ca5b3b120f3c680aa2993ec4c5246b4c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 18 Jan 2022 20:58:48 +0000 Subject: [PATCH 072/404] Plugins: Remove unused timeliner parameter --- volatility3/framework/plugins/timeliner.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 6bf592504..f7ec340ec 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -74,10 +74,6 @@ class Timeliner(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.StringRequirement(name = 'plugins', - description = "Comma separated list of plugins to run", - optional = True, - default = None), requirements.BooleanRequirement( name = 'record-config', description = "Whether to record the state of all the plugins once complete", From 7c238f93a06b496826271e61a010be9d2c4a9661 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 31 Jan 2022 20:15:39 +0900 Subject: [PATCH 073/404] Update __init__.py Correcting typos for Windows Constants --- volatility3/framework/constants/windows/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/windows/__init__.py b/volatility3/framework/constants/windows/__init__.py index 372897598..a19216605 100644 --- a/volatility3/framework/constants/windows/__init__.py +++ b/volatility3/framework/constants/windows/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""Volatility 3 Linux Constants. +"""Volatility 3 Windows Constants. Windows-specific values that aren't found in debug symbols """ From 338fcdd0ab7f59491017ed63d8e780e744c0fe88 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Wed, 2 Feb 2022 15:48:41 +0000 Subject: [PATCH 074/404] Add the psaux plugin for Linux command line argument listing --- volatility3/framework/plugins/linux/psaux.py | 90 ++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 volatility3/framework/plugins/linux/psaux.py diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py new file mode 100644 index 000000000..31777f458 --- /dev/null +++ b/volatility3/framework/plugins/linux/psaux.py @@ -0,0 +1,90 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from typing import Optional + +from volatility3.framework import symbols, exceptions, renderers, interfaces +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist + +class PsAux(pslist.PsList): + """ Lists processes with their command line arguments """ + + def _get_command_line_args(self, task: interfaces.objects.ObjectInterface, + name: str) -> Optional[str]: + """ + Reads the command line arguments of a process + These are stored on the userland stack + Kernel threads re-use the process data structure, but do not have a valid 'mm' pointer + + Parameters: + task: task_struct object of the process + name: string name of the process (from task.comm) + """ + + # kernel theads never have an mm as they do not have userland mappings + try: + mm = task.mm + except exceptions.InvalidAddressException: + mm = None + + if mm: + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + return renderers.UnreadableValue() + + proc_layer = self.context.layers[proc_layer_name] + + # read argv from userland + start = task.mm.arg_start + + # get the size of the arguments with sanity checking + size_to_read = task.mm.arg_end - task.mm.arg_start + if size_to_read < 1 or size_to_read > 4096: + return renderers.UnreadableValue() + + # attempt to read it all as partial values are invalid and misleading + try: + argv = proc_layer.read(start, size_to_read) + except exceptions.InvalidAddressException: + return renderers.UnreadableValue() + + # the arguments are null byte terminated, replace the nulls with spaces + s = argv.decode().split('\x00') + args = " ".join(s) + else: + # kernel thread + # [ ] mimics ps on a live system + # also helps identify malware masquerading as a kernel thread, which is fairly common + args = "[" + name + "]" + + # remove trailing space, if present + if len(args) > 1 and args[-1] == " ": + args = args[:-1] + + return args + + def _generator(self): + """ Generates a listing of processes along with command line arguments """ + + vmlinux = self.context.modules[self.config['kernel']] + + # walk the process list and report the arguments + for task in self.list_tasks(self.context, vmlinux.name): + pid = task.pid + + try: + ppid = task.parent.pid + except exceptions.InvalidAddressException: + ppid = 0 + + name = utility.array_to_string(task.comm) + + args = self._get_command_line_args(task, name) + + yield (0, (pid, ppid, name, args)) + + def run(self): + return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], self._generator()) + From a67121f1c17fd18acd13c4f7a63463b2b9fc5c8f Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 9 Feb 2022 10:40:37 +0200 Subject: [PATCH 075/404] Added data_offset to pattern matching result, fixes pdb scanning bug --- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 4c3788a56..ce492831e 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -357,4 +357,4 @@ class PdbSignatureScanner(interfaces.layers.ScannerInterface): guid = (16 * '{:02X}').format(g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, ga, gb, gc, gd, ge, gf) if match.start(0) < self.chunk_size: - yield (guid, a, pdb_name, match.start(0)) + yield (guid, a, pdb_name, data_offset + match.start(0)) From ce166b92637edb78abcc7e87139449019c6abaf6 Mon Sep 17 00:00:00 2001 From: trashcan122 Date: Wed, 9 Feb 2022 16:30:40 +0200 Subject: [PATCH 076/404] fix pdb mz parsing --- volatility3/framework/symbols/windows/pdbutil.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index ce492831e..cc7b22e03 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -131,8 +131,14 @@ class PDBUtility(interfaces.configuration.VersionableInterface): # Check it is actually the MZ header if mz_sig != b"MZ": return None - - nt_header_start = ord(layer.read(offset + 0x3C, 1)) + + nt_header_start = struct.unpack(" Date: Wed, 9 Feb 2022 20:07:13 +0000 Subject: [PATCH 077/404] Core: Slight speed-up for all single struct.unpack calls --- volatility3/framework/symbols/windows/extensions/pool.py | 4 ++-- .../framework/symbols/windows/extensions/registry.py | 9 ++++++--- volatility3/framework/symbols/windows/pdbutil.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index f75c6c417..d50d6e47a 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -128,8 +128,8 @@ class POOL_HEADER(objects.StructType): # --------------- if addr - optional_headers_length < 0: continue - padding_length = struct.unpack( - "L"): raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}") - return struct.unpack(">L", data)[0] + res, = struct.unpack(">L", data) + return res if self_type == RegValueTypes.REG_QWORD: if len(data) != struct.calcsize(" Date: Wed, 9 Feb 2022 20:19:17 +0000 Subject: [PATCH 078/404] Core: Bump the development version to 2.0.2 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 82ebd4936..665e62d30 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From edbbc0159fa7beeea60c72a6e32758ec79d9de17 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 10 Feb 2022 10:31:18 +0200 Subject: [PATCH 079/404] bug fix --- volatility3/framework/interfaces/objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 811327094..e589abd15 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -115,7 +115,7 @@ class ObjectInterface(metaclass = abc.ABCMeta): mask = context.layers[object_info.layer_name].address_mask normalized_offset = object_info.offset & mask - self._vol = collections.ChainMap({}, object_info, {'type_name': type_name, 'offset': normalized_offset}, kwargs) + self._vol = collections.ChainMap({}, {'type_name': type_name, 'offset': normalized_offset}, object_info, kwargs) self._context = context def __getattr__(self, attr: str) -> Any: From cf11de174ab9ec019f09bf2bdc851619cf1ec509 Mon Sep 17 00:00:00 2001 From: pudii Date: Fri, 11 Feb 2022 18:19:53 +0100 Subject: [PATCH 080/404] Implement LDRmodules plugin --- .../framework/plugins/windows/ldrmodules.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 volatility3/framework/plugins/windows/ldrmodules.py diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py new file mode 100644 index 000000000..77678a4c0 --- /dev/null +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -0,0 +1,97 @@ +from volatility3.framework import interfaces, constants +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, vadinfo + +class LdrModules(interfaces.plugins.PluginInterface): + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), + requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), + requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True), + ] + + def _generator(self, procs): + + pe_table_name = intermed.IntermediateSymbolTable.create(self.context, + self.config_path, + "windows", + "pe", + class_types = pe.class_types) + + def filter_function(x: interfaces.objects.ObjectInterface) -> bool: + try: + return not (x.get_private_memory() == 0 and x.ControlArea) + except AttributeError: + return False + + filter_func = filter_function + + for proc in procs: + proc_layer_name = proc.add_process_layer() + + load_order_mod = dict((mod.DllBase, mod) + for mod in proc.load_order_modules()) + init_order_mod = dict((mod.DllBase, mod) + for mod in proc.init_order_modules()) + mem_order_mod = dict((mod.DllBase, mod) + for mod in proc.mem_order_modules()) + + mapped_files = {} + for vad in vadinfo.VadInfo.list_vads(proc, filter_func = filter_func): + dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset = vad.get_start(), + layer_name = proc_layer_name) + try: + if dos_header.e_magic != 0x5A4D: + continue + except exceptions.PagedInvalidAddressException: + continue + + mapped_files[int(vad.get_start())] = str(vad.get_file_name() or '') + + for base in mapped_files.keys(): + # Does the base address exist in the PEB DLL lists? + load_mod = load_order_mod.get(base, None) + init_mod = init_order_mod.get(base, None) + mem_mod = mem_order_mod.get(base, None) + + yield (0, [int(proc.UniqueProcessId), + str(proc.ImageFileName.cast("string", + max_length = proc.ImageFileName.vol.count, + errors = 'replace')), + format_hints.Hex(base), + str(load_mod != None), + str(init_mod != None), + str(mem_mod != None), + str(mapped_files[base])]) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] + + return renderers.TreeGrid([("Pid", int), + ("Process", str), + ("Base", format_hints.Hex), + ("InLoad", str), + ("InInit", str), + ("InMem", str), + ("MappedPath", str)], + self._generator( + pslist.PsList.list_processes(context = self.context, + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, + filter_func = filter_func))) \ No newline at end of file From 9cabf5362b66266a34f861af5e89d0315fe92124 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Feb 2022 00:19:27 +0000 Subject: [PATCH 081/404] Timeliner: Write out directly to the body file Since the body file doesn't need sorting, we can output it immediately, and this also means that partial results can be recorded even in the run is terminted before it compeletes. Goes someway to improving #646 --- volatility3/framework/plugins/timeliner.py | 33 +++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 01f775eb3..8785f62e1 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -106,6 +106,15 @@ class Timeliner(interfaces.plugins.PluginInterface): row from each plugin.""" # Generate the results for each plugin data = [] + + # Open the bodyfile now, so we can start outputting to it immediately + if self.config.get('create-bodyfile', True): + file_data = self.open("volatility.body") + fp = io.TextIOWrapper(file_data, write_through = True) + else: + file_data = None + fp = None + for plugin in runable_plugins: plugin_name = plugin.__class__.__name__ self._progress_callback((runable_plugins.index(plugin) * 100) // len(runable_plugins), @@ -126,17 +135,9 @@ class Timeliner(interfaces.plugins.PluginInterface): times.get(TimeLinerType.ACCESSED, renderers.NotApplicableValue()), times.get(TimeLinerType.CHANGED, renderers.NotApplicableValue()) ])) - except Exception: - vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}") - vollog.log(logging.DEBUG, traceback.format_exc()) - for data_item in sorted(data, key = self._sort_function): - yield data_item - # Write out a body file if necessary - if self.config.get('create-bodyfile', True): - with self.open("volatility.body") as file_data: - with io.TextIOWrapper(file_data, write_through = True) as fp: - for (plugin_name, item) in self.timeline: + # Write each entry because the body file doesn't need to be sorted + if fp: times = self.timeline[(plugin_name, item)] # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime @@ -147,6 +148,18 @@ class Timeliner(interfaces.plugins.PluginInterface): self._text_format(times.get(TimeLinerType.MODIFIED, "")), self._text_format(times.get(TimeLinerType.CHANGED, "")), self._text_format(times.get(TimeLinerType.CREATED, "")))) + except Exception: + vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}") + vollog.log(logging.DEBUG, traceback.format_exc()) + + for data_item in sorted(data, key = self._sort_function): + yield data_item + + # Write out a body file if necessary + if self.config.get('create-bodyfile', True): + if fp: + fp.close() + file_data.close() def _sanitize_body_format(self, value): return value.replace("|", "_") From 12c3f340370f5e4dd1cb34d6803154c23164d234 Mon Sep 17 00:00:00 2001 From: pudii Date: Sun, 13 Feb 2022 18:29:35 +0100 Subject: [PATCH 082/404] Add comments and fix minor code issues --- .../framework/plugins/windows/ldrmodules.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 77678a4c0..42eeacd4d 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -15,7 +15,6 @@ class LdrModules(interfaces.plugins.PluginInterface): def get_requirements(cls): return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', @@ -43,6 +42,7 @@ class LdrModules(interfaces.plugins.PluginInterface): for proc in procs: proc_layer_name = proc.add_process_layer() + # Build dictionaries from different module lists, where the DllBase address is the key and value is the module object load_order_mod = dict((mod.DllBase, mod) for mod in proc.load_order_modules()) init_order_mod = dict((mod.DllBase, mod) @@ -50,18 +50,20 @@ class LdrModules(interfaces.plugins.PluginInterface): mem_order_mod = dict((mod.DllBase, mod) for mod in proc.mem_order_modules()) + # Build dictionary of mapped files, where the VAD start address is the key and value is the file name of the mapped file mapped_files = {} for vad in vadinfo.VadInfo.list_vads(proc, filter_func = filter_func): dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", offset = vad.get_start(), layer_name = proc_layer_name) try: + # Filter out VADs that do not start with a MZ header if dos_header.e_magic != 0x5A4D: continue except exceptions.PagedInvalidAddressException: continue - mapped_files[int(vad.get_start())] = str(vad.get_file_name() or '') + mapped_files[vad.get_start()] = vad.get_file_name() for base in mapped_files.keys(): # Does the base address exist in the PEB DLL lists? @@ -74,10 +76,10 @@ class LdrModules(interfaces.plugins.PluginInterface): max_length = proc.ImageFileName.vol.count, errors = 'replace')), format_hints.Hex(base), - str(load_mod != None), - str(init_mod != None), - str(mem_mod != None), - str(mapped_files[base])]) + load_mod != None, + init_mod != None, + mem_mod != None, + mapped_files[base]]) def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) @@ -86,12 +88,12 @@ class LdrModules(interfaces.plugins.PluginInterface): return renderers.TreeGrid([("Pid", int), ("Process", str), ("Base", format_hints.Hex), - ("InLoad", str), - ("InInit", str), - ("InMem", str), + ("InLoad", bool), + ("InInit", bool), + ("InMem", bool), ("MappedPath", str)], self._generator( pslist.PsList.list_processes(context = self.context, layer_name = kernel.layer_name, symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) \ No newline at end of file + filter_func = filter_func))) From 22a7328e4cba83bbb6965d9705b22a1b5755d626 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Feb 2022 20:45:42 +0000 Subject: [PATCH 083/404] Windows: Prevent infinite loop in mftscan --- volatility3/framework/plugins/windows/mftscan.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 16f0c9c95..654e26db7 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,11 @@ import datetime import logging -from volatility3.framework import constants, renderers, interfaces +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import mft - from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) @@ -55,7 +53,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" - # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): @@ -68,10 +65,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) - # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - + while attr_header.AttrType.is_valid_choice: vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") @@ -124,6 +120,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): conversion.wintime_to_datetime(attr_data.UpdatedTime), conversion.wintime_to_datetime(attr_data.AccessedTime), file_name) + # If there's no advancement the loop will never end, so break it now + if attr_header.Length == 0: + break + # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length # Get the next attribute From 094a3c0a474dedc13f17314ef41b083ef0dbbeb2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 17 Feb 2022 00:21:59 +0000 Subject: [PATCH 084/404] Volshell: Update to use KernelRequirement --- doc/source/volshell.rst | 66 ++++++++++++++-------------- volatility3/cli/__init__.py | 4 +- volatility3/cli/volshell/__init__.py | 3 +- volatility3/cli/volshell/generic.py | 29 ++++++++---- volatility3/cli/volshell/linux.py | 26 ++++++++--- volatility3/cli/volshell/mac.py | 28 +++++++++--- volatility3/cli/volshell/windows.py | 30 ++++++++++--- 7 files changed, 123 insertions(+), 63 deletions(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 3d4cad890..1e51f90ba 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -24,13 +24,14 @@ operating system mode for volshell, and the current layer available for use. :: - Volshell (Volatility 3 Framework) 1.0.1 + Volshell (Volatility 3 Framework) 2.0.2 Readline imported successfully PDB scanning finished Call help() to see available functions - Volshell mode: Generic - Current Layer: primary + Volshell mode : Generic + Current Layer : primary + Current Symbol Table: None (primary) >>> @@ -55,9 +56,9 @@ python environment, we can do the following: :: - (primary) >>> proc = ps()[0] - (primary) >>> proc - + (layer_name) >>> proc = ps()[0] + (layer_name) >>> proc + When printing a volatility structure, various information is output, in this case the `type_name`, the `layer` and `offset` that it's been constructed on, and the size of the structure. @@ -70,31 +71,31 @@ automagic). :: - (primary) >>> dt('_EPROCESS') - nt_symbols1!_EPROCESS (2624 bytes) - 0x0 : Pcb nt_symbols1!_KPROCESS - 0x438 : ProcessLock nt_symbols1!_EX_PUSH_LOCK - 0x440 : UniqueProcessId nt_symbols1!pointer - 0x448 : ActiveProcessLinks nt_symbols1!_LIST_ENTRY + (layer_name) >>> dt('_EPROCESS') + symbol_table_name1!_EPROCESS (1968 bytes) + 0x0 : Pcb symbol_table_name1!_KPROCESS + 0x2d8 : ProcessLock symbol_table_name1!_EX_PUSH_LOCK + 0x2e0 : RundownProtect symbol_table_name1!_EX_RUNDOWN_REF + 0x2e8 : UniqueProcessId symbol_table_name1!pointer ... It can also be provided with an object and will interpret the data for each in the process: :: - (primary) >>> dt(proc) - nt_symbols1!_EPROCESS (2624 bytes) - 0x0 : Pcb nt_symbols1!_KPROCESS 0x8c0bccf8d040 - 0x438 : ProcessLock nt_symbols1!_EX_PUSH_LOCK 0x8c0bccf8d478 - 0x440 : UniqueProcessId nt_symbols1!pointer 356 - 0x448 : ActiveProcessLinks nt_symbols1!_LIST_ENTRY 0x8c0bccf8d488 + (layer_name) >>> dt(proc) + symbol_table_name1!_EPROCESS (1968 bytes) + 0x0 : Pcb symbol_table_name1!_KPROCESS 0xe08ff2459040 + 0x2d8 : ProcessLock symbol_table_name1!_EX_PUSH_LOCK 0xe08ff2459318 + 0x2e0 : RundownProtect symbol_table_name1!_EX_RUNDOWN_REF 0xe08ff2459320 + 0x2e8 : UniqueProcessId symbol_table_name1!pointer 4 ... These values can be accessed directory as attributes :: - (primary) >>> proc.UniqueProcessId + (layer_name) >>> proc.UniqueProcessId 356 Pointer structures contain the value they point to, but attributes accessed are forwarded to the object they point to. @@ -102,7 +103,7 @@ This means that pointers do not need to be explicitly dereferenced to access und :: - (primary) >>> proc.Pcb.DirectoryTableBase + (layer_name) >>> proc.Pcb.DirectoryTableBase 4355817472 Running plugins @@ -114,26 +115,26 @@ were required: :: - (primary) >>> from volatility3.plugins.windows import pslist - (primary) >>> display_plugin_output(pslist.PsList) - Unable to validate the plugin requirements: ['plugins.Volshell.9QZLXJKFWESI0BAP3M1U7Y5VCT468GRN.PsList.primary', 'plugins.Volshell.9QZLXJKFWESI0BAP3M1U7Y5VCT468GRN.PsList.nt_symbols'] + (layer_name) >>> from volatility3.plugins.windows import pslist + (layer_name) >>> display_plugin_output(pslist.PsList) + Unable to validate the plugin requirements: ['plugins.Volshell.VH3FSA1JBG0QP9E62Z8OT5UCIMLNYKW4.PsList.kernel'] -We can see that it's made a temporary configuration path for the plugin, and that neither `primary` nor `nt_symbols` -was fulfilled. +We can see that it's made a temporary configuration path for the plugin, and that the `kernel` requirement +was not fulfilled. We can see all the options that the plugin can accept by access the `get_requirements()` method of the plugin. This is a classmethod, so can be called on an uninstantiated copy of the plugin. :: - (primary) >>> pslist.PsList.get_requirements() - [, , , , ] + (layer_name) >>> pslist.PsList.get_requirements() + [, , , ] We can provide arguments via the `dpo` method call: :: - (primary) >>> display_plugin_output(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) + (layer_name) >>> display_plugin_output(pslist.PsList, kernel = self.config['kernel']) PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output @@ -142,8 +143,9 @@ We can provide arguments via the `dpo` method call: 356 4 smss.exe 0x8c0bccf8d040 3 - N/A False 2021-03-13 17:25:33.000000 N/A Disabled ... -Here's we've provided the current layer as the TranslationLayerRequirement, and used the symbol tables requirement -requested by the volshell plugin itself. A different table could be loaded and provided instead. The context used +Here's we've provided the kernel name that was requested by the volshell plugin itself (the generic volshell does not +load a kernel module, and instead only has a TranslationLayerRequirement). +A different module could be created and provided instead. The context used by the `dpo` method is always `context`. Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by @@ -151,8 +153,8 @@ using the `generate_treegrid` or `gt` command. :: - (primary) >>> treegrid = gt(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) - (primary) >>> treegrid.populate() + (layer_name) >>> treegrid = gt(pslist.PsList, kernel = self.config['kernel']) + (layer_name) >>> treegrid.populate() Treegrids must be populated before the data in them can be accessed. This is where the plugin actually runs and produces data. diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 2c5e13211..4cdbd26e8 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,14 +19,14 @@ import os import sys import tempfile import traceback -from typing import Dict, Type, Union, Any +from typing import Any, Dict, Type, Union from urllib import parse, request import volatility3.plugins import volatility3.symbols from volatility3 import framework from volatility3.cli import text_renderer, volargparse -from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins, configuration +from volatility3.framework import automagic, configuration, constants, contexts, exceptions, interfaces, plugins from volatility3.framework.automagic import stacker from volatility3.framework.configuration import requirements diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 7b8a759a6..812d44337 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -7,12 +7,11 @@ import json import logging import os import sys -import glob import volatility3.plugins import volatility3.symbols from volatility3 import cli, framework -from volatility3.cli.volshell import generic, windows, linux, mac +from volatility3.cli.volshell import generic, linux, mac, windows from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins # Make sure we log everything diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 8f81a0420..29accb529 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -8,11 +8,11 @@ import random import string import struct import sys -from typing import Any, Dict, List, Optional, Tuple, Union, Type, Iterable -from urllib import request, parse +from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union +from urllib import parse, request from volatility3.cli import text_renderer, volshell -from volatility3.framework import renderers, interfaces, objects, plugins, exceptions +from volatility3.framework import exceptions, interfaces, objects, plugins, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, physical, resources @@ -32,6 +32,7 @@ class Volshell(interfaces.plugins.PluginInterface): super().__init__(*args, **kwargs) self.__current_layer: Optional[str] = None self.__console = None + self.__kernel = None def random_string(self, length: int = 32) -> str: return ''.join(random.sample(string.ascii_uppercase + string.digits, length)) @@ -57,8 +58,6 @@ class Volshell(interfaces.plugins.PluginInterface): Return a TreeGrid but this is always empty since the point of this plugin is to run interactively """ - self.__current_layer = self.config['primary'] - # Try to enable tab completion try: import readline @@ -79,9 +78,10 @@ class Volshell(interfaces.plugins.PluginInterface): banner = f""" Call help() to see available functions - Volshell mode: {mode} - Current Layer: {self.current_layer} - """ + Volshell mode : {mode} + Current Layer : {self.current_layer} + Current Symbol Table: {self.current_symbol_table} +""" sys.ps1 = f"({self.current_layer}) >>> " self.__console = code.InteractiveConsole(locals = self._construct_locals_dict()) @@ -174,12 +174,23 @@ class Volshell(interfaces.plugins.PluginInterface): @property def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.config['primary'] return self.__current_layer + @property + def current_symbol_table(self): + return None + + @property + def kernel(self): + """No default kernel for generic volshell""" + return None + def change_layer(self, layer_name = None): """Changes the current default layer""" if not layer_name: - layer_name = self.config['primary'] + layer_name = self.current_layer self.__current_layer = layer_name sys.ps1 = f"({self.current_layer}) >>> " diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 4338ae06f..a58ff78f6 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -5,7 +5,7 @@ from typing import Any, List, Tuple, Union from volatility3.cli.volshell import generic -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.plugins.linux import pslist @@ -15,8 +15,8 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return (super().get_requirements() + [ - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), + return ([ + requirements.ModuleRequirement(name = "kernel", description = "Linux kernel module"), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) @@ -37,14 +37,14 @@ class Volshell(generic.Volshell): def list_tasks(self): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_tasks(self.context, self.config['primary'], self.config['vmlinux'])) + return list(pslist.PsList.list_tasks(self.context, self.current_layer, self.current_symbol_table)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (['ct', 'change_task', 'cp'], self.change_task), (['lt', 'list_tasks', 'ps'], self.list_tasks), - (['symbols'], self.context.symbol_space[self.config['vmlinux']]), + (['symbols'], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get('pid', None) is not None: self.change_task(self.config['pid']) @@ -64,3 +64,19 @@ class Volshell(generic.Volshell): if symbol_table is None: symbol_table = self.config['vmlinux'] return super().display_symbols(symbol_table) + + @property + def kernel(self): + if self.__kernel is None: + self.__kernel = self.context.modules[self.config['kernel']] + return self.__kernel + + @property + def current_symbol_table(self): + return self.kernel.symbol_table_name + + @property + def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.kernel.layer_name + return self.__current_layer diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 8218848ba..644a02bd2 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -15,9 +15,9 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return (super().get_requirements() + [ - requirements.SymbolTableRequirement(name = "darwin", description = "Darwin kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + return ([ + requirements.ModuleRequirement(name = "kernel", description = "Darwin kernel module"), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) @@ -37,14 +37,14 @@ class Volshell(generic.Volshell): def list_tasks(self): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_tasks(self.context, self.config['primary'], self.config['darwin'])) + return list(pslist.PsList.list_tasks(self.context, self.current_layer, self.current_symbol_table)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (['ct', 'change_task', 'cp'], self.change_task), (['lt', 'list_tasks', 'ps'], self.list_tasks), - (['symbols'], self.context.symbol_space[self.config['darwin']]), + (['symbols'], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get('pid', None) is not None: self.change_task(self.config['pid']) @@ -62,5 +62,21 @@ class Volshell(generic.Volshell): def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: - symbol_table = self.config['darwin'] + symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) + + @property + def kernel(self): + if self.__kernel is None: + self.__kernel = self.context.modules[self.config['kernel']] + return self.__kernel + + @property + def current_symbol_table(self): + return self.kernel.symbol_table_name + + @property + def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.kernel.layer_name + return self.__current_layer diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 6c191ad28..c35a8dfc7 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -5,7 +5,7 @@ from typing import Any, List, Tuple, Union from volatility3.cli.volshell import generic -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.plugins.windows import pslist @@ -15,8 +15,8 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return (super().get_requirements() + [ - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + return ([ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel'), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) @@ -34,14 +34,14 @@ class Volshell(generic.Volshell): def list_processes(self): """Returns a list of EPROCESS objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_processes(self.context, self.config['primary'], self.config['nt_symbols'])) + return list(pslist.PsList.list_processes(self.context, self.current_layer, self.current_symbol_table)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (['cp', 'change_process'], self.change_process), (['lp', 'list_processes', 'ps'], self.list_processes), - (['symbols'], self.context.symbol_space[self.config['nt_symbols']]), + (['symbols'], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get('pid', None) is not None: self.change_process(self.config['pid']) @@ -53,11 +53,27 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['nt_symbols'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: - symbol_table = self.config['nt_symbols'] + symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) + + @property + def kernel(self): + if self.__kernel is None: + self.__kernel = self.context.modules[self.config['kernel']] + return self.__kernel + + @property + def current_symbol_table(self): + return self.kernel.symbol_table_name + + @property + def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.kernel.layer_name + return self.__current_layer From 9fb59e4714f15bf571e4c70afa3c7fb290510040 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 17 Feb 2022 01:43:50 +0000 Subject: [PATCH 085/404] Volshell: Further improvements for mac/linux --- volatility3/cli/volshell/generic.py | 57 ++++++++++++++++++++++++----- volatility3/cli/volshell/linux.py | 12 +----- volatility3/cli/volshell/mac.py | 14 +------ volatility3/cli/volshell/windows.py | 10 ----- 4 files changed, 50 insertions(+), 43 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 29accb529..94634d005 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -31,8 +31,9 @@ class Volshell(interfaces.plugins.PluginInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__current_layer: Optional[str] = None + self.__current_symbol_table: Optional[str] = None + self.__current_kernel_name: Optional[str] = None self.__console = None - self.__kernel = None def random_string(self, length: int = 32) -> str: return ''.join(random.sample(string.ascii_uppercase + string.digits, length)) @@ -78,9 +79,10 @@ class Volshell(interfaces.plugins.PluginInterface): banner = f""" Call help() to see available functions - Volshell mode : {mode} - Current Layer : {self.current_layer} - Current Symbol Table: {self.current_symbol_table} + Volshell mode : {mode} + Current Layer : {self.current_layer} + Current Symbol Table : {self.current_symbol_table} + Current Kernel : {self.current_kernel_name} """ sys.ps1 = f"({self.current_layer}) >>> " @@ -121,7 +123,10 @@ class Volshell(interfaces.plugins.PluginInterface): (['dw', 'display_words'], self.display_words), (['dd', 'display_doublewords'], self.display_doublewords), (['dq', 'display_quadwords'], self.display_quadwords), (['dis', 'disassemble'], self.disassemble), - (['cl', 'change_layer'], self.change_layer), (['context'], self.context), (['self'], self), + (['cl', 'change_layer'], self.change_layer), + (['cs', 'change_symboltable'], self.change_symbol_table), + (['ck', 'change_kernel'], self.change_kernel), + (['context'], self.context), (['self'], self), (['dpo', 'display_plugin_output'], self.display_plugin_output), (['gt', 'generate_treegrid'], self.generate_treegrid), (['rt', 'render_treegrid'], self.render_treegrid), @@ -180,20 +185,52 @@ class Volshell(interfaces.plugins.PluginInterface): @property def current_symbol_table(self): - return None + if self.__current_symbol_table is None and self.kernel: + self.__current_symbol_table = self.kernel.symbol_table_name + return self.__current_symbol_table + + @property + def current_kernel_name(self): + if self.__current_kernel_name is None: + self.__current_kernel_name = self.config.get('kernel', None) + return self.__current_kernel_name @property def kernel(self): - """No default kernel for generic volshell""" - return None + """Returns the current kernel object""" + if self.current_kernel_name not in self.context.modules: + return None + return self.context.modules[self.current_kernel_name] - def change_layer(self, layer_name = None): + def change_layer(self, layer_name: str = None): """Changes the current default layer""" if not layer_name: layer_name = self.current_layer - self.__current_layer = layer_name + if layer_name not in self.context.layers: + print(f"Layer {layer_name} not present in context") + else: + self.__current_layer = layer_name sys.ps1 = f"({self.current_layer}) >>> " + def change_symbol_table(self, symbol_table_name: str = None): + """Changes the current_symbol_table""" + if not symbol_table_name: + print("No symbol table provided, not changing current symbol table") + if symbol_table_name not in self.context.symbol_space: + print(f"Symbol table {symbol_table_name} not present in context symbol_space") + else: + self.__current_symbol_table = symbol_table_name + print(f"Current Symbol Table: {self.current_symbol_table}") + + def change_kernel(self, kernel_name: str = None): + if not kernel_name: + print("No kernel module name provided, not changing current kernel") + if kernel_name not in self.context.modules: + print(f"Kernel module {kernel_name} not found in the context module list") + else: + self.__current_kernel_name = kernel_name + print(f"Current kernel : {self.current_kernel_name}") + def display_bytes(self, offset, count = 128, layer_name = None): """Displays byte values and ASCII characters""" remaining_data = self._read_data(offset, count = count, layer_name = layer_name) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index a58ff78f6..97a488743 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -37,7 +37,7 @@ class Volshell(generic.Volshell): def list_tasks(self): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_tasks(self.context, self.current_layer, self.current_symbol_table)) + return list(pslist.PsList.list_tasks(self.context, self.current_kernel_name)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() @@ -65,16 +65,6 @@ class Volshell(generic.Volshell): symbol_table = self.config['vmlinux'] return super().display_symbols(symbol_table) - @property - def kernel(self): - if self.__kernel is None: - self.__kernel = self.context.modules[self.config['kernel']] - return self.__kernel - - @property - def current_symbol_table(self): - return self.kernel.symbol_table_name - @property def current_layer(self): if self.__current_layer is None: diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 644a02bd2..305f80505 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -34,10 +34,10 @@ class Volshell(generic.Volshell): return print(f"No task with task ID {pid} found") - def list_tasks(self): + def list_tasks(self, method = None): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_tasks(self.context, self.current_layer, self.current_symbol_table)) + return list(pslist.PsList.get_list_tasks(method)(self.context, self.current_kernel_name)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() @@ -65,16 +65,6 @@ class Volshell(generic.Volshell): symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) - @property - def kernel(self): - if self.__kernel is None: - self.__kernel = self.context.modules[self.config['kernel']] - return self.__kernel - - @property - def current_symbol_table(self): - return self.kernel.symbol_table_name - @property def current_layer(self): if self.__current_layer is None: diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index c35a8dfc7..2cc5d3e1d 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -62,16 +62,6 @@ class Volshell(generic.Volshell): symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) - @property - def kernel(self): - if self.__kernel is None: - self.__kernel = self.context.modules[self.config['kernel']] - return self.__kernel - - @property - def current_symbol_table(self): - return self.kernel.symbol_table_name - @property def current_layer(self): if self.__current_layer is None: From e6c3c94a10a087a465c25f8f3b4a3e6e86b7eadb Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 17 Feb 2022 01:46:54 +0000 Subject: [PATCH 086/404] Volshell: Update docs slightly --- doc/source/volshell.rst | 7 ++++--- volatility3/cli/volshell/generic.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 1e51f90ba..de3c4398a 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -29,9 +29,10 @@ operating system mode for volshell, and the current layer available for use. Call help() to see available functions - Volshell mode : Generic - Current Layer : primary - Current Symbol Table: None + Volshell mode : Generic + Current Layer : primary + Current Symbol Table : None + Current Kernel Name : None (primary) >>> diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 94634d005..19e263a03 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -82,7 +82,7 @@ class Volshell(interfaces.plugins.PluginInterface): Volshell mode : {mode} Current Layer : {self.current_layer} Current Symbol Table : {self.current_symbol_table} - Current Kernel : {self.current_kernel_name} + Current Kernel Name : {self.current_kernel_name} """ sys.ps1 = f"({self.current_layer}) >>> " From da7dd322711187f6473eaf6f4fc116b021764ef3 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 17 Feb 2022 15:31:48 +0200 Subject: [PATCH 087/404] Improve slow pdb scanning --- volatility3/framework/automagic/pdbscan.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 8179339c8..3e350b071 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -192,9 +192,15 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): if not physical: layer_to_scan = virtual_layer_name + start_scan_address = 0 + if not physical and context.layers[layer_to_scan].metadata.architecture in ["Intel64"]: + # TODO: change this value accordingly when 5-Level paging is supported. + start_scan_address = (0x1f0 << 39) + kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] kernels = PDBUtility.pdbname_scan(ctx = context, layer_name = layer_to_scan, + start = start_scan_address, page_size = vlayer.page_size, pdb_names = kernel_pdb_names, progress_callback = progress_callback) From 30368751368e7874f452614d3f738e5da28ea8a8 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 17 Feb 2022 16:10:44 +0200 Subject: [PATCH 088/404] run optimized scan before slow scan --- volatility3/framework/automagic/pdbscan.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 3e350b071..5db66a3d0 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -146,8 +146,13 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): return None return (virtual_layer_name, kernel['mz_offset'], kernel) + vollog.debug("Kernel base determination - optimized scan virtual layer") + valid_kernel = self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, True, False, progress_callback) + if valid_kernel != None: + return valid_kernel + vollog.debug("Kernel base determination - slow scan virtual layer") - return self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, False, progress_callback) + return self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, False, False, progress_callback) def method_fixed_mapping(self, context: interfaces.context.ContextInterface, @@ -175,12 +180,13 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vollog.debug(f"Potential kernel_virtual_offset caused a page fault: {hex(kvo)}") vollog.debug("Kernel base determination - testing fixed base address") - return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, True, progress_callback) + return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, False, True, progress_callback) def _method_layer_pdb_scan(self, context: interfaces.context.ContextInterface, vlayer: layers.intel.Intel, test_kernel: Callable, + optimized: bool = False, physical: bool = True, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: # TODO: Verify this is a windows image @@ -193,7 +199,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): layer_to_scan = virtual_layer_name start_scan_address = 0 - if not physical and context.layers[layer_to_scan].metadata.architecture in ["Intel64"]: + if optimized and not physical and context.layers[layer_to_scan].metadata.architecture in ["Intel64"]: # TODO: change this value accordingly when 5-Level paging is supported. start_scan_address = (0x1f0 << 39) From 8bcb7b42276c98f675904bd9d6fdfb0d565641ef Mon Sep 17 00:00:00 2001 From: cpuu Date: Fri, 18 Feb 2022 13:25:46 +0900 Subject: [PATCH 089/404] Add offset information in pslist plugin for Linux --- volatility3/framework/plugins/linux/pslist.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 5672bb56e..dd1832576 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -6,6 +6,7 @@ from typing import Callable, Iterable, List, Any from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints class PsList(interfaces.plugins.PluginInterface): @@ -57,7 +58,7 @@ class PsList(interfaces.plugins.PluginInterface): if task.parent: ppid = task.parent.pid name = utility.array_to_string(task.comm) - yield (0, (pid, ppid, name)) + yield (0, (format_hints.Hex(task.vol.offset), name, pid, ppid)) @classmethod def list_tasks( @@ -84,4 +85,4 @@ class PsList(interfaces.plugins.PluginInterface): yield task def run(self): - return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str)], self._generator()) + return renderers.TreeGrid([("OFFSET", format_hints.Hex), ("COMM", str), ("PID", int), ("PPID", int)], self._generator()) From bfc4c50e671404e3aa6d9262facd1a05951e3317 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 20 Feb 2022 14:49:06 +0200 Subject: [PATCH 090/404] related --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index b5a7db286..107689dc8 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -136,7 +136,7 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): if k not in ["context", "data_format", "object_info", "type_name"]: kwargs[k] = v kwargs['new_value'] = self.__new_value - return (self._context, self._vol.maps[-2]['type_name'], self._vol.maps[-3], self._data_format), kwargs + return (self._context, self._vol.maps[-3]['type_name'], self._vol.maps[-2], self._data_format), kwargs @classmethod def _unmarshall(cls, context: interfaces.context.ContextInterface, data_format: DataFormatInfo, From f2e3df27f48c09e75d42dc56419698ef8e5001af Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 21 Feb 2022 14:57:31 +0200 Subject: [PATCH 091/404] fix read whole module --- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 35bc62d18..585e96b6d 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -146,7 +146,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): max_size = pe_data.OPTIONAL_HEADER.SizeOfImage # Proper data - virtual_data = layer.read(offset, max_size) + virtual_data = layer.read(offset, max_size, pad=True) pe_data = pefile.PE(data = virtual_data) # De-virtualize the memory From 58697479bb819fe6c3f17182bb419a03bc4541d7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 23 Feb 2022 00:00:07 +0000 Subject: [PATCH 092/404] Layers: Fix opening UNC paths on windows --- volatility3/framework/layers/resources.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 7ace25290..f8705edca 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -10,10 +10,11 @@ import logging import lzma import os import ssl +import sys import urllib.parse import urllib.request import zipfile -from typing import Optional, Any, IO, List +from typing import Any, IO, List, Optional from urllib import error from volatility3 import framework @@ -100,6 +101,19 @@ class ResourceAccessor(object): """ urllib.request.install_opener(urllib.request.build_opener(*self._handlers)) + # Python bug 46654 + if sys.platform == 'win32': + # We only need to worry about UNC paths on windows, on linux they'd be smb:// and need pysmb or similar + parsed_url = urllib.parse.urlparse(url, scheme = 'file') + if parsed_url.scheme == 'file' and parsed_url.netloc: + # Change the netloc to '/' and then prepend the netloc to the path + # Urlunparse will remove extra initial slashes from path, hence setting netloc + new_url = urllib.parse.urlunparse((parsed_url.scheme, '/', + '/' + parsed_url.netloc + parsed_url.path, parsed_url.params, + parsed_url.query, parsed_url.fragment)) + vollog.log(constants.LOGLEVEL_VVVV, f'UNC path detected, converted path {url} to {new_url}') + url = new_url + try: fp = urllib.request.urlopen(url, context = self._context) except error.URLError as excp: From 579a0b873515dc94795f9bef0efddc4f743cd372 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 23 Feb 2022 00:08:10 +0000 Subject: [PATCH 093/404] Layers: More documentation and don't break correct URLs --- volatility3/framework/layers/resources.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index f8705edca..ac25b5cc2 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -105,7 +105,9 @@ class ResourceAccessor(object): if sys.platform == 'win32': # We only need to worry about UNC paths on windows, on linux they'd be smb:// and need pysmb or similar parsed_url = urllib.parse.urlparse(url, scheme = 'file') - if parsed_url.scheme == 'file' and parsed_url.netloc: + # Only worry about file scheme URLs, make sure that there's either a host or + # the unparsing left an extra slash at the start (which will get lost with urlunparse) + if parsed_url.scheme == 'file' and (parsed_url.netloc or parsed_url.path.startswith('//')): # Change the netloc to '/' and then prepend the netloc to the path # Urlunparse will remove extra initial slashes from path, hence setting netloc new_url = urllib.parse.urlunparse((parsed_url.scheme, '/', From 265b2825697ecb8c94ba8654acd6191ba0055fdd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 23 Feb 2022 22:53:54 +0000 Subject: [PATCH 094/404] Objects: Don't try to read 0 bytes when unmarshalling --- volatility3/framework/objects/__init__.py | 6 +++++- .../symbols/windows/extensions/__init__.py | 16 ++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 107689dc8..c191d5562 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -141,7 +141,11 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): @classmethod def _unmarshall(cls, context: interfaces.context.ContextInterface, data_format: DataFormatInfo, object_info: interfaces.objects.ObjectInformation) -> TUnion[int, float, bool, bytes, str]: - data = context.layers.read(object_info.layer_name, object_info.offset, data_format.length) + # Don't try to lookup a 0 length data format, incase it's at an invalid offset. Length 0 means b'' + if data_format.length > 0: + data = context.layers.read(object_info.layer_name, object_info.offset, data_format.length) + else: + data = b'' return convert_data_to_value(data, cls._struct_type, data_format) class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 84c47e733..616744093 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -7,16 +7,17 @@ import datetime import functools import logging import math -from typing import Iterable, Iterator, Optional, Union, Tuple, List +from typing import Iterable, Iterator, List, Optional, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols from volatility3.framework.layers import intel from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic -from volatility3.framework.symbols.windows.extensions import pool, pe, kdbg +from volatility3.framework.symbols.windows.extensions import kdbg, pe, pool vollog = logging.getLogger(__name__) + # Keep these in a basic module, to prevent import cycles when symbol providers require them @@ -461,10 +462,13 @@ class UNICODE_STRING(objects.StructType): # We explicitly do *not* catch errors here, we allow an exception to be thrown # (otherwise there's no way to determine anything went wrong) # It's up to the user of this method to catch exceptions - return self.Buffer.dereference().cast("string", - max_length = self.Length, - errors = "replace", - encoding = "utf16") + + # We manually construct an object rather than casting a dereferenced pointer in case + # the buffer length is 0 and the pointer is a NULL pointer + return self._context.object(self.vol.type_name.split(constants.BANG)[0] + constants.BANG + 'string', + layer_name = self.Buffer.vol.layer_name, + offset = self.Buffer, + max_length = self.Length, errors = 'replace', encoding = 'utf16') String = property(get_string) From 78b3553b2ab8d4a318df021d09b191b9192add5b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 25 Feb 2022 16:33:54 +0000 Subject: [PATCH 095/404] Objects: Implement minor code optimization by @paulkermann --- volatility3/framework/objects/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index c191d5562..472370e6b 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -142,10 +142,9 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): def _unmarshall(cls, context: interfaces.context.ContextInterface, data_format: DataFormatInfo, object_info: interfaces.objects.ObjectInformation) -> TUnion[int, float, bool, bytes, str]: # Don't try to lookup a 0 length data format, incase it's at an invalid offset. Length 0 means b'' + data = b'' if data_format.length > 0: data = context.layers.read(object_info.layer_name, object_info.offset, data_format.length) - else: - data = b'' return convert_data_to_value(data, cls._struct_type, data_format) class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): From 1b09f20b5c226622408dc26494364195b9705d88 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 13:21:34 +0000 Subject: [PATCH 096/404] Windows: Raise PE extraction size and make it a constant --- .../framework/constants/windows/__init__.py | 2 ++ .../framework/symbols/windows/extensions/pe.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/constants/windows/__init__.py b/volatility3/framework/constants/windows/__init__.py index a19216605..7face984a 100644 --- a/volatility3/framework/constants/windows/__init__.py +++ b/volatility3/framework/constants/windows/__init__.py @@ -8,3 +8,5 @@ Windows-specific values that aren't found in debug symbols KERNEL_MODULE_NAMES = ["ntkrnlmp", "ntkrnlpa", "ntkrpamp", "ntoskrnl"] """The list of names that kernel modules can have within the windows OS""" + +PE_MAX_EXTRACTION_SIZE = 1024 * 1024 * 256 diff --git a/volatility3/framework/symbols/windows/extensions/pe.py b/volatility3/framework/symbols/windows/extensions/pe.py index df461318f..2f271da5d 100644 --- a/volatility3/framework/symbols/windows/extensions/pe.py +++ b/volatility3/framework/symbols/windows/extensions/pe.py @@ -2,15 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Generator, Tuple import logging +from typing import Generator, Tuple -from volatility3.framework import constants -from volatility3.framework import objects, interfaces +from volatility3.framework import constants, interfaces, objects from volatility3.framework.renderers import conversion vollog = logging.getLogger(__name__) + class IMAGE_DOS_HEADER(objects.StructType): def get_nt_header(self) -> interfaces.objects.ObjectInterface: @@ -77,12 +77,13 @@ class IMAGE_DOS_HEADER(objects.StructType): image_base_type = nt_header.OptionalHeader.ImageBase.vol.type_name member_size = self._context.symbol_space.get_type(image_base_type).size try: - newval = objects.convert_value_to_data(self.vol.offset, int, nt_header.OptionalHeader.ImageBase.vol.data_format) + newval = objects.convert_value_to_data(self.vol.offset, int, + nt_header.OptionalHeader.ImageBase.vol.data_format) new_pe = raw_data[:image_base_offset] + newval + raw_data[image_base_offset + member_size:] except OverflowError: vollog.warning("Volatility was unable to fix the image base for the PE file at base address {:#x}. " \ - "This will cause issues with many static analysis tools if you do not inform the " \ - "tool of the in-memory load address.".format(self.vol.offset)) + "This will cause issues with many static analysis tools if you do not inform the " \ + "tool of the in-memory load address.".format(self.vol.offset)) new_pe = raw_data return new_pe @@ -109,7 +110,7 @@ class IMAGE_DOS_HEADER(objects.StructType): size_of_image = nt_header.OptionalHeader.SizeOfImage # no legitimate PE is going to be larger than this - if size_of_image > (1024 * 1024 * 100): + if size_of_image > constants.windows.PE_MAX_EXTRACTION_SIZE: raise ValueError(f"The claimed SizeOfImage is too large: {size_of_image}") read_layer = self._context.layers[layer_name] From f156d237a4129da9394f81e72949bef58f3e0b76 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Feb 2022 13:36:54 +0900 Subject: [PATCH 097/404] Add 'ImportError' handling of the capstone module on malfind plugin. --- volatility3/framework/plugins/windows/malfind.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index bfd29a254..7fd032ef5 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -13,6 +13,12 @@ from volatility3.plugins.windows import pslist, vadinfo vollog = logging.getLogger(__name__) +try: + import capstone + + has_capstone = True +except ImportError: + has_capstone = False class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" From 9868aeb9060687b240637470d690de4327cab3c3 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Feb 2022 14:12:25 +0900 Subject: [PATCH 098/404] Add Error Raise point --- volatility3/framework/plugins/windows/malfind.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 7fd032ef5..ae10a6c65 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -134,7 +134,11 @@ class Malfind(interfaces.plugins.PluginInterface): else: architecture = "intel64" - disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) + if has_capstone: + disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) + else: + raise exceptions.MissingModuleException( + "capstone", "Requires capstone to disassembly data") file_output = "Disabled" if self.config['dump']: From 58782fcfe1ab7f495897571f9fd1a682b6fc48f9 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 3 Mar 2022 02:00:32 +0900 Subject: [PATCH 099/404] Typo Error Fix - Context module object Args code comment --- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 518215ab4..ab81beb5e 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -141,7 +141,7 @@ class Context(interfaces.context.ContextInterface): layer_name: The layer within the context in which the module exists offset: The offset at which the module exists in the layer native_layer_name: The default native layer for objects constructed by the module - size: The size, in bytes, that the module occupys from offset location within the layer named layer_name + size: The size, in bytes, that the module occupies from offset location within the layer named layer_name """ if size: return SizedModule.create(self, diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index c52e1aaa5..b8470ae47 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -129,7 +129,7 @@ class ContextInterface(metaclass = ABCMeta): layer_name: The layer the module is associated with (which layer the module lives within) offset: The initial/base offset of the module (used as the offset for relative symbols) native_layer_name: The default native_layer_name to use when the module constructs objects - size: The size, in bytes, that the module occupys from offset location within the layer named layer_name + size: The size, in bytes, that the module occupies from offset location within the layer named layer_name Returns: A module object From dae88605778a7b627ccd924e0a1ea7b8742ca0e0 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 3 Mar 2022 02:02:43 +0900 Subject: [PATCH 100/404] Restore PR --- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index ab81beb5e..2fd00c531 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -141,7 +141,7 @@ class Context(interfaces.context.ContextInterface): layer_name: The layer within the context in which the module exists offset: The offset at which the module exists in the layer native_layer_name: The default native layer for objects constructed by the module - size: The size, in bytes, that the module occupies from offset location within the layer named layer_name + size: The size, in bytes, that the module occupy from offset location within the layer named layer_name """ if size: return SizedModule.create(self, diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index b8470ae47..b70d55baa 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -129,7 +129,7 @@ class ContextInterface(metaclass = ABCMeta): layer_name: The layer the module is associated with (which layer the module lives within) offset: The initial/base offset of the module (used as the offset for relative symbols) native_layer_name: The default native_layer_name to use when the module constructs objects - size: The size, in bytes, that the module occupies from offset location within the layer named layer_name + size: The size, in bytes, that the module occupy from offset location within the layer named layer_name Returns: A module object From 49308eb18dd0035188ab7939dc825904d477066f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 3 Mar 2022 02:03:31 +0900 Subject: [PATCH 101/404] Restore PR --- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 2fd00c531..518215ab4 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -141,7 +141,7 @@ class Context(interfaces.context.ContextInterface): layer_name: The layer within the context in which the module exists offset: The offset at which the module exists in the layer native_layer_name: The default native layer for objects constructed by the module - size: The size, in bytes, that the module occupy from offset location within the layer named layer_name + size: The size, in bytes, that the module occupys from offset location within the layer named layer_name """ if size: return SizedModule.create(self, diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index b70d55baa..c52e1aaa5 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -129,7 +129,7 @@ class ContextInterface(metaclass = ABCMeta): layer_name: The layer the module is associated with (which layer the module lives within) offset: The initial/base offset of the module (used as the offset for relative symbols) native_layer_name: The default native_layer_name to use when the module constructs objects - size: The size, in bytes, that the module occupy from offset location within the layer named layer_name + size: The size, in bytes, that the module occupys from offset location within the layer named layer_name Returns: A module object From 670401eac71d39cd24cea9a17ef0062bb9722756 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 3 Mar 2022 20:35:39 +0000 Subject: [PATCH 102/404] Windows: Test unicode strings for length 0 In some tests we were checking whether asking for the string value threw an InvalidAddressException through an error as to whether we should look elsewhere for the data. As of commit 265b2825 we now treat 0-length strings as valid (as per #652), meaning we need to check for length 0 as well as invalid pointers. If this crops up often, we may need to revisit the decision to make sure its in keeping with how windows treats zero length strings, but for now we only did it once for registry keys. Closes #665 --- .../symbols/windows/extensions/__init__.py | 19 +++++++++++-------- .../symbols/windows/extensions/registry.py | 18 ++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 616744093..dc0de1dda 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -307,12 +307,15 @@ class MMVAD(MMVAD_SHORT): try: # this is for xp and 2003 if self.has_member("ControlArea"): - file_name = self.ControlArea.FilePointer.FileName.get_string() + filename_obj = self.ControlArea.FilePointer.FileName # this is for vista through windows 7 else: - file_name = self.Subsection.ControlArea.FilePointer.dereference().cast( - "_FILE_OBJECT").FileName.get_string() + filename_obj = self.Subsection.ControlArea.FilePointer.dereference().cast( + "_FILE_OBJECT").FileName + + if filename_obj.Length > 0: + file_name = filename_obj.get_string() except exceptions.InvalidAddressException: pass @@ -902,8 +905,8 @@ class CONTROL_AREA(objects.StructType): return False # The first SubsectionBase should not be page aligned - #subsection = self.get_subsection() - #if subsection.SubsectionBase & self.PAGE_MASK == 0: + # subsection = self.get_subsection() + # if subsection.SubsectionBase & self.PAGE_MASK == 0: # return False except exceptions.InvalidAddressException: return False @@ -952,7 +955,7 @@ class CONTROL_AREA(objects.StructType): subsection_offset = starting_sector * 0x200 # Similar to the check in is_valid(), make sure the SubsectionBase is not page aligned. - #if subsection.SubsectionBase & self.PAGE_MASK == 0: + # if subsection.SubsectionBase & self.PAGE_MASK == 0: # break ptecount = 0 @@ -983,8 +986,8 @@ class CONTROL_AREA(objects.StructType): # Currently just a temporary workaround to deal with custom bit flag # in the PFN field for pages in transition state. # See https://github.com/volatilityfoundation/volatility3/pull/475 - physoffset = (mmpte.u.Trans.PageFrameNumber & (( 1 << 33 ) - 1 ) ) << 12 - + physoffset = (mmpte.u.Trans.PageFrameNumber & ((1 << 33) - 1)) << 12 + yield physoffset, file_offset, self.PAGE_SIZE # Go to the next PTE entry diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index f30bb5eb0..47ff24506 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -5,10 +5,10 @@ import enum import logging import struct -from typing import Optional, Iterable, Union +from typing import Iterable, Optional, Union -from volatility3.framework import constants, exceptions, objects, interfaces -from volatility3.framework.layers.registry import RegistryHive, RegistryInvalidIndex, RegistryFormatException +from volatility3.framework import constants, exceptions, interfaces, objects +from volatility3.framework.layers.registry import RegistryFormatException, RegistryHive, RegistryInvalidIndex vollog = logging.getLogger(__name__) @@ -76,7 +76,9 @@ class CMHIVE(objects.StructType): for attr in ["FileFullPath", "FileUserName", "HiveRootPath"]: try: - return getattr(self, attr).get_string() + name = getattr(self, attr) + if name.Length > 0: + return name.get_string() except (AttributeError, exceptions.InvalidAddressException): pass @@ -269,7 +271,7 @@ class CM_KEY_VALUE(objects.StructType): if self_type == RegValueTypes.REG_DWORD_BIG_ENDIAN: if len(data) != struct.calcsize(">L"): raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}") - res, = struct.unpack(">L", data) + res, = struct.unpack(">L", data) return res if self_type == RegValueTypes.REG_QWORD: if len(data) != struct.calcsize(" Date: Sat, 5 Mar 2022 16:27:07 +0900 Subject: [PATCH 103/404] Context Typo Error, MFT Symbol JSON Prettier --- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/context.py | 2 +- volatility3/framework/symbols/windows/mft.json | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 518215ab4..ab81beb5e 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -141,7 +141,7 @@ class Context(interfaces.context.ContextInterface): layer_name: The layer within the context in which the module exists offset: The offset at which the module exists in the layer native_layer_name: The default native layer for objects constructed by the module - size: The size, in bytes, that the module occupys from offset location within the layer named layer_name + size: The size, in bytes, that the module occupies from offset location within the layer named layer_name """ if size: return SizedModule.create(self, diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index c52e1aaa5..b8470ae47 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -129,7 +129,7 @@ class ContextInterface(metaclass = ABCMeta): layer_name: The layer the module is associated with (which layer the module lives within) offset: The initial/base offset of the module (used as the offset for relative symbols) native_layer_name: The default native_layer_name to use when the module constructs objects - size: The size, in bytes, that the module occupys from offset location within the layer named layer_name + size: The size, in bytes, that the module occupies from offset location within the layer named layer_name Returns: A module object diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index b71be6444..17edd45dd 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -270,7 +270,8 @@ "offset": 8, "type": { "kind": "base", - "name": "unsigned char" } + "name": "unsigned char" + } }, "NameLength": { "offset": 9, @@ -322,7 +323,8 @@ "offset": 8, "type": { "kind": "base", - "name": "unsigned short" } + "name": "unsigned short" + } } }, "kind": "struct", From f060562b278352e8c3cfadd65850688717c47b15 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Mar 2022 16:29:09 +0900 Subject: [PATCH 104/404] Rebase --- volatility3/framework/plugins/windows/malfind.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index ae10a6c65..e70fd0f75 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -13,12 +13,6 @@ from volatility3.plugins.windows import pslist, vadinfo vollog = logging.getLogger(__name__) -try: - import capstone - - has_capstone = True -except ImportError: - has_capstone = False class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" @@ -134,12 +128,8 @@ class Malfind(interfaces.plugins.PluginInterface): else: architecture = "intel64" - if has_capstone: - disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) - else: - raise exceptions.MissingModuleException( - "capstone", "Requires capstone to disassembly data") - + disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) + file_output = "Disabled" if self.config['dump']: file_output = "Error outputting to file" From 639f87a0a4e642ce02c848b4ae2639efc64639ff Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Mar 2022 16:29:40 +0900 Subject: [PATCH 105/404] Remove Tab --- volatility3/framework/plugins/windows/malfind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index e70fd0f75..bfd29a254 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -129,7 +129,7 @@ class Malfind(interfaces.plugins.PluginInterface): architecture = "intel64" disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) - + file_output = "Disabled" if self.config['dump']: file_output = "Error outputting to file" From 06961ce53742a4f266d4892f7b7d8120dc34388b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Mar 2022 16:32:38 +0900 Subject: [PATCH 106/404] Initialize MBR Parser --- .../framework/plugins/windows/mbrparser.py | 76 ++++++++++++ .../symbols/windows/extensions/mbr.py | 111 ++++++++++++++++++ .../framework/symbols/windows/mbr.json | 30 +++++ 3 files changed, 217 insertions(+) create mode 100644 volatility3/framework/plugins/windows/mbrparser.py create mode 100644 volatility3/framework/symbols/windows/extensions/mbr.py create mode 100644 volatility3/framework/symbols/windows/mbr.json diff --git a/volatility3/framework/plugins/windows/mbrparser.py b/volatility3/framework/plugins/windows/mbrparser.py new file mode 100644 index 000000000..37a4612f8 --- /dev/null +++ b/volatility3/framework/plugins/windows/mbrparser.py @@ -0,0 +1,76 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import datetime +import logging + +from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mbr +from volatility3.plugins import yarascan + +vollog = logging.getLogger(__name__) + + +class MBRParser(interfaces.plugins.PluginInterface): + """ Scans for and parses potential Master Boot Records (MBRs) """ + + _required_framework_version = (2, 0, 1) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, + version = (2, 0, 0)), + ] + + @classmethod + def levenshtein(self, s1, s2): + if len(s1) < len(s2): + return self.levenshtein(s2, s1) + + if len(s2) == 0: + return len(s1) + + previous_row = range(len(s2) + 1) + for i, c1 in enumerate(s1): + current_row = [i + 1] + for j, c2 in enumerate(s2): + insertions = previous_row[j + 1] + 1 + deletions = current_row[j] + 1 + substitutions = previous_row[j] + (c1 != c2) + current_row.append(min(insertions, deletions, substitutions)) + previous_row = current_row + + return previous_row[-1] + + def _generator(self): + layer = self.context.layers[self.config['primary']] + rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/\x55\xaa/'}) + symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, + config_path = self.config_path, + sub_path = "windows", + filename = "mbr", + class_types = { + 'PARTITION_ENTRY': mbr.PARTITION_ENTRY, + }) + + for offset, _rule_name, _name, _value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): + try: + yield 1, (format_hints.Hex(offset), _value) + + except exceptions.PagedInvalidAddressException: + pass + + def run(self): + return renderers.TreeGrid([ + ('Offset', format_hints.Hex), + ('Record Type', str), + ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py new file mode 100644 index 000000000..eeb97d332 --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -0,0 +1,111 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework import objects + +import struct + +PartitionTypes = { + 0x00:"Empty", + 0x01:"FAT12,CHS", + 0x04:"FAT16 16-32MB,CHS", + 0x05:"Microsoft Extended", + 0x06:"FAT16 32MB,CHS", + 0x07:"NTFS", + 0x0b:"FAT32,CHS", + 0x0c:"FAT32,LBA", + 0x0e:"FAT16, 32MB-2GB,LBA", + 0x0f:"Microsoft Extended, LBA", + 0x11:"Hidden FAT12,CHS", + 0x14:"Hidden FAT16,16-32MB,CHS", + 0x16:"Hidden FAT16,32MB-2GB,CHS", + 0x18:"AST SmartSleep Partition", + 0x1b:"Hidden FAT32,CHS", + 0x1c:"Hidden FAT32,LBA", + 0x1e:"Hidden FAT16,32MB-2GB,LBA", + 0x27:"PQservice", + 0x39:"Plan 9 partition", + 0x3c:"PartitionMagic recovery partition", + 0x42:"Microsoft MBR,Dynamic Disk", + 0x44:"GoBack partition", + 0x51:"Novell", + 0x52:"CP/M", + 0x63:"Unix System V", + 0x64:"PC-ARMOUR protected partition", + 0x82:"Solaris x86 or Linux Swap", + 0x83:"Linux", + 0x84:"Hibernation", + 0x85:"Linux Extended", + 0x86:"NTFS Volume Set", + 0x87:"NTFS Volume Set", + 0x9f:"BSD/OS", + 0xa0:"Hibernation", + 0xa1:"Hibernation", + 0xa5:"FreeBSD", + 0xa6:"OpenBSD", + 0xa8:"Mac OSX", + 0xa9:"NetBSD", + 0xab:"Mac OSX Boot", + 0xaf:"MacOS X HFS", + 0xb7:"BSDI", + 0xb8:"BSDI Swap", + 0xbb:"Boot Wizard hidden", + 0xbe:"Solaris 8 boot partition", + 0xd8:"CP/M-86", + 0xde:"Dell PowerEdge Server utilities (FAT fs)", + 0xdf:"DG/UX virtual disk manager partition", + 0xeb:"BeOS BFS", + 0xee:"EFI GPT Disk", + 0xef:"EFI System Partition", + 0xfb:"VMWare File System", + 0xfc:"VMWare Swap", +} + +class PARTITION_ENTRY(objects.StructType): + def get_value(self, char): + padded = "\x00\x00\x00" + str(char) + val = int(struct.unpack('>I', padded)[0]) + return val + + def get_type(self): + return PartitionTypes.get(self.get_value(self.PartitionType), "Invalid") + + def is_bootable(self): + return self.get_value(self.BootableFlag) == 0x80 + + def is_bootable_and_used(self): + return self.is_bootable() and self.is_used() + + def is_valid(self): + return self.get_type() != "Invalid" + + def is_used(self): + return self.get_type() != "Empty" and self.is_valid() + + def StartingSector(self): + return self.StartingCHS[1] % 64 + + def StartingCylinder(self): + return (self.StartingCHS[1] - self.StartingSector()) * 4 + self.StartingCHS[2] + + def EndingSector(self): + return self.EndingCHS[1] % 64 + + def EndingCylinder(self): + return (self.EndingCHS[1] - self.EndingSector()) * 4 + self.EndingCHS[2] + + def __str__(self): + processed_entry = "" + bootable = self.get_value(self.BootableFlag) + processed_entry = "Boot flag: {0:#x} {1}\n".format(bootable, "(Bootable)" if self.is_bootable() else '') + processed_entry += "Partition type: {0:#x} ({1})\n".format(self.get_value(self.PartitionType), self.get_type()) + processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.StartingLBA) + processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.StartingCylinder(), + self.StartingCHS[0], + self.StartingSector()) + processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.EndingCylinder(), + self.EndingCHS[0], + self.EndingSector()) + processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.SizeInSectors) + return processed_entry \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json new file mode 100644 index 000000000..83bd48f13 --- /dev/null +++ b/volatility3/framework/symbols/windows/mbr.json @@ -0,0 +1,30 @@ +{ + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Donghyun Kim", + "comment": "Using structures defined in File System Forensic Analysis pg 353+", + "datetime": "2022-01-03T13:37:00" + }, + "format": "6.1.0" + }, + { + 'PARTITION_ENTRY': [ 0x10, { + 'BootableFlag': [0x0, ['char']], # 0x80 is bootable + 'StartingCHS': [0x1, ['array', 3, ['unsigned char']]], + 'PartitionType': [0x4, ['char']], + 'EndingCHS': [0x5, ['array', 3, ['unsigned char']]], + 'StartingLBA': [0x8, ['unsigned int']], + 'SizeInSectors': [0xc, ['int']], + }], + 'PARTITION_TABLE': [ 0x200, { + 'DiskSignature': [ 0x1b8, ['array', 4, ['unsigned char']]], + 'Unused': [ 0x1bc, ['unsigned short']], + 'Entry1': [ 0x1be, ['PARTITION_ENTRY']], + 'Entry2': [ 0x1ce, ['PARTITION_ENTRY']], + 'Entry3': [ 0x1de, ['PARTITION_ENTRY']], + 'Entry4': [ 0x1ee, ['PARTITION_ENTRY']], + 'Signature': [0x1fe, ['unsigned short']], + }] + } +} \ No newline at end of file From 7570e82786f49db3e0aed591ce6a13b17a97570a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Mar 2022 17:47:37 +0900 Subject: [PATCH 107/404] Configuration Yara Rules --- .../framework/plugins/windows/mbrparser.py | 16 ++--- .../framework/symbols/windows/mbr.json | 64 +++++++++++++------ 2 files changed, 49 insertions(+), 31 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrparser.py b/volatility3/framework/plugins/windows/mbrparser.py index 37a4612f8..10afa53d2 100644 --- a/volatility3/framework/plugins/windows/mbrparser.py +++ b/volatility3/framework/plugins/windows/mbrparser.py @@ -2,13 +2,11 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import datetime import logging from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import mbr from volatility3.plugins import yarascan @@ -52,19 +50,13 @@ class MBRParser(interfaces.plugins.PluginInterface): def _generator(self): layer = self.context.layers[self.config['primary']] - rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/\x55\xaa/'}) - symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, - config_path = self.config_path, - sub_path = "windows", - filename = "mbr", - class_types = { - 'PARTITION_ENTRY': mbr.PARTITION_ENTRY, - }) + # TODO : YARA RULE HEX + rules = yarascan.YaraScan.process_yara_options({'yara_rules': "55 aa"}) for offset, _rule_name, _name, _value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): try: - yield 1, (format_hints.Hex(offset), _value) + yield 0, (format_hints.Hex(offset), _name) except exceptions.PagedInvalidAddressException: pass @@ -72,5 +64,5 @@ class MBRParser(interfaces.plugins.PluginInterface): def run(self): return renderers.TreeGrid([ ('Offset', format_hints.Hex), - ('Record Type', str), + ("Name", str) ], self._generator()) diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 83bd48f13..84162c96d 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -8,23 +8,49 @@ }, "format": "6.1.0" }, - { - 'PARTITION_ENTRY': [ 0x10, { - 'BootableFlag': [0x0, ['char']], # 0x80 is bootable - 'StartingCHS': [0x1, ['array', 3, ['unsigned char']]], - 'PartitionType': [0x4, ['char']], - 'EndingCHS': [0x5, ['array', 3, ['unsigned char']]], - 'StartingLBA': [0x8, ['unsigned int']], - 'SizeInSectors': [0xc, ['int']], - }], - 'PARTITION_TABLE': [ 0x200, { - 'DiskSignature': [ 0x1b8, ['array', 4, ['unsigned char']]], - 'Unused': [ 0x1bc, ['unsigned short']], - 'Entry1': [ 0x1be, ['PARTITION_ENTRY']], - 'Entry2': [ 0x1ce, ['PARTITION_ENTRY']], - 'Entry3': [ 0x1de, ['PARTITION_ENTRY']], - 'Entry4': [ 0x1ee, ['PARTITION_ENTRY']], - 'Signature': [0x1fe, ['unsigned short']], - }] - } + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": true, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "int", + "size": 1, + "signed": false, + "endian": "little" + }, + "wchar": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + } + }, + "symbols": {} } \ No newline at end of file From 244751e9aebf30c2f592c576b3a57ddbbe24b9ed Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 6 Mar 2022 18:48:00 +0000 Subject: [PATCH 108/404] Layers: Better checks on PAE page tables This checks that the very top level table points to the next four pages, as we'd expected in general. This relies on the same assumptions as the existing PAE detection did, ie that the PAE page_map maps the next four pages immediately. Previously we didn't check that the top page was valid, once we found the self-referential pointer. This adds in an appropriate check to reduce false positives. Closes #631. --- volatility3/framework/automagic/windows.py | 27 ++++++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 71548ca40..b93ee244c 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -28,9 +28,9 @@ The self-referential indices for older versions of windows are listed below: """ import logging import struct -from typing import Generator, List, Optional, Tuple, Type, Iterable +from typing import Generator, Iterable, List, Optional, Tuple, Type -from volatility3.framework import interfaces, layers, constants +from volatility3.framework import constants, interfaces, layers from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel @@ -116,10 +116,27 @@ class DtbSelfRefPae(DtbSelfReferential): mask = 0x3FFFFFFFFFF000, reserved_bits = 0x0) - def __call__(self, *args, **kwargs): - dtb = super().__call__(*args, **kwargs) + @staticmethod + def _and_bytes(abytes, bbytes): + return bytes([a & b for a, b in zip(abytes[::-1], bbytes[::-1])][::-1]) + + def __call__(self, data: bytes, data_offset: int, page_offset: int) -> Optional[Tuple[int, int]]: + dtb = super().__call__(data, data_offset, page_offset) if dtb: - return dtb[0] - 0x4000, dtb[1] + # Find the top page + top_pae_page = dtb[0] - 0x4000 + # The top page should map to the next four pages after it + # Build what we expect the page table to be + expected_table = b''.join([struct.pack(self.ptr_struct, top_pae_page + (i * 0x1000)) for i in range(1, 5)]) + # Mask off the page bits of top level page map + page_table_mask = b"\x00\xf0\xff\xff\xff\xff\xff\xff" * 4 + page_table = data[top_pae_page - data_offset: top_pae_page - data_offset + (4 * self.ptr_size)] + # Compare them + anded_bytes = self._and_bytes(page_table, page_table_mask) + if (anded_bytes == expected_table): + return top_pae_page, dtb[1] + # Return None since the dtb value *isn't* None + return None return dtb From 68903c63df92c2726fb9cfe75f902ada808a1d30 Mon Sep 17 00:00:00 2001 From: Samuel Zurowski Date: Sun, 6 Mar 2022 20:20:24 -0500 Subject: [PATCH 109/404] Added task_struct function to get each task_struct from the thread_nodes structure --- .../symbols/linux/extensions/__init__.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0edd60608..b48027bf5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -201,6 +201,24 @@ class task_struct(generic.GenericIntelProcess): yield (start, end - start) + def get_thread_nodes(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns a list of the task_struct based on the list_head + thread_node structure.""" + + task_symbol_table_name = self.get_symbol_table_name() + + parent = self.group_leader + for task in self.thread_node.to_list( + f"{task_symbol_table_name}{constants.BANG}task_struct", + "thread_node" + ): + + if task.group_leader != parent: continue + + yield task + + + class fs_struct(objects.StructType): From a1023e51f59aaf85b50d318b36a072772efaef1d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 7 Mar 2022 13:54:35 +0900 Subject: [PATCH 110/404] Fix Renderes, Scanners, MFT Symbol Typo Error --- volatility3/framework/interfaces/renderers.py | 2 +- volatility3/framework/layers/scanners/__init__.py | 4 ++-- volatility3/framework/symbols/windows/mft.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index 7f80425a4..9368009a9 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""All plugins output a TreeGrid object which must then be rendered (eithe by a +"""All plugins output a TreeGrid object which must then be rendered (either by a GUI, or as text output, html output or in some other form. This module defines both the output format (:class:`TreeGrid`) and the diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index 85c8390d9..ec66f2708 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -31,7 +31,7 @@ class BytesScanner(layers.ScannerInterface): class RegExScanner(layers.ScannerInterface): """A scanner that can be provided with a bytes-object regular expression pattern - The scanner will scqn all blocks for the regular expression and report the absolute offset of any finds + The scanner will scan all blocks for the regular expression and report the absolute offset of any finds The default flags include DOTALL, since the searches are through binary data and the newline character should have no specific significance in such searches""" @@ -95,7 +95,7 @@ class MultiStringScanner(layers.ScannerInterface): else: suffixes.append(re.escape(bytes([entry]))) else: - # If we've fininshed one of the strings at this point, remember it for later + # If we've finished one of the strings at this point, remember it for later finished = True if len(suffixes) == 1: diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index 17edd45dd..e5de8f3fa 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -134,7 +134,7 @@ "kind": "base", "name": "unsigned char" } - } + } }, "UpdateSequenceOffset": { "offset": 4, @@ -192,7 +192,7 @@ "name": "unsigned int" } }, - "AlocatedSize": { + "AllocatedSize": { "offset": 28, "type":{ "kind": "base", From 34a732a4f09f123746753c5b77661d44c92aeb2b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 7 Mar 2022 14:00:03 +0900 Subject: [PATCH 111/404] Fix Object Typo Error --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 472370e6b..1fa6dd62a 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -704,7 +704,7 @@ class AggregateType(interfaces.objects.ObjectInterface): tmp_list[member] = (relative_offset, new_child) # If there's trouble with mutability, consider making update_vol return a clone with the changes # (there will be a few other places that will be necessary) and/or making these part of the - # permanent dictionaries rather than the non-clonable ones + # permanent dictionaries rather than the non-cloneable ones template.update_vol(members = tmp_list) @classmethod From 4087236957d2567d9054f1ceb41700b054e3b835 Mon Sep 17 00:00:00 2001 From: Samuel Zurowski Date: Mon, 7 Mar 2022 19:18:17 -0500 Subject: [PATCH 112/404] Changed named and used thread_group instead to ensure all threads are grabbed --- .../symbols/linux/extensions/__init__.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b48027bf5..2fb832930 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -201,25 +201,22 @@ class task_struct(generic.GenericIntelProcess): yield (start, end - start) - def get_thread_nodes(self) -> Iterable[interfaces.objects.ObjectInterface]: + def get_threads(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns a list of the task_struct based on the list_head thread_node structure.""" task_symbol_table_name = self.get_symbol_table_name() - parent = self.group_leader - for task in self.thread_node.to_list( + # iterating through the thread_list from thread_group + # this allows iterating through pointers to grab the + # threads and using the thread_group offset to get the + # corresponding task_struct + for task in self.thread_group.to_list( f"{task_symbol_table_name}{constants.BANG}task_struct", - "thread_node" + "thread_group" ): - - if task.group_leader != parent: continue - yield task - - - class fs_struct(objects.StructType): def get_root_dentry(self): From 6c1fe42a3791f1702df1c2733c886cb96c7a1100 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 01:53:04 +0900 Subject: [PATCH 113/404] Fix Docs, Framework, Windows Plugin Typo Error --- doc/source/complex-plugin.rst | 2 +- doc/source/simple-plugin.rst | 4 ++-- volatility3/framework/interfaces/configuration.py | 2 +- volatility3/framework/objects/__init__.py | 2 +- volatility3/framework/plugins/windows/privileges.py | 4 ++-- volatility3/framework/plugins/windows/psscan.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/complex-plugin.rst b/doc/source/complex-plugin.rst index f06b398e8..8ab8a5186 100644 --- a/doc/source/complex-plugin.rst +++ b/doc/source/complex-plugin.rst @@ -300,7 +300,7 @@ This will mean that when a specific structure is loaded from the symbol_space, i `StructType`, but instead is instantiated using the NewStructureClass, meaning new methods can be called directly on it. If the situation really calls for an entirely new object, that isn't covered by one of the existing -:py:class:`~volatility3.framework.objects.PrimativeObject` objects (such as +:py:class:`~volatility3.framework.objects.PrimitiveObject` objects (such as :py:class:`~volatility3.framework.objects.Integer`, :py:class:`~volatility3.framework.objects.Boolean`, :py:class:`~volatility3.framework.objects.Float`, diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 4e499b186..9360ccf40 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -206,9 +206,9 @@ information may not be provided. The plugin then takes the process's ``BaseDllName`` value, and calls :py:meth:`~volatility3.framework.symbols.windows.extensions.UNICODE_STRING.get_string` on it. All structure attributes, as defined by the symbols, are directly accessible and use the case-style of the symbol library it came from (in Windows, -attributes are CamelCase), such as ``entry.BaseDllName`` in this instance. Any attribtues not defined by the symbol but added +attributes are CamelCase), such as ``entry.BaseDllName`` in this instance. Any attributes not defined by the symbol but added by Volatility extensions cannot be properties (in case they overlap with the attributes defined in the symbol libraries) -and are therefore always methods and prepended with ``get_``, in this example ``BaseDllName.get_string()``. +and are therefore always methods and pretended with ``get_``, in this example ``BaseDllName.get_string()``. Finally, ``FullDllName`` is populated. These operations read from memory, and as such, the memory image may be unable to read the data at a particular offset. This will cause an exception to be thrown. In Volatility 3, exceptions are thrown diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 7dc046a3e..c39dba680 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -73,7 +73,7 @@ class HierarchicalDict(collections.abc.Mapping): separator: str = CONFIG_SEPARATOR) -> None: """ Args: - initial_dict: A dictionary to populate the HierachicalDict with initially + initial_dict: A dictionary to populate the HierarchicalDict with initially separator: A custom hierarchy separator (defaults to CONFIG_SEPARATOR) """ if not (isinstance(separator, str) and len(separator) == 1): diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 1fa6dd62a..e0f927ec9 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -206,7 +206,7 @@ class Bytes(PrimitiveObject, bytes): length: int = 1, **kwargs) -> 'Bytes': """Creates the appropriate class and returns it so that the native type - is inherritted. + is inherited. The only reason the kwargs is added, is so that the inheriting types can override __init__ without needing to diff --git a/volatility3/framework/plugins/windows/privileges.py b/volatility3/framework/plugins/windows/privileges.py index eaafbeae6..2d48a30f7 100644 --- a/volatility3/framework/plugins/windows/privileges.py +++ b/volatility3/framework/plugins/windows/privileges.py @@ -55,14 +55,14 @@ class Privs(interfaces.plugins.PluginInterface): try: process_token = task.Token.dereference().cast("_TOKEN") except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, 'Skeep invalid token.') + vollog.log(constants.LOGLEVEL_VVV, 'Skip invalid token.') continue for value, present, enabled, default in process_token.privileges(): # Skip privileges whose bit positions cannot be # translated to a privilege name if not self.privilege_info.get(int(value)): - vollog.log(constants.LOGLEVEL_VVV, f'Skeep invalid privilege ({value}).') + vollog.log(constants.LOGLEVEL_VVV, f'Skip invalid privilege ({value}).') continue name, desc = self.privilege_info.get(int(value)) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 237c0edcd..a0601aef1 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -85,7 +85,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): context: The context to retrieve required elements (layers, symbol tables) from layer_name: The name of the layer on which to operate symbol_table: The name of the table containing the kernel symbols - proc: the process object with phisical address + proc: the process object with physical address Returns: A process object on virtual address layer From 18770d0cd3b0dff138a33b98d39e77743086ac3c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 02:09:47 +0900 Subject: [PATCH 114/404] Fix glossary.rst Typo Error --- doc/source/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 68bc41e4c..66dabfafe 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -145,7 +145,7 @@ Struct, Structure Symbol This is used in many different contexts, as a short term for many things. Within Volatility, a symbol is a - construct that usually encompasses a specific type :ref:`type` at a specfific :ref:`offset`, + construct that usually encompasses a specific type :ref:`type` at a specific :ref:`offset`, representing a particular instance of that type within the memory of a compiled and running program. An example would be the location in memory of a list of active tcp endpoints maintained by the networking stack within an operating system. From acc3f6f352d9446c773fd3ebf5891f67ba9d214b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 13:27:53 +0900 Subject: [PATCH 115/404] Update Symbol Table, Load Physical Layer --- .../framework/plugins/windows/mbrparser.py | 68 ------ .../framework/plugins/windows/mbrscan.py | 80 +++++++ .../symbols/windows/extensions/mbr.py | 110 +-------- .../framework/symbols/windows/mbr.json | 208 +++++++++++++++++- 4 files changed, 292 insertions(+), 174 deletions(-) delete mode 100644 volatility3/framework/plugins/windows/mbrparser.py create mode 100644 volatility3/framework/plugins/windows/mbrscan.py diff --git a/volatility3/framework/plugins/windows/mbrparser.py b/volatility3/framework/plugins/windows/mbrparser.py deleted file mode 100644 index 10afa53d2..000000000 --- a/volatility3/framework/plugins/windows/mbrparser.py +++ /dev/null @@ -1,68 +0,0 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -import logging - -from volatility3.framework import exceptions, interfaces, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols.windows.extensions import mbr -from volatility3.plugins import yarascan - -vollog = logging.getLogger(__name__) - - -class MBRParser(interfaces.plugins.PluginInterface): - """ Scans for and parses potential Master Boot Records (MBRs) """ - - _required_framework_version = (2, 0, 1) - - @classmethod - def get_requirements(cls): - return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, - version = (2, 0, 0)), - ] - - @classmethod - def levenshtein(self, s1, s2): - if len(s1) < len(s2): - return self.levenshtein(s2, s1) - - if len(s2) == 0: - return len(s1) - - previous_row = range(len(s2) + 1) - for i, c1 in enumerate(s1): - current_row = [i + 1] - for j, c2 in enumerate(s2): - insertions = previous_row[j + 1] + 1 - deletions = current_row[j] + 1 - substitutions = previous_row[j] + (c1 != c2) - current_row.append(min(insertions, deletions, substitutions)) - previous_row = current_row - - return previous_row[-1] - - def _generator(self): - layer = self.context.layers[self.config['primary']] - # TODO : YARA RULE HEX - rules = yarascan.YaraScan.process_yara_options({'yara_rules': "55 aa"}) - - for offset, _rule_name, _name, _value in layer.scan(context = self.context, - scanner = yarascan.YaraScanner(rules = rules)): - try: - yield 0, (format_hints.Hex(offset), _name) - - except exceptions.PagedInvalidAddressException: - pass - - def run(self): - return renderers.TreeGrid([ - ('Offset', format_hints.Hex), - ("Name", str) - ], self._generator()) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py new file mode 100644 index 000000000..3876d2ac8 --- /dev/null +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -0,0 +1,80 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from volatility3.framework import constants, interfaces, renderers, symbols +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mbr + +vollog = logging.getLogger(__name__) + +class MBRScan(interfaces.plugins.PluginInterface): + """ Scans for and parses potential Master Boot Records (MBRs) """ + + _required_framework_version = (2, 0, 1) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]) + ] + + def _generator(self): + kernel = self.context.modules[self.config['kernel']] + physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) + + layer = self.context.layers[physical_layer_name] + architecture = "intel" if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) else "intel64" + + symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, + config_path = self.config_path, + sub_path = "windows", + filename = "mbr", + class_types = { + 'PARTITION_TABLE': mbr.PARTITION_TABLE, + 'PARTITION_ENTRY': mbr.PARTITION_ENTRY + }) + + partition_table_object = symbol_table + constants.BANG + "PARTITION_TABLE" + + mbr_signature = b"\x55\xAA" + mbr_length = 0x200 + boot_code_length = 0x1B8 + + for offset, _value in layer.scan(context = self.context, scanner = scanners.MultiStringScanner(patterns = [mbr_signature])): + mbr_start_offset = offset - (mbr_length - len(mbr_signature)) + partition_table = self.context.object(partition_table_object, offset = mbr_start_offset, layer_name = layer.name) + + boot_code = layer.read(mbr_start_offset, boot_code_length, pad = True) + + if boot_code: + all_zeros = boot_code.count(b"\x00") == len(boot_code) + + if not all_zeros: + partition_type = partition_table.FirstEntry.PartitionType + + + if partition_type.is_valid_choice: + yield 0, ( + format_hints.Hex(offset), + partition_type.lookup(), + interfaces.renderers.Disassembly(boot_code, 0, architecture), + format_hints.HexBytes(boot_code) + ) + else: + vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + + def run(self): + return renderers.TreeGrid([ + ('Offset', format_hints.Hex), + ('PartitionType', str), + ("Disasm", interfaces.renderers.Disassembly), + ("Hexdump", format_hints.HexBytes) + ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index eeb97d332..9bcefe12e 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -6,106 +6,14 @@ from volatility3.framework import objects import struct -PartitionTypes = { - 0x00:"Empty", - 0x01:"FAT12,CHS", - 0x04:"FAT16 16-32MB,CHS", - 0x05:"Microsoft Extended", - 0x06:"FAT16 32MB,CHS", - 0x07:"NTFS", - 0x0b:"FAT32,CHS", - 0x0c:"FAT32,LBA", - 0x0e:"FAT16, 32MB-2GB,LBA", - 0x0f:"Microsoft Extended, LBA", - 0x11:"Hidden FAT12,CHS", - 0x14:"Hidden FAT16,16-32MB,CHS", - 0x16:"Hidden FAT16,32MB-2GB,CHS", - 0x18:"AST SmartSleep Partition", - 0x1b:"Hidden FAT32,CHS", - 0x1c:"Hidden FAT32,LBA", - 0x1e:"Hidden FAT16,32MB-2GB,LBA", - 0x27:"PQservice", - 0x39:"Plan 9 partition", - 0x3c:"PartitionMagic recovery partition", - 0x42:"Microsoft MBR,Dynamic Disk", - 0x44:"GoBack partition", - 0x51:"Novell", - 0x52:"CP/M", - 0x63:"Unix System V", - 0x64:"PC-ARMOUR protected partition", - 0x82:"Solaris x86 or Linux Swap", - 0x83:"Linux", - 0x84:"Hibernation", - 0x85:"Linux Extended", - 0x86:"NTFS Volume Set", - 0x87:"NTFS Volume Set", - 0x9f:"BSD/OS", - 0xa0:"Hibernation", - 0xa1:"Hibernation", - 0xa5:"FreeBSD", - 0xa6:"OpenBSD", - 0xa8:"Mac OSX", - 0xa9:"NetBSD", - 0xab:"Mac OSX Boot", - 0xaf:"MacOS X HFS", - 0xb7:"BSDI", - 0xb8:"BSDI Swap", - 0xbb:"Boot Wizard hidden", - 0xbe:"Solaris 8 boot partition", - 0xd8:"CP/M-86", - 0xde:"Dell PowerEdge Server utilities (FAT fs)", - 0xdf:"DG/UX virtual disk manager partition", - 0xeb:"BeOS BFS", - 0xee:"EFI GPT Disk", - 0xef:"EFI System Partition", - 0xfb:"VMWare File System", - 0xfc:"VMWare Swap", -} +class PARTITION_TABLE(objects.StructType): + + def get_disk_signature(self) -> str: + signature = self.DiskSignature.values + return signature class PARTITION_ENTRY(objects.StructType): - def get_value(self, char): - padded = "\x00\x00\x00" + str(char) - val = int(struct.unpack('>I', padded)[0]) - return val - - def get_type(self): - return PartitionTypes.get(self.get_value(self.PartitionType), "Invalid") - - def is_bootable(self): - return self.get_value(self.BootableFlag) == 0x80 - - def is_bootable_and_used(self): - return self.is_bootable() and self.is_used() - - def is_valid(self): - return self.get_type() != "Invalid" - - def is_used(self): - return self.get_type() != "Empty" and self.is_valid() - - def StartingSector(self): - return self.StartingCHS[1] % 64 - - def StartingCylinder(self): - return (self.StartingCHS[1] - self.StartingSector()) * 4 + self.StartingCHS[2] - - def EndingSector(self): - return self.EndingCHS[1] % 64 - - def EndingCylinder(self): - return (self.EndingCHS[1] - self.EndingSector()) * 4 + self.EndingCHS[2] - - def __str__(self): - processed_entry = "" - bootable = self.get_value(self.BootableFlag) - processed_entry = "Boot flag: {0:#x} {1}\n".format(bootable, "(Bootable)" if self.is_bootable() else '') - processed_entry += "Partition type: {0:#x} ({1})\n".format(self.get_value(self.PartitionType), self.get_type()) - processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.StartingLBA) - processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.StartingCylinder(), - self.StartingCHS[0], - self.StartingSector()) - processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.EndingCylinder(), - self.EndingCHS[0], - self.EndingSector()) - processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.SizeInSectors) - return processed_entry \ No newline at end of file + + def get_partition_type(self, type: int) -> str: + + return "Hello" diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 84162c96d..382af403e 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -2,9 +2,9 @@ "metadata": { "producer": { "version": "0.0.1", - "name": "Donghyun Kim", - "comment": "Using structures defined in File System Forensic Analysis pg 353+", - "datetime": "2022-01-03T13:37:00" + "name": "Donghyun Kim (@digitalisx99)", + "comment": "Using structures defined in File System Forensic Analysis pg 88+", + "datetime": "2022-03-05T10:53:00" }, "format": "6.1.0" }, @@ -32,6 +32,12 @@ "size": 4, "signed": false, "endian": "little" + }, + "int": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 4 }, "unsigned short": { "kind": "int", @@ -45,12 +51,204 @@ "signed": false, "endian": "little" }, + "char": { + "endian": "little", + "kind": "char", + "signed": true, + "size": 1 + }, "wchar": { "kind": "int", "size": 2, "signed": true, "endian": "little" } - }, - "symbols": {} + }, + "symbols": {}, + "enums": { + "BootableFlag":{ + "base": "unsigned char", + "constants": { + "Bootable": 0, + "Non-Bootable": 128 + }, + "size": 1 + }, + "PartitionTypes": { + "base": "unsigned char", + "constants": { + "Empty": 0, + "FAT12,CHS": 1, + "FAT16 16-32MB,CHS": 4, + "Microsoft Extended": 5, + "FAT16 32MB,CHS": 6, + "NTFS": 7, + "FAT32,CHS": 11, + "FAT32,LBA": 12, + "FAT16, 32MB-2GB,LBA": 14, + "Microsoft Extended, LBA": 15, + "Hidden FAT12,CHS": 17, + "Hidden FAT16,16-32MB,CHS": 20, + "Hidden FAT16,32MB-2GB,CHS": 22, + "AST SmartSleep Partition": 24, + "Hidden FAT32,CHS": 27, + "Hidden FAT32,LBA": 28, + "Hidden FAT16,32MB-2GB,LBA": 30, + "PQservice": 39, + "Plan 9 partition": 57, + "PartitionMagic recovery partition": 60, + "Microsoft MBR,Dynamic Disk": 66, + "GoBack partition": 68, + "Novell": 81, + "CP/M": 82, + "Unix System V": 99, + "PC-ARMOUR protected partition": 100, + "Solaris x86 or Linux Swap": 130, + "Linux": 131, + "Hibernation": 132, + "Linux Extended": 133, + "NTFS Volume Set": 134, + "NTFS Volume Set": 135, + "BSD/OS": 159, + "Hibernation": 160, + "Hibernation": 161, + "FreeBSD": 165, + "OpenBSD": 166, + "Mac OSX": 168, + "NetBSD": 169, + "Mac OSX Boot": 171, + "MacOS X HFS": 175, + "BSDI": 183, + "BSDI Swap": 184, + "Boot Wizard hidden": 187, + "Solaris 8 boot partition": 190, + "CP/M-86": 216, + "Dell PowerEdge Server utilities (FAT fs)": 222, + "DG/UX virtual disk manager partition": 223, + "BeOS BFS": 235, + "EFI GPT Disk": 238, + "EFI System Partition": 239, + "VMWare File System": 251, + "VMWare Swap": 252 + }, + "size": 1 + } + }, + "user_types": { + "PARTITION_ENTRY":{ + "fields": { + "BootableFlag": { + "offset": 0, + "type": { + "kind": "enum", + "name": "BootableFlag" + } + }, + "StartingCHS": { + "offset": 1, + "type": { + "count": 3, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "PartitionType": { + "offset": 4, + "type": { + "kind": "enum", + "name": "PartitionTypes" + } + }, + "EndingCHS": { + "offset": 5, + "type": { + "count": 3, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "StartingLBA": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "SizeInSectors": { + "offset": 12, + "type": { + "kind": "base", + "name": "int" + } + } + }, + "kind": "struct", + "size": 16 + }, + "PARTITION_TABLE":{ + "fields":{ + "DiskSignature": { + "offset": 440, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "Unused": { + "offset": 444, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "FirstEntry":{ + "offset": 446, + "type": { + "kind": "struct", + "name": "PARTITION_ENTRY" + } + }, + "SecondEntry":{ + "offset": 462, + "type": { + "kind": "struct", + "name": "PARTITION_ENTRY" + } + }, + "ThirdEntry":{ + "offset": 478, + "type": { + "kind": "struct", + "name": "PARTITION_ENTRY" + } + }, + "FourthEntry":{ + "offset": 494, + "type": { + "kind": "struct", + "name": "PARTITION_ENTRY" + } + }, + "Signature":{ + "offset": 510, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 512 + } + } } \ No newline at end of file From eda765d61d316d1d49d0008164a310743d710d5a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 16:12:32 +0900 Subject: [PATCH 116/404] Update MBR Partition Entry Object Function --- .../framework/plugins/windows/mbrscan.py | 32 ++++++------ .../symbols/windows/extensions/mbr.py | 51 +++++++++++++++++-- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 3876d2ac8..6bfed2bde 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -56,25 +56,27 @@ class MBRScan(interfaces.plugins.PluginInterface): if boot_code: all_zeros = boot_code.count(b"\x00") == len(boot_code) - - if not all_zeros: - partition_type = partition_table.FirstEntry.PartitionType - - if partition_type.is_valid_choice: - yield 0, ( - format_hints.Hex(offset), - partition_type.lookup(), - interfaces.renderers.Disassembly(boot_code, 0, architecture), - format_hints.HexBytes(boot_code) - ) + if not all_zeros: + partition_entry_list = ["FirstEntry", "SecondEntry", "ThirdEntry", "FourthEntry"] + #partition_type = getattr(partition_table, "FirstEntry").PartitionType + yield 0, ( + format_hints.Hex(offset), + partition_table.FirstEntry.get_bootable_flag(), + partition_table.FirstEntry.get_partition_type(), + format_hints.Hex(partition_table.FirstEntry.get_starting_chs()) + #interfaces.renderers.Disassembly(boot_code, 0, architecture), + #format_hints.HexBytes(boot_code) + ) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") def run(self): return renderers.TreeGrid([ - ('Offset', format_hints.Hex), - ('PartitionType', str), - ("Disasm", interfaces.renderers.Disassembly), - ("Hexdump", format_hints.HexBytes) + ("Offset", format_hints.Hex), + ("Bootable", bool), + ("Partition Type", str), + ("Starting CHS",format_hints.Hex) + #("Disasm", interfaces.renderers.Disassembly), + #("Hexdump", format_hints.HexBytes) ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 9bcefe12e..c02d741df 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -4,8 +4,6 @@ from volatility3.framework import objects -import struct - class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: @@ -14,6 +12,49 @@ class PARTITION_TABLE(objects.StructType): class PARTITION_ENTRY(objects.StructType): - def get_partition_type(self, type: int) -> str: - - return "Hello" + def get_bootable_flag(self) -> int: + return self.BootableFlag + + def is_bootable(self) -> bool: + return False if not (self.BootableFlag == 0x80) else True + + def get_partition_type(self) -> str: + return self.PartitionType.lookup() if self.PartitionType.is_valid_choice else "Not Defined PartitionType" + + def get_starting_chs(self): + return self.StartingCHS[0] + + def get_ending_chs(self): + return self.EndingCHS[0] + + def get_starting_sector(self): + return self.StartingCHS[1] % 64 + + def get_starting_cylinder(self): + return (self.StartingCHS[1] - self.get_starting_sector()) * 4 + self.StartingCHS[2] + + def get_ending_sector(self): + return self.EndingCHS[1] % 64 + + def get_ending_cylinder(self): + return (self.EndingCHS[1] - self.get_ending_sector()) * 4 + self.EndingCHS[2] + + def get_starting_lba(self): + return self.StartingLBA + + def get_size_in_sectors(self): + return self.SizeInSectors + + def __str__(self): + processed_entry = "" + processed_entry = "Boot flag: {0:#x} {1}\n".format(self.is_bootable(), "(Bootable)" if self.is_bootable() else '') + processed_entry += "Partition type: {0:#x} ({1})\n".format(self.get_value(self.PartitionType), self.get_type()) + processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.StartingLBA) + processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.StartingCylinder(), + self.StartingCHS[0], + self.StartingSector()) + processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.EndingCylinder(), + self.EndingCHS[0], + self.EndingSector()) + processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.SizeInSectors) + return processed_entry From e0a512e9ff17a81b7c36c79da01c894eab72bbc5 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 16:13:25 +0900 Subject: [PATCH 117/404] Add EOF of MBR Symbol --- volatility3/framework/symbols/windows/mft.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index e5de8f3fa..6881c92be 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -466,4 +466,4 @@ "size": 1024 } } -} \ No newline at end of file +} From b01333115b06ba106250916e2390888870794b13 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 17:08:27 +0900 Subject: [PATCH 118/404] __str__ Formatting --- .../framework/plugins/windows/mbrscan.py | 26 ++++++------ .../symbols/windows/extensions/mbr.py | 40 +++++++++++++------ .../framework/symbols/windows/mbr.json | 2 +- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 6bfed2bde..beda342fb 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -58,15 +58,18 @@ class MBRScan(interfaces.plugins.PluginInterface): all_zeros = boot_code.count(b"\x00") == len(boot_code) if not all_zeros: - partition_entry_list = ["FirstEntry", "SecondEntry", "ThirdEntry", "FourthEntry"] - #partition_type = getattr(partition_table, "FirstEntry").PartitionType + + first_entry = partition_table.FirstEntry + second_entry = partition_table.SecondEntry + third_entry = partition_table.ThirdEntry + fourth_entry = partition_table.FourthEntry + yield 0, ( format_hints.Hex(offset), - partition_table.FirstEntry.get_bootable_flag(), - partition_table.FirstEntry.get_partition_type(), - format_hints.Hex(partition_table.FirstEntry.get_starting_chs()) - #interfaces.renderers.Disassembly(boot_code, 0, architecture), - #format_hints.HexBytes(boot_code) + partition_table.get_disk_signature(), + str(partition_table.FirstEntry), + interfaces.renderers.Disassembly(boot_code, 0, architecture), + format_hints.HexBytes(boot_code) ) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") @@ -74,9 +77,8 @@ class MBRScan(interfaces.plugins.PluginInterface): def run(self): return renderers.TreeGrid([ ("Offset", format_hints.Hex), - ("Bootable", bool), - ("Partition Type", str), - ("Starting CHS",format_hints.Hex) - #("Disasm", interfaces.renderers.Disassembly), - #("Hexdump", format_hints.HexBytes) + ("Disk Signature", str), + ("First Entry", str), + ("Disasm", interfaces.renderers.Disassembly), + ("Hexdump", format_hints.HexBytes) ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index c02d741df..e4aaefad1 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -7,8 +7,12 @@ from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: - signature = self.DiskSignature.values - return signature + return "{0:02x}-{1:02x}-{2:02x}-{3:02x}".format( + self.DiskSignature[0], + self.DiskSignature[1], + self.DiskSignature[2], + self.DiskSignature[3] + ) class PARTITION_ENTRY(objects.StructType): @@ -46,15 +50,25 @@ class PARTITION_ENTRY(objects.StructType): return self.SizeInSectors def __str__(self): - processed_entry = "" - processed_entry = "Boot flag: {0:#x} {1}\n".format(self.is_bootable(), "(Bootable)" if self.is_bootable() else '') - processed_entry += "Partition type: {0:#x} ({1})\n".format(self.get_value(self.PartitionType), self.get_type()) - processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.StartingLBA) - processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.StartingCylinder(), - self.StartingCHS[0], - self.StartingSector()) - processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.EndingCylinder(), - self.EndingCHS[0], - self.EndingSector()) - processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.SizeInSectors) + processed_entry = "========= Partition Info =========\n" + processed_entry += "Boot Flag: {0:#x} {1}\n".format( + self.is_bootable(), + "(Bootable)" if self.is_bootable() else '' + ) + processed_entry += "Partition Type: {0:#x} ({1})\n".format( + self.PartitionType, + self.get_partition_type() + ) + processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.get_starting_lba()) + processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format( + self.get_starting_cylinder(), + self.get_starting_chs(), + self.get_starting_sector() + ) + processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format( + self.get_ending_cylinder(), + self.get_ending_chs(), + self.get_ending_sector() + ) + processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.get_size_in_sectors()) return processed_entry diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 382af403e..9173633d1 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -251,4 +251,4 @@ "size": 512 } } -} \ No newline at end of file +} From 5de6462fae23f4702d5b7c209e9d151c6589dd91 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 17:09:36 +0900 Subject: [PATCH 119/404] Restore mft.json --- volatility3/framework/symbols/windows/mft.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index 6881c92be..e5de8f3fa 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -466,4 +466,4 @@ "size": 1024 } } -} +} \ No newline at end of file From b6a14e6de4fab2ec2473b1f3492a6a69f063f1b3 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 23:42:21 +0900 Subject: [PATCH 120/404] Add Symbol code comment, hash --- .../framework/plugins/windows/mbrscan.py | 29 +++++++++++++------ .../symbols/windows/extensions/mbr.py | 26 +++++++++++++---- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index beda342fb..fcc9dabb1 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -3,6 +3,7 @@ # import logging +import hashlib from volatility3.framework import constants, interfaces, renderers, symbols from volatility3.framework.configuration import requirements @@ -52,22 +53,30 @@ class MBRScan(interfaces.plugins.PluginInterface): mbr_start_offset = offset - (mbr_length - len(mbr_signature)) partition_table = self.context.object(partition_table_object, offset = mbr_start_offset, layer_name = layer.name) - boot_code = layer.read(mbr_start_offset, boot_code_length, pad = True) + full_mbr = layer.read(mbr_start_offset, mbr_length, pad = True) + boot_code = full_mbr[:boot_code_length] if boot_code: all_zeros = boot_code.count(b"\x00") == len(boot_code) if not all_zeros: - - first_entry = partition_table.FirstEntry - second_entry = partition_table.SecondEntry - third_entry = partition_table.ThirdEntry - fourth_entry = partition_table.FourthEntry + bootcode_hash = hashlib.md5(boot_code).hexdigest() + full_bootcode_hash = hashlib.md5(full_mbr).hexdigest() + partition_entries = [ partition_table.FirstEntry, partition_table.SecondEntry, + partition_table.ThirdEntry, partition_table.FourthEntry ] + partition_info = "" + + for index, partition_entry_object in enumerate(partition_entries): + partition_entry_object.set_index(index) + partition_info += str(partition_entry_object) + yield 0, ( format_hints.Hex(offset), partition_table.get_disk_signature(), - str(partition_table.FirstEntry), + bootcode_hash, + full_bootcode_hash, + partition_info, interfaces.renderers.Disassembly(boot_code, 0, architecture), format_hints.HexBytes(boot_code) ) @@ -76,9 +85,11 @@ class MBRScan(interfaces.plugins.PluginInterface): def run(self): return renderers.TreeGrid([ - ("Offset", format_hints.Hex), + ("Potential MBR at Physical Offset", format_hints.Hex), ("Disk Signature", str), - ("First Entry", str), + ("Bootcode md5", str), + ("Bootcode (FULL) md5", str), + ("Partition Entries Info", str), ("Disasm", interfaces.renderers.Disassembly), ("Hexdump", format_hints.HexBytes) ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index e4aaefad1..8e02db822 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -7,6 +7,7 @@ from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: + """Get Disk Signature (GUID).""" return "{0:02x}-{1:02x}-{2:02x}-{3:02x}".format( self.DiskSignature[0], self.DiskSignature[1], @@ -15,42 +16,57 @@ class PARTITION_TABLE(objects.StructType): ) class PARTITION_ENTRY(objects.StructType): + + def set_index(self, index:int): + self.index = index def get_bootable_flag(self) -> int: + """Get Bootable Flag.""" return self.BootableFlag def is_bootable(self) -> bool: + """Check Bootable Partition.""" return False if not (self.BootableFlag == 0x80) else True def get_partition_type(self) -> str: + """Get Partition Type.""" return self.PartitionType.lookup() if self.PartitionType.is_valid_choice else "Not Defined PartitionType" def get_starting_chs(self): + """Get Starting CHS (Cylinder Header Sector) Address.""" return self.StartingCHS[0] def get_ending_chs(self): + """Get Ending CHS (Cylinder Header Sector) Address.""" return self.EndingCHS[0] def get_starting_sector(self): + """Get Starting Sector.""" return self.StartingCHS[1] % 64 - def get_starting_cylinder(self): - return (self.StartingCHS[1] - self.get_starting_sector()) * 4 + self.StartingCHS[2] - def get_ending_sector(self): + """Get Ending Sector.""" return self.EndingCHS[1] % 64 + def get_starting_cylinder(self): + """Get Starting Cylinder.""" + return (self.StartingCHS[1] - self.get_starting_sector()) * 4 + self.StartingCHS[2] + def get_ending_cylinder(self): + """Get Ending Cylinder.""" return (self.EndingCHS[1] - self.get_ending_sector()) * 4 + self.EndingCHS[2] def get_starting_lba(self): + """Get Starting LBA (Logical Block Addressing).""" return self.StartingLBA def get_size_in_sectors(self): + """Get Size in Sectors.""" return self.SizeInSectors def __str__(self): - processed_entry = "========= Partition Info =========\n" + """Get overall of Partition Entry Info""" + processed_entry = "\n===== Partition Table #{0} =====\n".format(self.index+1) processed_entry += "Boot Flag: {0:#x} {1}\n".format( self.is_bootable(), "(Bootable)" if self.is_bootable() else '' @@ -70,5 +86,5 @@ class PARTITION_ENTRY(objects.StructType): self.get_ending_chs(), self.get_ending_sector() ) - processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.get_size_in_sectors()) + processed_entry += "Size in Sectors: {0:#x} ({0})\n".format(self.get_size_in_sectors()) return processed_entry From 7c00b2f4ea04a9d7fa171ee37301e090d42ae383 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 10 Mar 2022 00:13:46 +0900 Subject: [PATCH 121/404] Add Code Comment, Hash Funtion, Exception --- .../framework/plugins/windows/mbrscan.py | 78 +++++++++++-------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index fcc9dabb1..f0d52f418 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -5,7 +5,7 @@ import logging import hashlib -from volatility3.framework import constants, interfaces, renderers, symbols +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints @@ -27,13 +27,19 @@ class MBRScan(interfaces.plugins.PluginInterface): architectures = ["Intel32", "Intel64"]) ] + @classmethod + def get_hash(cls, data:bytes) -> str: + return hashlib.md5(data).hexdigest() + def _generator(self): kernel = self.context.modules[self.config['kernel']] physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) + # Decide of Memory Dump Architecture layer = self.context.layers[physical_layer_name] architecture = "intel" if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) else "intel64" + # Read in the Symbol File symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, config_path = self.config_path, sub_path = "windows", @@ -45,43 +51,51 @@ class MBRScan(interfaces.plugins.PluginInterface): partition_table_object = symbol_table + constants.BANG + "PARTITION_TABLE" + # Define Signature and Data Length mbr_signature = b"\x55\xAA" mbr_length = 0x200 - boot_code_length = 0x1B8 + bootcode_length = 0x1B8 + # Scan the Layer for Raw Master Boot Record (MBR) and parse the fields for offset, _value in layer.scan(context = self.context, scanner = scanners.MultiStringScanner(patterns = [mbr_signature])): - mbr_start_offset = offset - (mbr_length - len(mbr_signature)) - partition_table = self.context.object(partition_table_object, offset = mbr_start_offset, layer_name = layer.name) + try: + mbr_start_offset = offset - (mbr_length - len(mbr_signature)) + partition_table = self.context.object(partition_table_object, offset = mbr_start_offset, layer_name = layer.name) - full_mbr = layer.read(mbr_start_offset, mbr_length, pad = True) - boot_code = full_mbr[:boot_code_length] - - if boot_code: - all_zeros = boot_code.count(b"\x00") == len(boot_code) - - if not all_zeros: - bootcode_hash = hashlib.md5(boot_code).hexdigest() - full_bootcode_hash = hashlib.md5(full_mbr).hexdigest() - - partition_entries = [ partition_table.FirstEntry, partition_table.SecondEntry, - partition_table.ThirdEntry, partition_table.FourthEntry ] - partition_info = "" - - for index, partition_entry_object in enumerate(partition_entries): - partition_entry_object.set_index(index) - partition_info += str(partition_entry_object) + # Extract only BootCode + full_mbr = layer.read(mbr_start_offset, mbr_length, pad = True) + bootcode = full_mbr[:bootcode_length] - yield 0, ( - format_hints.Hex(offset), - partition_table.get_disk_signature(), - bootcode_hash, - full_bootcode_hash, - partition_info, - interfaces.renderers.Disassembly(boot_code, 0, architecture), - format_hints.HexBytes(boot_code) - ) - else: - vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + if bootcode: + all_zeros = bootcode.count(b"\x00") == len(bootcode) + + if not all_zeros: + partition_entries = [ + partition_table.FirstEntry, + partition_table.SecondEntry, + partition_table.ThirdEntry, + partition_table.FourthEntry + ] + partition_info = "\n" + + for index, partition_entry_object in enumerate(partition_entries): + partition_entry_object.set_index(index) + partition_info += str(partition_entry_object) + + yield 0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_info, + interfaces.renderers.Disassembly(bootcode, 0, architecture), + format_hints.HexBytes(bootcode) + ) + else: + vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + + except exceptions.PagedInvalidAddressException: + pass def run(self): return renderers.TreeGrid([ From a35fa04f00929dc1a4e50c3db5e107bdcf6b49b4 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 10 Mar 2022 01:12:49 +0900 Subject: [PATCH 122/404] Update BootableFlag Symbol --- .../symbols/windows/extensions/mbr.py | 7 ++++-- .../framework/symbols/windows/mbr.json | 22 ++++--------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 8e02db822..7cb4c1463 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -2,6 +2,8 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import struct + from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): @@ -18,6 +20,7 @@ class PARTITION_TABLE(objects.StructType): class PARTITION_ENTRY(objects.StructType): def set_index(self, index:int): + """Set Partition Entry Index.""" self.index = index def get_bootable_flag(self) -> int: @@ -26,7 +29,7 @@ class PARTITION_ENTRY(objects.StructType): def is_bootable(self) -> bool: """Check Bootable Partition.""" - return False if not (self.BootableFlag == 0x80) else True + return False if not (self.get_bootable_flag() == 0x80) else True def get_partition_type(self) -> str: """Get Partition Type.""" @@ -68,7 +71,7 @@ class PARTITION_ENTRY(objects.StructType): """Get overall of Partition Entry Info""" processed_entry = "\n===== Partition Table #{0} =====\n".format(self.index+1) processed_entry += "Boot Flag: {0:#x} {1}\n".format( - self.is_bootable(), + self.get_bootable_flag(), "(Bootable)" if self.is_bootable() else '' ) processed_entry += "Partition Type: {0:#x} ({1})\n".format( diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 9173633d1..122c020c3 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -34,10 +34,10 @@ "endian": "little" }, "int": { - "endian": "little", "kind": "int", + "size": 4, "signed": true, - "size": 4 + "endian": "little" }, "unsigned short": { "kind": "int", @@ -51,12 +51,6 @@ "signed": false, "endian": "little" }, - "char": { - "endian": "little", - "kind": "char", - "signed": true, - "size": 1 - }, "wchar": { "kind": "int", "size": 2, @@ -66,14 +60,6 @@ }, "symbols": {}, "enums": { - "BootableFlag":{ - "base": "unsigned char", - "constants": { - "Bootable": 0, - "Non-Bootable": 128 - }, - "size": 1 - }, "PartitionTypes": { "base": "unsigned char", "constants": { @@ -140,8 +126,8 @@ "BootableFlag": { "offset": 0, "type": { - "kind": "enum", - "name": "BootableFlag" + "kind": "base", + "name": "unsigned char" } }, "StartingCHS": { From 1b71aad3669ea4325aecf564ffb1e1c5999de139 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 10 Mar 2022 01:12:49 +0900 Subject: [PATCH 123/404] Update BootableFlag Symbol --- .../symbols/windows/extensions/mbr.py | 7 ++++-- .../framework/symbols/windows/mbr.json | 22 ++++--------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 8e02db822..7cb4c1463 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -2,6 +2,8 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import struct + from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): @@ -18,6 +20,7 @@ class PARTITION_TABLE(objects.StructType): class PARTITION_ENTRY(objects.StructType): def set_index(self, index:int): + """Set Partition Entry Index.""" self.index = index def get_bootable_flag(self) -> int: @@ -26,7 +29,7 @@ class PARTITION_ENTRY(objects.StructType): def is_bootable(self) -> bool: """Check Bootable Partition.""" - return False if not (self.BootableFlag == 0x80) else True + return False if not (self.get_bootable_flag() == 0x80) else True def get_partition_type(self) -> str: """Get Partition Type.""" @@ -68,7 +71,7 @@ class PARTITION_ENTRY(objects.StructType): """Get overall of Partition Entry Info""" processed_entry = "\n===== Partition Table #{0} =====\n".format(self.index+1) processed_entry += "Boot Flag: {0:#x} {1}\n".format( - self.is_bootable(), + self.get_bootable_flag(), "(Bootable)" if self.is_bootable() else '' ) processed_entry += "Partition Type: {0:#x} ({1})\n".format( diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 9173633d1..122c020c3 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -34,10 +34,10 @@ "endian": "little" }, "int": { - "endian": "little", "kind": "int", + "size": 4, "signed": true, - "size": 4 + "endian": "little" }, "unsigned short": { "kind": "int", @@ -51,12 +51,6 @@ "signed": false, "endian": "little" }, - "char": { - "endian": "little", - "kind": "char", - "signed": true, - "size": 1 - }, "wchar": { "kind": "int", "size": 2, @@ -66,14 +60,6 @@ }, "symbols": {}, "enums": { - "BootableFlag":{ - "base": "unsigned char", - "constants": { - "Bootable": 0, - "Non-Bootable": 128 - }, - "size": 1 - }, "PartitionTypes": { "base": "unsigned char", "constants": { @@ -140,8 +126,8 @@ "BootableFlag": { "offset": 0, "type": { - "kind": "enum", - "name": "BootableFlag" + "kind": "base", + "name": "unsigned char" } }, "StartingCHS": { From eba7ad1c0ddd875c4027a2b2e7c189ff74377820 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Mar 2022 21:13:25 +0000 Subject: [PATCH 124/404] Renderers: Use built-in python CSV support --- volatility3/cli/text_renderer.py | 44 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 8663bb995..eadf10d37 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -1,6 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import csv import datetime import json import logging @@ -8,7 +9,7 @@ import random import string import sys from functools import wraps -from typing import Callable, Any, List, Tuple, Dict +from typing import Any, Callable, Dict, List, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.renderers import format_hints @@ -66,7 +67,6 @@ def multitypedata_as_text(value: format_hints.MultiTypeData) -> str: def optional(func: Callable) -> Callable: - @wraps(func) def wrapped(x: Any) -> str: if isinstance(x, interfaces.renderers.BaseAbsentValue): @@ -80,7 +80,6 @@ def optional(func: Callable) -> Callable: def quoted_optional(func: Callable) -> Callable: - @wraps(func) def wrapped(x: Any) -> str: result = optional(func)(x) @@ -193,16 +192,17 @@ class NoneRenderer(CLIRenderer): if not grid.populated: grid.populate(lambda x, y: True, True) + class CSVRenderer(CLIRenderer): _type_renderers = { - format_hints.Bin: quoted_optional(lambda x: f"0b{x:b}"), - format_hints.Hex: quoted_optional(lambda x: f"0x{x:x}"), - format_hints.HexBytes: quoted_optional(hex_bytes_as_text), - format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), - interfaces.renderers.Disassembly: quoted_optional(display_disassembly), - bytes: quoted_optional(lambda x: " ".join([f"{b:02x}" for b in x])), - datetime.datetime: quoted_optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), - 'default': quoted_optional(lambda x: f"{x}") + format_hints.Bin: optional(lambda x: f"0b{x:b}"), + format_hints.Hex: optional(lambda x: f"0x{x:x}"), + format_hints.HexBytes: optional(hex_bytes_as_text), + format_hints.MultiTypeData: optional(multitypedata_as_text), + interfaces.renderers.Disassembly: optional(display_disassembly), + bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), + 'default': optional(lambda x: f"{x}") } name = "csv" @@ -219,28 +219,27 @@ class CSVRenderer(CLIRenderer): """ outfd = sys.stdout - line = ['"TreeDepth"'] + header_list = ['TreeDepth'] for column in grid.columns: # Ignore the type because namedtuples don't realize they have accessible attributes - line.append("{}".format('"' + column.name + '"')) - outfd.write(f"{','.join(line)}") + header_list.append(f"{column.name}") + + writer = csv.DictWriter(outfd, header_list) def visitor(node: interfaces.renderers.TreeNode, accumulator): - accumulator.write("\n") # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case - accumulator.write(str(max(0, node.path_depth - 1)) + ",") - line = [] + row = {'TreeDepth': str(max(0, node.path_depth - 1))} for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get(column.type, self._type_renderers['default']) - line.append(renderer(node.values[column_index])) - accumulator.write(f"{','.join(line)}") + row[f'{column.name}'] = renderer(node.values[column_index]) + accumulator.writerow(row) return accumulator if not grid.populated: - grid.populate(visitor, outfd) + grid.populate(visitor, writer) else: - grid.visit(node = None, function = visitor, initial_accumulator = outfd) + grid.visit(node = None, function = visitor, initial_accumulator = writer) outfd.write("\n") @@ -274,7 +273,8 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths = dict([(column.name, len(column.name)) for column in grid.columns]) def visitor( - node: interfaces.renderers.TreeNode, accumulator: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]] + node: interfaces.renderers.TreeNode, + accumulator: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]] ) -> List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]]: # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case max_column_widths[tree_indent_column] = max(max_column_widths.get(tree_indent_column, 0), node.path_depth) From 0de8c645a4d2159f36c960a6e2c27981798d8599 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Mar 2022 21:20:04 +0000 Subject: [PATCH 125/404] Renderers: Add column headers for CSV --- volatility3/cli/text_renderer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index eadf10d37..8e07d58d1 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -225,6 +225,7 @@ class CSVRenderer(CLIRenderer): header_list.append(f"{column.name}") writer = csv.DictWriter(outfd, header_list) + writer.writeheader() def visitor(node: interfaces.renderers.TreeNode, accumulator): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case From f97348616f559f0849264fbba2b8e9bb8cdae5b8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 14 Mar 2022 09:47:34 +0900 Subject: [PATCH 126/404] Define 'all_zero' default value, Update output column name --- volatility3/framework/plugins/windows/mbrscan.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index f0d52f418..8f0d7a3a4 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -66,6 +66,8 @@ class MBRScan(interfaces.plugins.PluginInterface): full_mbr = layer.read(mbr_start_offset, mbr_length, pad = True) bootcode = full_mbr[:bootcode_length] + all_zeros = None + if bootcode: all_zeros = bootcode.count(b"\x00") == len(bootcode) @@ -101,8 +103,8 @@ class MBRScan(interfaces.plugins.PluginInterface): return renderers.TreeGrid([ ("Potential MBR at Physical Offset", format_hints.Hex), ("Disk Signature", str), - ("Bootcode md5", str), - ("Bootcode (FULL) md5", str), + ("Bootcode MD5", str), + ("Full MBR MD5", str), ("Partition Entries Info", str), ("Disasm", interfaces.renderers.Disassembly), ("Hexdump", format_hints.HexBytes) From 7c89fc3f070814ba27bd61fdd1141cb1eb0b883c Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Tue, 15 Mar 2022 14:22:36 +0200 Subject: [PATCH 127/404] bug fix :( --- volatility3/framework/plugins/windows/malfind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index bfd29a254..700ced8ee 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -53,7 +53,7 @@ class Malfind(interfaces.plugins.PluginInterface): """ CHUNK_SIZE = 0x1000 - all_zero_page = "\x00" * CHUNK_SIZE + all_zero_page = b"\x00" * CHUNK_SIZE offset = 0 vad_length = vad.get_end() - vad.get_start() From fa723ec134e881cc7c3a4987bb1b6eb176a45ac8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Mar 2022 01:29:20 +0000 Subject: [PATCH 128/404] CLI: Implement specifying a config name to write --- volatility3/cli/__init__.py | 25 +++++++++++++++++++++++-- volatility3/cli/volshell/__init__.py | 15 +++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 4cdbd26e8..8cf9621a3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,6 +19,7 @@ import os import sys import tempfile import traceback +from datetime import datetime from typing import Any, Dict, Type, Union from urllib import parse, request @@ -157,6 +158,10 @@ class CommandLine: help = "Write configuration JSON file out to config.json", default = False, action = 'store_true') + parser.add_argument("--save-config", + help = "Save configuration JSON file to a file", + default = None, + type = str) parser.add_argument("--clear-cache", help = "Clears out all short-term cached items", default = False, @@ -320,8 +325,15 @@ class CommandLine: self.file_handler_class_factory()) if args.write_config: - vollog.debug("Writing out configuration data to config.json") - with open("config.json", "w") as f: + args.save_config = 'config.json' + if args.save_config: + vollog.debug("Writing out configuration data to {args.save_config}") + if os.path.exists(os.path.abspath(args.save_config)): + # Backup existing file + backup_filename = self.find_backup_filename(args.save_config) + vollog.debug(f"Backing up existing file to {backup_filename}") + os.rename(args.save_config, backup_filename) + with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) @@ -334,6 +346,15 @@ class CommandLine: except (exceptions.VolatilityException) as excp: self.process_exceptions(excp) + def find_backup_filename(self, original: str): + suffix = "" + new_name = f"{original}.{datetime.strftime(datetime.today(), '%y%m%d')}.bak" + while os.path.exists(f"{new_name}{suffix}"): + if not suffix: + suffix = 1 + suffix += 1 + return f"{new_name}{suffix}" + @classmethod def location_from_file(cls, filename: str) -> str: """Returns the URL location from a file parameter (which may be a URL) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 812d44337..42e82e5bf 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -85,6 +85,10 @@ class VolShell(cli.CommandLine): help = "Write configuration JSON file out to config.json", default = False, action = 'store_true') + parser.add_argument("--save-config", + help = "Save configuration JSON file to a file", + default = None, + type = str) parser.add_argument("--clear-cache", help = "Clears out all short-term cached items", default = False, @@ -234,8 +238,15 @@ class VolShell(cli.CommandLine): self.file_handler_class_factory()) if args.write_config: - vollog.debug("Writing out configuration data to config.json") - with open("config.json", "w") as f: + args.save_config = 'config.json' + if args.save_config: + vollog.debug("Writing out configuration data to {args.save_config}") + if os.path.exists(os.path.abspath(args.save_config)): + # Backup existing file + backup_filename = self.find_backup_filename(args.save_config) + vollog.debug(f"Backing up existing file to {backup_filename}") + os.rename(args.save_config, backup_filename) + with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) From eb38756dbebbd3a6cae366ab5cc5b045faa00b10 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Mar 2022 01:35:47 +0000 Subject: [PATCH 129/404] CLI: Add deprecation warning to --write-config --- volatility3/cli/__init__.py | 5 ++++- volatility3/cli/volshell/__init__.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 8cf9621a3..d22fa154a 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -324,11 +324,14 @@ class CommandLine: constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) + backup_filename = True if args.write_config: + vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' + backup_filename = False if args.save_config: vollog.debug("Writing out configuration data to {args.save_config}") - if os.path.exists(os.path.abspath(args.save_config)): + if os.path.exists(os.path.abspath(args.save_config)) and backup_filename: # Backup existing file backup_filename = self.find_backup_filename(args.save_config) vollog.debug(f"Backing up existing file to {backup_filename}") diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 42e82e5bf..fbf79b117 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -237,11 +237,14 @@ class VolShell(cli.CommandLine): constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) + backup_filename = True if args.write_config: + vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' + backup_filename = False if args.save_config: vollog.debug("Writing out configuration data to {args.save_config}") - if os.path.exists(os.path.abspath(args.save_config)): + if os.path.exists(os.path.abspath(args.save_config)) and backup_filename: # Backup existing file backup_filename = self.find_backup_filename(args.save_config) vollog.debug(f"Backing up existing file to {backup_filename}") From 0f4f4f2b3ac652c0a2ed1a3eaf0ef464f8f939fa Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Mar 2022 01:41:18 +0000 Subject: [PATCH 130/404] CLI: Add configuration option for blatting over config files --- volatility3/cli/__init__.py | 2 +- volatility3/cli/volshell/__init__.py | 2 +- volatility3/framework/constants/__init__.py | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index d22fa154a..1933607c3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -324,7 +324,7 @@ class CommandLine: constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) - backup_filename = True + backup_filename = constants.BACKUP_EXISTING_CONFIG_OUTPUT if args.write_config: vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index fbf79b117..f3bed1b73 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -237,7 +237,7 @@ class VolShell(cli.CommandLine): constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) - backup_filename = True + backup_filename = constants.BACKUP_EXISTING_CONFIG_OUTPUT if args.write_config: vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 665e62d30..4af4408af 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,7 +9,7 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys -from typing import Optional, Callable +from typing import Callable, Optional import volatility3.framework.constants.linux import volatility3.framework.constants.windows @@ -80,6 +80,7 @@ ProgressCallback = Optional[Callable[[float, str], None]] OS_CATEGORIES = ['windows', 'mac', 'linux'] + class Parallelism(enum.IntEnum): """An enumeration listing the different types of parallelism applied to volatility.""" @@ -100,3 +101,6 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" + +BACKUP_EXISTING_CONFIG_OUTPUT = True +"""Whether existing files are backed up or overwritten when writing configuration output""" From 3ab3fa3ed8bbcd6f5b5e4a8d9f5361c587086249 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 16 Mar 2022 17:39:06 +0900 Subject: [PATCH 131/404] Remove Windows Symbol Initialize unuse import --- volatility3/framework/symbols/windows/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index 3aa607574..f09dadedf 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import volatility3.framework.symbols.windows.extensions.pool + from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import extensions from volatility3.framework.symbols.windows.extensions import registry, pool From 1645443d3bad7e672dec09d22ddc95f5f4d7e272 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 16 Mar 2022 22:30:57 +0900 Subject: [PATCH 132/404] Remove Hexdump Column --- volatility3/framework/plugins/windows/mbrscan.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 8f0d7a3a4..3db107567 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -90,8 +90,7 @@ class MBRScan(interfaces.plugins.PluginInterface): self.get_hash(bootcode), self.get_hash(full_mbr), partition_info, - interfaces.renderers.Disassembly(bootcode, 0, architecture), - format_hints.HexBytes(bootcode) + interfaces.renderers.Disassembly(bootcode, 0, architecture) ) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") @@ -106,6 +105,5 @@ class MBRScan(interfaces.plugins.PluginInterface): ("Bootcode MD5", str), ("Full MBR MD5", str), ("Partition Entries Info", str), - ("Disasm", interfaces.renderers.Disassembly), - ("Hexdump", format_hints.HexBytes) + ("Disasm", interfaces.renderers.Disassembly) ], self._generator()) From b02783baf11861847681fc8a5362c173a7772baf Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 16 Mar 2022 22:37:44 +0900 Subject: [PATCH 133/404] Remove index initialize, __str__ method by partition entry logic update --- .../symbols/windows/extensions/mbr.py | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 7cb4c1463..3fdb67ee3 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -18,10 +18,6 @@ class PARTITION_TABLE(objects.StructType): ) class PARTITION_ENTRY(objects.StructType): - - def set_index(self, index:int): - """Set Partition Entry Index.""" - self.index = index def get_bootable_flag(self) -> int: """Get Bootable Flag.""" @@ -66,28 +62,3 @@ class PARTITION_ENTRY(objects.StructType): def get_size_in_sectors(self): """Get Size in Sectors.""" return self.SizeInSectors - - def __str__(self): - """Get overall of Partition Entry Info""" - processed_entry = "\n===== Partition Table #{0} =====\n".format(self.index+1) - processed_entry += "Boot Flag: {0:#x} {1}\n".format( - self.get_bootable_flag(), - "(Bootable)" if self.is_bootable() else '' - ) - processed_entry += "Partition Type: {0:#x} ({1})\n".format( - self.PartitionType, - self.get_partition_type() - ) - processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.get_starting_lba()) - processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format( - self.get_starting_cylinder(), - self.get_starting_chs(), - self.get_starting_sector() - ) - processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format( - self.get_ending_cylinder(), - self.get_ending_chs(), - self.get_ending_sector() - ) - processed_entry += "Size in Sectors: {0:#x} ({0})\n".format(self.get_size_in_sectors()) - return processed_entry From 02394f89a8120d9cfc09bdf1e123d3a3cd3984a5 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 17 Mar 2022 02:22:51 +0900 Subject: [PATCH 134/404] Add return type hint, Add full option, Update yield data --- .../framework/plugins/windows/mbrscan.py | 184 +++++++++++++++--- 1 file changed, 157 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 3db107567..60b403ae0 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -5,6 +5,8 @@ import logging import hashlib +from typing import Iterator, List, Tuple + from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners @@ -21,17 +23,21 @@ class MBRScan(interfaces.plugins.PluginInterface): _version = (1, 0, 0) @classmethod - def get_requirements(cls): + def get_requirements(cls)-> List[interfaces.configuration.RequirementInterface]: return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]) + architectures = ["Intel32", "Intel64"]), + requirements.BooleanRequirement(name = 'full', + description ="It analyzes and provides all the information in the partition entry. (It returns a lot of information, so we recommend you render it in CSV.)", + default = False, + optional = True) ] @classmethod def get_hash(cls, data:bytes) -> str: return hashlib.md5(data).hexdigest() - def _generator(self): + def _generator(self) -> Iterator[Tuple]: kernel = self.context.modules[self.config['kernel']] physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) @@ -72,38 +78,162 @@ class MBRScan(interfaces.plugins.PluginInterface): all_zeros = bootcode.count(b"\x00") == len(bootcode) if not all_zeros: - partition_entries = [ - partition_table.FirstEntry, - partition_table.SecondEntry, - partition_table.ThirdEntry, - partition_table.FourthEntry - ] - partition_info = "\n" - - for index, partition_entry_object in enumerate(partition_entries): - partition_entry_object.set_index(index) - partition_info += str(partition_entry_object) - - yield 0, ( + if not self.config.get("full", True): + yield (0, ( format_hints.Hex(offset), partition_table.get_disk_signature(), self.get_hash(bootcode), self.get_hash(full_mbr), - partition_info, + partition_table.FirstEntry.is_bootable(), + partition_table.FirstEntry.get_partition_type(), + format_hints.Hex(partition_table.FirstEntry.get_size_in_sectors()), + partition_table.SecondEntry.is_bootable(), + partition_table.SecondEntry.get_partition_type(), + format_hints.Hex(partition_table.SecondEntry.get_size_in_sectors()), + partition_table.ThirdEntry.is_bootable(), + partition_table.ThirdEntry.get_partition_type(), + format_hints.Hex(partition_table.ThirdEntry.get_size_in_sectors()), + partition_table.FourthEntry.is_bootable(), + partition_table.FourthEntry.get_partition_type(), + format_hints.Hex(partition_table.FourthEntry.get_size_in_sectors()), interfaces.renderers.Disassembly(bootcode, 0, architecture) - ) + )) + else: + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_table.FirstEntry.is_bootable(), + format_hints.Hex(partition_table.FirstEntry.get_bootable_flag()), + partition_table.FirstEntry.get_partition_type(), + format_hints.Hex(partition_table.FirstEntry.PartitionType), + format_hints.Hex(partition_table.FirstEntry.get_starting_lba()), + partition_table.FirstEntry.get_starting_cylinder(), + partition_table.FirstEntry.get_starting_chs(), + partition_table.FirstEntry.get_starting_sector(), + partition_table.FirstEntry.get_ending_cylinder(), + partition_table.FirstEntry.get_ending_chs(), + partition_table.FirstEntry.get_ending_sector(), + format_hints.Hex(partition_table.FirstEntry.get_size_in_sectors()), + partition_table.SecondEntry.is_bootable(), + format_hints.Hex(partition_table.SecondEntry.get_bootable_flag()), + partition_table.SecondEntry.get_partition_type(), + format_hints.Hex(partition_table.SecondEntry.PartitionType), + format_hints.Hex(partition_table.SecondEntry.get_starting_lba()), + partition_table.SecondEntry.get_starting_cylinder(), + partition_table.SecondEntry.get_starting_chs(), + partition_table.SecondEntry.get_starting_sector(), + partition_table.SecondEntry.get_ending_cylinder(), + partition_table.SecondEntry.get_ending_chs(), + partition_table.SecondEntry.get_ending_sector(), + format_hints.Hex(partition_table.SecondEntry.get_size_in_sectors()), + partition_table.ThirdEntry.is_bootable(), + format_hints.Hex(partition_table.ThirdEntry.get_bootable_flag()), + partition_table.ThirdEntry.get_partition_type(), + format_hints.Hex(partition_table.ThirdEntry.PartitionType), + format_hints.Hex(partition_table.ThirdEntry.get_starting_lba()), + partition_table.ThirdEntry.get_starting_cylinder(), + partition_table.ThirdEntry.get_starting_chs(), + partition_table.ThirdEntry.get_starting_sector(), + partition_table.ThirdEntry.get_ending_cylinder(), + partition_table.ThirdEntry.get_ending_chs(), + partition_table.ThirdEntry.get_ending_sector(), + format_hints.Hex(partition_table.ThirdEntry.get_size_in_sectors()), + partition_table.FourthEntry.is_bootable(), + format_hints.Hex(partition_table.FourthEntry.get_bootable_flag()), + partition_table.FourthEntry.get_partition_type(), + format_hints.Hex(partition_table.FourthEntry.PartitionType), + format_hints.Hex(partition_table.FourthEntry.get_starting_lba()), + partition_table.FourthEntry.get_starting_cylinder(), + partition_table.FourthEntry.get_starting_chs(), + partition_table.FourthEntry.get_starting_sector(), + partition_table.FourthEntry.get_ending_cylinder(), + partition_table.FourthEntry.get_ending_chs(), + partition_table.FourthEntry.get_ending_sector(), + format_hints.Hex(partition_table.FourthEntry.get_size_in_sectors()), + interfaces.renderers.Disassembly(bootcode, 0, architecture) + )) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") except exceptions.PagedInvalidAddressException: pass - def run(self): - return renderers.TreeGrid([ - ("Potential MBR at Physical Offset", format_hints.Hex), - ("Disk Signature", str), - ("Bootcode MD5", str), - ("Full MBR MD5", str), - ("Partition Entries Info", str), - ("Disasm", interfaces.renderers.Disassembly) - ], self._generator()) + def run(self)-> renderers.TreeGrid: + if not self.config.get("full", True): + return renderers.TreeGrid([ + ("Potential MBR at Physical Offset", format_hints.Hex), + ("Disk Signature", str), + ("Bootcode MD5", str), + ("Full MBR MD5", str), + ("PartABootable", bool), + ("PartAType", str), + ("PartASectorInSize", format_hints.Hex), + ("PartBBootable", bool), + ("PartBType", str), + ("PartBSectorInSize", format_hints.Hex), + ("PartCBootable", bool), + ("PartCType", str), + ("PartCSectorInSize", format_hints.Hex), + ("PartDBootable", bool), + ("PartDType", str), + ("PartDSectorInSize", format_hints.Hex), + ("Disasm", interfaces.renderers.Disassembly) + ], self._generator()) + else: + return renderers.TreeGrid([ + ("Potential MBR at Physical Offset", format_hints.Hex), + ("Disk Signature", str), + ("Bootcode MD5", str), + ("Full MBR MD5", str), + ("PartABootable", bool), + ("PartABootFlag", format_hints.Hex), + ("PartAType", str), + ("PartATypeRaw", format_hints.Hex), + ("PartAStartingLBA", format_hints.Hex), + ("PartAStartingCylinder", int), + ("PartAStartingCHS", int), + ("PartAStartingSector", int), + ("PartAEndingCylinder", int), + ("PartAEndingCHS", int), + ("PartAEndingSector", int), + ("PartASectorInSize", format_hints.Hex), + ("PartBBootable", bool), + ("PartBBootFlag", format_hints.Hex), + ("PartBType", str), + ("PartBTypeRaw", format_hints.Hex), + ("PartBStartingLBA", format_hints.Hex), + ("PartBStartingCylinder", int), + ("PartBStartingCHS", int), + ("PartBStartingSector", int), + ("PartBEndingCylinder", int), + ("PartBEndingCHS", int), + ("PartBEndingSector", int), + ("PartBSectorInSize", format_hints.Hex), + ("PartCBootable", bool), + ("PartCBootFlag", format_hints.Hex), + ("PartCType", str), + ("PartCTypeRaw", format_hints.Hex), + ("PartCStartingLBA", format_hints.Hex), + ("PartCStartingCylinder", int), + ("PartCStartingCHS", int), + ("PartCStartingSector", int), + ("PartCEndingCylinder", int), + ("PartCEndingCHS", int), + ("PartCEndingSector", int), + ("PartCSectorInSize", format_hints.Hex), + ("PartDBootable", bool), + ("PartDBootFlag", format_hints.Hex), + ("PartDType", str), + ("PartDTypeRaw", format_hints.Hex), + ("PartDStartingLBA", format_hints.Hex), + ("PartDStartingCylinder", int), + ("PartDStartingCHS", int), + ("PartDStartingSector", int), + ("PartDEndingCylinder", int), + ("PartDEndingCHS", int), + ("PartDEndingSector", int), + ("PartDSectorInSize", format_hints.Hex), + ("Disasm", interfaces.renderers.Disassembly) + ], self._generator()) From a6217784cea49698975730ed6782a1c0023ecd46 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Mar 2022 20:12:22 +0000 Subject: [PATCH 135/404] Windows: Tidy up hashdump plugin Shouldn't have merged this with mention of profiles. Also fixes #678. --- .../framework/plugins/windows/hashdump.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index 4d0b25b5c..e9f8047e0 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -4,10 +4,10 @@ import binascii import hashlib import logging -from struct import unpack, pack -from typing import List, Tuple, Optional +from struct import pack, unpack +from typing import List, Optional, Tuple -from Crypto.Cipher import ARC4, DES, AES +from Crypto.Cipher import AES, ARC4, DES from Crypto.Hash import MD5 from volatility3.framework import interfaces, renderers @@ -28,7 +28,7 @@ class Hashdump(interfaces.plugins.PluginInterface): def get_requirements(cls): return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) ] @@ -63,7 +63,8 @@ class Hashdump(interfaces.plugins.PluginInterface): def get_hive_key(cls, hive: registry.RegistryHive, key: str): result = None try: - result = hive.get_key(key) + if hive: + result = hive.get_key(key) except KeyError: vollog.info( f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image") @@ -132,7 +133,7 @@ class Hashdump(interfaces.plugins.PluginInterface): rc4_key = md5.digest() rc4 = ARC4.new(rc4_key) - hbootkey = rc4.encrypt(sam_data[0x80:0xA0]) # lgtm [py/weak-cryptographic-algorithm] + hbootkey = rc4.encrypt(sam_data[0x80:0xA0]) # lgtm [py/weak-cryptographic-algorithm] return hbootkey elif revision == 3: # AES encrypted @@ -151,7 +152,7 @@ class Hashdump(interfaces.plugins.PluginInterface): des2 = DES.new(des_k2, DES.MODE_ECB) cipher = AES.new(hbootkey[:16], AES.MODE_CBC, salt) obfkey = cipher.decrypt(enc_hash) - return des1.decrypt(obfkey[:8]) + des2.decrypt(obfkey[8:16]) # lgtm [py/weak-cryptographic-algorithm] + return des1.decrypt(obfkey[:8]) + des2.decrypt(obfkey[8:16]) # lgtm [py/weak-cryptographic-algorithm] @classmethod def get_user_hashes(cls, user: registry.CM_KEY_NODE, samhive: registry.RegistryHive, @@ -229,9 +230,9 @@ class Hashdump(interfaces.plugins.PluginInterface): md5.update(hbootkey[:0x10] + pack(" Optional[bytes]: @@ -253,13 +254,9 @@ class Hashdump(interfaces.plugins.PluginInterface): # replaces the dump_hashes method in vol2 def _generator(self, syshive: registry.RegistryHive, samhive: registry.RegistryHive): if syshive is None: - vollog.debug("SYSTEM address is None: Did you use the correct profile?") - yield (0, (renderers.NotAvailableValue(), renderers.NotAvailableValue(), renderers.NotAvailableValue(), - renderers.NotAvailableValue())) + vollog.debug("SYSTEM address is None: No system hive found") if samhive is None: - vollog.debug("SAM address is None: Did you use the correct profile?") - yield (0, (renderers.NotAvailableValue(), renderers.NotAvailableValue(), renderers.NotAvailableValue(), - renderers.NotAvailableValue())) + vollog.debug("SAM address is None: No SAM hive found") bootkey = self.get_bootkey(syshive) hbootkey = self.get_hbootkey(samhive, bootkey) if hbootkey: From cb8a1fb90c7e1571b82bc6bc58cd45f5ffcfff0e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Mar 2022 20:36:26 +0000 Subject: [PATCH 136/404] CLI: Fail on overwriting a config file --- volatility3/cli/__init__.py | 9 ++------- volatility3/cli/volshell/__init__.py | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 1933607c3..8d198e57c 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -324,18 +324,13 @@ class CommandLine: constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) - backup_filename = constants.BACKUP_EXISTING_CONFIG_OUTPUT if args.write_config: vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' - backup_filename = False if args.save_config: vollog.debug("Writing out configuration data to {args.save_config}") - if os.path.exists(os.path.abspath(args.save_config)) and backup_filename: - # Backup existing file - backup_filename = self.find_backup_filename(args.save_config) - vollog.debug(f"Backing up existing file to {backup_filename}") - os.rename(args.save_config, backup_filename) + if os.path.exists(os.path.abspath(args.save_config)): + parser.error(f"Cannot write configuration: file {args.save_config} already exists") with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) except exceptions.UnsatisfiedException as excp: diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index f3bed1b73..30fe75e06 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -237,18 +237,13 @@ class VolShell(cli.CommandLine): constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) - backup_filename = constants.BACKUP_EXISTING_CONFIG_OUTPUT if args.write_config: vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' - backup_filename = False if args.save_config: vollog.debug("Writing out configuration data to {args.save_config}") - if os.path.exists(os.path.abspath(args.save_config)) and backup_filename: - # Backup existing file - backup_filename = self.find_backup_filename(args.save_config) - vollog.debug(f"Backing up existing file to {backup_filename}") - os.rename(args.save_config, backup_filename) + if os.path.exists(os.path.abspath(args.save_config)): + parser.error(f"Cannot write configuration: file {args.save_config} already exists") with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) except exceptions.UnsatisfiedException as excp: From ed1a19d1ac2bb6267ce1627f60e80b34c59f1047 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 18 Mar 2022 01:07:59 +0900 Subject: [PATCH 137/404] Add hex dump column if full data option --- volatility3/framework/plugins/windows/mbrscan.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 60b403ae0..3602e5e78 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -28,7 +28,7 @@ class MBRScan(interfaces.plugins.PluginInterface): requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), requirements.BooleanRequirement(name = 'full', - description ="It analyzes and provides all the information in the partition entry. (It returns a lot of information, so we recommend you render it in CSV.)", + description ="It analyzes and provides all the information in the partition entry and bootcode hexdump. (It returns a lot of information, so we recommend you render it in CSV.)", default = False, optional = True) ] @@ -152,7 +152,8 @@ class MBRScan(interfaces.plugins.PluginInterface): partition_table.FourthEntry.get_ending_chs(), partition_table.FourthEntry.get_ending_sector(), format_hints.Hex(partition_table.FourthEntry.get_size_in_sectors()), - interfaces.renderers.Disassembly(bootcode, 0, architecture) + interfaces.renderers.Disassembly(bootcode, 0, architecture), + format_hints.HexBytes(bootcode) )) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") @@ -235,5 +236,6 @@ class MBRScan(interfaces.plugins.PluginInterface): ("PartDEndingCHS", int), ("PartDEndingSector", int), ("PartDSectorInSize", format_hints.Hex), - ("Disasm", interfaces.renderers.Disassembly) + ("Disasm", interfaces.renderers.Disassembly), + ("Bootcode", format_hints.HexBytes) ], self._generator()) From 174036cc727b98a53e7d83dee9cfc82dcd370382 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 19 Mar 2022 15:26:10 +0900 Subject: [PATCH 138/404] Fix Typo Error for Disassembly rendering code comment --- volatility3/cli/text_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 8e07d58d1..1ddfcca84 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -101,7 +101,7 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str: disasm: Input disassembly objects Returns: - A string as rendererd by capstone where available, otherwise output as if it were just bytes + A string as rendered by capstone where available, otherwise output as if it were just bytes """ if CAPSTONE_PRESENT: From 1aad1c8b1a933f46ae2753bb209dd986e49ce99e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 21 Mar 2022 00:32:24 +0900 Subject: [PATCH 139/404] Initialize devicetree plugin --- volatility3/framework/plugins/windows/devicetree.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 volatility3/framework/plugins/windows/devicetree.py diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py new file mode 100644 index 000000000..e69de29bb From 5af889b0593947854f161cbccd965f5b5036994b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 21 Mar 2022 15:11:51 +0900 Subject: [PATCH 140/404] Fix operating system comparision syntax for create cache path constant. --- volatility3/framework/constants/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 665e62d30..629d5db80 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -11,7 +11,6 @@ import os.path import sys from typing import Optional, Callable -import volatility3.framework.constants.linux import volatility3.framework.constants.windows PLUGINS_PATH = [ @@ -63,7 +62,7 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" -if sys.platform == 'windows': +if sys.platform == 'win32': CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") os.makedirs(CACHE_PATH, exist_ok = True) From 37f6750c92668407e07ec7e8d641a4195490a95f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 23 Mar 2022 07:55:53 +0900 Subject: [PATCH 141/404] ReImport volatility.framework.constants.linux --- volatility3/framework/constants/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 629d5db80..063862be8 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -11,6 +11,7 @@ import os.path import sys from typing import Optional, Callable +import volatility3.framework.constants.linux import volatility3.framework.constants.windows PLUGINS_PATH = [ From cf4ef0fa38eb7e68e51986a35ae91da4f9a04d5a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 23 Mar 2022 15:12:54 +0900 Subject: [PATCH 142/404] Set __init__ and fix description of mac environment plugins --- volatility3/framework/plugins/mac/__init__.py | 8 ++++++++ volatility3/framework/plugins/mac/ifconfig.py | 2 +- volatility3/framework/plugins/mac/mount.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/__init__.py b/volatility3/framework/plugins/mac/__init__.py index e69de29bb..ef6762bee 100644 --- a/volatility3/framework/plugins/mac/__init__.py +++ b/volatility3/framework/plugins/mac/__init__.py @@ -0,0 +1,8 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +"""All core mac plugins. + +These modules should only be imported from volatility3.plugins NOT +volatility3.framework.plugins +""" diff --git a/volatility3/framework/plugins/mac/ifconfig.py b/volatility3/framework/plugins/mac/ifconfig.py index c366a19f0..99666b763 100644 --- a/volatility3/framework/plugins/mac/ifconfig.py +++ b/volatility3/framework/plugins/mac/ifconfig.py @@ -9,7 +9,7 @@ from volatility3.framework.symbols import mac class Ifconfig(plugins.PluginInterface): - """Lists loaded kernel modules""" + """ Lists network interface information for all devices """ _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index 398559446..6486d00ff 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -12,7 +12,7 @@ from volatility3.framework.symbols import mac class Mount(plugins.PluginInterface): """A module containing a collection of plugins that produce data typically - foundin Mac's mount command""" + founding Mac's mount command""" _required_framework_version = (2, 0, 0) From 95825e99dfe1dafae062e7c84dbd0f6ca96b26d6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 23 Mar 2022 16:09:19 +0900 Subject: [PATCH 143/404] Update Vollog level if all zero mbr data, PagedInvalidAddressException handling --- volatility3/framework/plugins/windows/mbrscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 3602e5e78..1e78c1c25 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -156,10 +156,10 @@ class MBRScan(interfaces.plugins.PluginInterface): format_hints.HexBytes(bootcode) )) else: - vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + vollog.log(constants.LOGLEVEL_VVVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") except exceptions.PagedInvalidAddressException: - pass + continue def run(self)-> renderers.TreeGrid: if not self.config.get("full", True): From 64b8f681f4c778f3ed20350baed69b8ea2e9b2de Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 23 Mar 2022 18:05:42 +0900 Subject: [PATCH 144/404] Update sentence by code review --- volatility3/framework/plugins/mac/ifconfig.py | 2 +- volatility3/framework/plugins/mac/mount.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/ifconfig.py b/volatility3/framework/plugins/mac/ifconfig.py index 99666b763..330c13f07 100644 --- a/volatility3/framework/plugins/mac/ifconfig.py +++ b/volatility3/framework/plugins/mac/ifconfig.py @@ -9,7 +9,7 @@ from volatility3.framework.symbols import mac class Ifconfig(plugins.PluginInterface): - """ Lists network interface information for all devices """ + """Lists network interface information for all devices""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index 6486d00ff..ba3ab83c8 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -12,7 +12,7 @@ from volatility3.framework.symbols import mac class Mount(plugins.PluginInterface): """A module containing a collection of plugins that produce data typically - founding Mac's mount command""" + found in Mac's mount command""" _required_framework_version = (2, 0, 0) From 02d90e9e42974959440fb9a45aa585ad9870d24b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 24 Mar 2022 08:45:29 +0000 Subject: [PATCH 145/404] CLI: Remove unnecessary extra code --- volatility3/cli/__init__.py | 9 --------- volatility3/framework/constants/__init__.py | 3 --- 2 files changed, 12 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 8d198e57c..35ad84011 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -344,15 +344,6 @@ class CommandLine: except (exceptions.VolatilityException) as excp: self.process_exceptions(excp) - def find_backup_filename(self, original: str): - suffix = "" - new_name = f"{original}.{datetime.strftime(datetime.today(), '%y%m%d')}.bak" - while os.path.exists(f"{new_name}{suffix}"): - if not suffix: - suffix = 1 - suffix += 1 - return f"{new_name}{suffix}" - @classmethod def location_from_file(cls, filename: str) -> str: """Returns the URL location from a file parameter (which may be a URL) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 4af4408af..f3d31dd2e 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -101,6 +101,3 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" - -BACKUP_EXISTING_CONFIG_OUTPUT = True -"""Whether existing files are backed up or overwritten when writing configuration output""" From 2cfc24f7d3c4af2b710819c80925963c216fcce9 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 15:50:15 +0900 Subject: [PATCH 146/404] Changes in structure and data return for efficient partition entries data display --- .../framework/plugins/windows/mbrscan.py | 204 ++++++------------ .../symbols/windows/extensions/mbr.py | 2 - 2 files changed, 65 insertions(+), 141 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 1e78c1c25..991d48bf9 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -78,88 +78,57 @@ class MBRScan(interfaces.plugins.PluginInterface): all_zeros = bootcode.count(b"\x00") == len(bootcode) if not all_zeros: - if not self.config.get("full", True): - yield (0, ( - format_hints.Hex(offset), - partition_table.get_disk_signature(), - self.get_hash(bootcode), - self.get_hash(full_mbr), - partition_table.FirstEntry.is_bootable(), - partition_table.FirstEntry.get_partition_type(), - format_hints.Hex(partition_table.FirstEntry.get_size_in_sectors()), - partition_table.SecondEntry.is_bootable(), - partition_table.SecondEntry.get_partition_type(), - format_hints.Hex(partition_table.SecondEntry.get_size_in_sectors()), - partition_table.ThirdEntry.is_bootable(), - partition_table.ThirdEntry.get_partition_type(), - format_hints.Hex(partition_table.ThirdEntry.get_size_in_sectors()), - partition_table.FourthEntry.is_bootable(), - partition_table.FourthEntry.get_partition_type(), - format_hints.Hex(partition_table.FourthEntry.get_size_in_sectors()), - interfaces.renderers.Disassembly(bootcode, 0, architecture) - )) - else: - yield (0, ( - format_hints.Hex(offset), - partition_table.get_disk_signature(), - self.get_hash(bootcode), - self.get_hash(full_mbr), - partition_table.FirstEntry.is_bootable(), - format_hints.Hex(partition_table.FirstEntry.get_bootable_flag()), - partition_table.FirstEntry.get_partition_type(), - format_hints.Hex(partition_table.FirstEntry.PartitionType), - format_hints.Hex(partition_table.FirstEntry.get_starting_lba()), - partition_table.FirstEntry.get_starting_cylinder(), - partition_table.FirstEntry.get_starting_chs(), - partition_table.FirstEntry.get_starting_sector(), - partition_table.FirstEntry.get_ending_cylinder(), - partition_table.FirstEntry.get_ending_chs(), - partition_table.FirstEntry.get_ending_sector(), - format_hints.Hex(partition_table.FirstEntry.get_size_in_sectors()), - partition_table.SecondEntry.is_bootable(), - format_hints.Hex(partition_table.SecondEntry.get_bootable_flag()), - partition_table.SecondEntry.get_partition_type(), - format_hints.Hex(partition_table.SecondEntry.PartitionType), - format_hints.Hex(partition_table.SecondEntry.get_starting_lba()), - partition_table.SecondEntry.get_starting_cylinder(), - partition_table.SecondEntry.get_starting_chs(), - partition_table.SecondEntry.get_starting_sector(), - partition_table.SecondEntry.get_ending_cylinder(), - partition_table.SecondEntry.get_ending_chs(), - partition_table.SecondEntry.get_ending_sector(), - format_hints.Hex(partition_table.SecondEntry.get_size_in_sectors()), - partition_table.ThirdEntry.is_bootable(), - format_hints.Hex(partition_table.ThirdEntry.get_bootable_flag()), - partition_table.ThirdEntry.get_partition_type(), - format_hints.Hex(partition_table.ThirdEntry.PartitionType), - format_hints.Hex(partition_table.ThirdEntry.get_starting_lba()), - partition_table.ThirdEntry.get_starting_cylinder(), - partition_table.ThirdEntry.get_starting_chs(), - partition_table.ThirdEntry.get_starting_sector(), - partition_table.ThirdEntry.get_ending_cylinder(), - partition_table.ThirdEntry.get_ending_chs(), - partition_table.ThirdEntry.get_ending_sector(), - format_hints.Hex(partition_table.ThirdEntry.get_size_in_sectors()), - partition_table.FourthEntry.is_bootable(), - format_hints.Hex(partition_table.FourthEntry.get_bootable_flag()), - partition_table.FourthEntry.get_partition_type(), - format_hints.Hex(partition_table.FourthEntry.PartitionType), - format_hints.Hex(partition_table.FourthEntry.get_starting_lba()), - partition_table.FourthEntry.get_starting_cylinder(), - partition_table.FourthEntry.get_starting_chs(), - partition_table.FourthEntry.get_starting_sector(), - partition_table.FourthEntry.get_ending_cylinder(), - partition_table.FourthEntry.get_ending_chs(), - partition_table.FourthEntry.get_ending_sector(), - format_hints.Hex(partition_table.FourthEntry.get_size_in_sectors()), - interfaces.renderers.Disassembly(bootcode, 0, architecture), - format_hints.HexBytes(bootcode) - )) + + partition_entries = [ + partition_table.FirstEntry, partition_table.SecondEntry, + partition_table.ThirdEntry, partition_table.FourthEntry + ] + + for partition_index, partition_entry_object in enumerate(partition_entries, start=1): + # Output disassembly information and bootcode for each partition entry is inefficient, + # so it can only be processed in the last index. + last_partition_index = 4 + bootcode_buf = bootcode if(partition_index == last_partition_index) else b"" + + if not self.config.get("full", True): + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_index, + partition_entry_object.is_bootable(), + partition_entry_object.get_partition_type(), + format_hints.Hex(partition_entry_object.get_size_in_sectors()), + interfaces.renderers.Disassembly(bootcode_buf, 0, architecture) + )) + else: + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_index, + partition_entry_object.is_bootable(), + format_hints.Hex(partition_entry_object.get_bootable_flag()), + partition_entry_object.get_partition_type(), + format_hints.Hex(partition_entry_object.PartitionType), + format_hints.Hex(partition_entry_object.get_starting_lba()), + partition_entry_object.get_starting_cylinder(), + partition_entry_object.get_starting_chs(), + partition_entry_object.get_starting_sector(), + partition_entry_object.get_ending_cylinder(), + partition_entry_object.get_ending_chs(), + partition_entry_object.get_ending_sector(), + format_hints.Hex(partition_entry_object.get_size_in_sectors()), + interfaces.renderers.Disassembly(bootcode_buf, 0, architecture), + format_hints.HexBytes(bootcode_buf) + )) else: - vollog.log(constants.LOGLEVEL_VVVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") except exceptions.PagedInvalidAddressException: - continue + pass def run(self)-> renderers.TreeGrid: if not self.config.get("full", True): @@ -168,18 +137,10 @@ class MBRScan(interfaces.plugins.PluginInterface): ("Disk Signature", str), ("Bootcode MD5", str), ("Full MBR MD5", str), - ("PartABootable", bool), - ("PartAType", str), - ("PartASectorInSize", format_hints.Hex), - ("PartBBootable", bool), - ("PartBType", str), - ("PartBSectorInSize", format_hints.Hex), - ("PartCBootable", bool), - ("PartCType", str), - ("PartCSectorInSize", format_hints.Hex), - ("PartDBootable", bool), - ("PartDType", str), - ("PartDSectorInSize", format_hints.Hex), + ("PartitionIndex", int), + ("Bootable", bool), + ("PartitionType", str), + ("SectorInSize", format_hints.Hex), ("Disasm", interfaces.renderers.Disassembly) ], self._generator()) else: @@ -188,54 +149,19 @@ class MBRScan(interfaces.plugins.PluginInterface): ("Disk Signature", str), ("Bootcode MD5", str), ("Full MBR MD5", str), - ("PartABootable", bool), - ("PartABootFlag", format_hints.Hex), - ("PartAType", str), - ("PartATypeRaw", format_hints.Hex), - ("PartAStartingLBA", format_hints.Hex), - ("PartAStartingCylinder", int), - ("PartAStartingCHS", int), - ("PartAStartingSector", int), - ("PartAEndingCylinder", int), - ("PartAEndingCHS", int), - ("PartAEndingSector", int), - ("PartASectorInSize", format_hints.Hex), - ("PartBBootable", bool), - ("PartBBootFlag", format_hints.Hex), - ("PartBType", str), - ("PartBTypeRaw", format_hints.Hex), - ("PartBStartingLBA", format_hints.Hex), - ("PartBStartingCylinder", int), - ("PartBStartingCHS", int), - ("PartBStartingSector", int), - ("PartBEndingCylinder", int), - ("PartBEndingCHS", int), - ("PartBEndingSector", int), - ("PartBSectorInSize", format_hints.Hex), - ("PartCBootable", bool), - ("PartCBootFlag", format_hints.Hex), - ("PartCType", str), - ("PartCTypeRaw", format_hints.Hex), - ("PartCStartingLBA", format_hints.Hex), - ("PartCStartingCylinder", int), - ("PartCStartingCHS", int), - ("PartCStartingSector", int), - ("PartCEndingCylinder", int), - ("PartCEndingCHS", int), - ("PartCEndingSector", int), - ("PartCSectorInSize", format_hints.Hex), - ("PartDBootable", bool), - ("PartDBootFlag", format_hints.Hex), - ("PartDType", str), - ("PartDTypeRaw", format_hints.Hex), - ("PartDStartingLBA", format_hints.Hex), - ("PartDStartingCylinder", int), - ("PartDStartingCHS", int), - ("PartDStartingSector", int), - ("PartDEndingCylinder", int), - ("PartDEndingCHS", int), - ("PartDEndingSector", int), - ("PartDSectorInSize", format_hints.Hex), + ("PartitionIndex", int), + ("Bootable", bool), + ("BootFlag", format_hints.Hex), + ("PartitionType", str), + ("PartitionTypeRaw", format_hints.Hex), + ("StartingLBA", format_hints.Hex), + ("StartingCylinder", int), + ("StartingCHS", int), + ("StartingSector", int), + ("EndingCylinder", int), + ("EndingCHS", int), + ("EndingSector", int), + ("SectorInSize", format_hints.Hex), ("Disasm", interfaces.renderers.Disassembly), ("Bootcode", format_hints.HexBytes) ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 3fdb67ee3..8100371fd 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -2,8 +2,6 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import struct - from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): From b07859db7f55bae7d9c9cb15aea978c06968beda Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 17:02:41 +0900 Subject: [PATCH 147/404] Add vollog for PagedInvalidAddressException --- volatility3/framework/plugins/windows/mbrscan.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 991d48bf9..cdc40a7be 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -126,10 +126,12 @@ class MBRScan(interfaces.plugins.PluginInterface): )) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + continue - except exceptions.PagedInvalidAddressException: - pass - + except exceptions.PagedInvalidAddressException as excp: + vollog.debug(f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") + continue + def run(self)-> renderers.TreeGrid: if not self.config.get("full", True): return renderers.TreeGrid([ From a49292ab483426d7db1c5be4c0a31db39859da36 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 17:21:46 +0900 Subject: [PATCH 148/404] Fix type for partition --- volatility3/framework/symbols/windows/mbr.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 122c020c3..2a6ec7779 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -170,7 +170,7 @@ "offset": 12, "type": { "kind": "base", - "name": "int" + "name": "unsigned int" } } }, From f8443b994dc4ac1f512e49928d555e6b2abcadd6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 17:29:07 +0900 Subject: [PATCH 149/404] Refactoring for partition index --- volatility3/framework/plugins/windows/mbrscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index cdc40a7be..fc473ae7e 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -87,7 +87,7 @@ class MBRScan(interfaces.plugins.PluginInterface): for partition_index, partition_entry_object in enumerate(partition_entries, start=1): # Output disassembly information and bootcode for each partition entry is inefficient, # so it can only be processed in the last index. - last_partition_index = 4 + last_partition_index = len(partition_entries) bootcode_buf = bootcode if(partition_index == last_partition_index) else b"" if not self.config.get("full", True): From a9dabf90d71543ee335ec1a20252d9f27b68ef93 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 23:22:23 +0900 Subject: [PATCH 150/404] Adjust vollog log level of Exception --- volatility3/framework/plugins/windows/mbrscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index fc473ae7e..2148efc41 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -125,11 +125,11 @@ class MBRScan(interfaces.plugins.PluginInterface): format_hints.HexBytes(bootcode_buf) )) else: - vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + vollog.log(constants.LOGLEVEL_VVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") continue except exceptions.PagedInvalidAddressException as excp: - vollog.debug(f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") + vollog.log(constants.LOGLEVEL_VVV, f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") continue def run(self)-> renderers.TreeGrid: From 8a99c17d4266f7e869404f37a4e60e61bb6cfe90 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 23:27:25 +0900 Subject: [PATCH 151/404] Adjust vollog log level of Exception --- volatility3/framework/plugins/windows/mbrscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 2148efc41..ba74a7b7e 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -125,11 +125,11 @@ class MBRScan(interfaces.plugins.PluginInterface): format_hints.HexBytes(bootcode_buf) )) else: - vollog.log(constants.LOGLEVEL_VVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + vollog.log(constants.LOGLEVEL_VVVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") continue except exceptions.PagedInvalidAddressException as excp: - vollog.log(constants.LOGLEVEL_VVV, f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") + vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") continue def run(self)-> renderers.TreeGrid: From ff433dd4b8fb0bb3d7125614e7c295358f512709 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 26 Mar 2022 01:22:17 +0900 Subject: [PATCH 152/404] Change the empty byte to NotApplicableValue for efficient partition data output. --- .../framework/plugins/windows/mbrscan.py | 49 +++++++++++++++---- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index ba74a7b7e..39e962c96 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -84,14 +84,45 @@ class MBRScan(interfaces.plugins.PluginInterface): partition_table.ThirdEntry, partition_table.FourthEntry ] + if not self.config.get("full", True): + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + interfaces.renderers.Disassembly(bootcode, 0, architecture) + )) + else: + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + interfaces.renderers.Disassembly(bootcode, 0, architecture), + format_hints.HexBytes(bootcode) + )) + for partition_index, partition_entry_object in enumerate(partition_entries, start=1): - # Output disassembly information and bootcode for each partition entry is inefficient, - # so it can only be processed in the last index. - last_partition_index = len(partition_entries) - bootcode_buf = bootcode if(partition_index == last_partition_index) else b"" if not self.config.get("full", True): - yield (0, ( + yield (1, ( format_hints.Hex(offset), partition_table.get_disk_signature(), self.get_hash(bootcode), @@ -100,10 +131,10 @@ class MBRScan(interfaces.plugins.PluginInterface): partition_entry_object.is_bootable(), partition_entry_object.get_partition_type(), format_hints.Hex(partition_entry_object.get_size_in_sectors()), - interfaces.renderers.Disassembly(bootcode_buf, 0, architecture) + renderers.NotApplicableValue() )) else: - yield (0, ( + yield (1, ( format_hints.Hex(offset), partition_table.get_disk_signature(), self.get_hash(bootcode), @@ -121,8 +152,8 @@ class MBRScan(interfaces.plugins.PluginInterface): partition_entry_object.get_ending_chs(), partition_entry_object.get_ending_sector(), format_hints.Hex(partition_entry_object.get_size_in_sectors()), - interfaces.renderers.Disassembly(bootcode_buf, 0, architecture), - format_hints.HexBytes(bootcode_buf) + renderers.NotApplicableValue(), + renderers.NotApplicableValue() )) else: vollog.log(constants.LOGLEVEL_VVVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") From d17c56af1ce3799485700e2b613c18a2d219ee8e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 26 Mar 2022 01:23:12 +0900 Subject: [PATCH 153/404] Remove space the plugin description --- volatility3/framework/plugins/windows/mbrscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 39e962c96..d064e7d29 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -17,7 +17,7 @@ from volatility3.framework.symbols.windows.extensions import mbr vollog = logging.getLogger(__name__) class MBRScan(interfaces.plugins.PluginInterface): - """ Scans for and parses potential Master Boot Records (MBRs) """ + """Scans for and parses potential Master Boot Records (MBRs)""" _required_framework_version = (2, 0, 1) _version = (1, 0, 0) From 5c402f33e9bc967bd12a3fcfa36eb40b7a8b8b87 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 26 Mar 2022 01:28:36 +0900 Subject: [PATCH 154/404] Improvement of MBR extension's incomplete return type --- .../framework/symbols/windows/extensions/mbr.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 8100371fd..fc7996c52 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -29,34 +29,34 @@ class PARTITION_ENTRY(objects.StructType): """Get Partition Type.""" return self.PartitionType.lookup() if self.PartitionType.is_valid_choice else "Not Defined PartitionType" - def get_starting_chs(self): + def get_starting_chs(self) -> int: """Get Starting CHS (Cylinder Header Sector) Address.""" return self.StartingCHS[0] - def get_ending_chs(self): + def get_ending_chs(self) -> int: """Get Ending CHS (Cylinder Header Sector) Address.""" return self.EndingCHS[0] - def get_starting_sector(self): + def get_starting_sector(self) -> int: """Get Starting Sector.""" return self.StartingCHS[1] % 64 - def get_ending_sector(self): + def get_ending_sector(self) -> int: """Get Ending Sector.""" return self.EndingCHS[1] % 64 - def get_starting_cylinder(self): + def get_starting_cylinder(self) -> int: """Get Starting Cylinder.""" return (self.StartingCHS[1] - self.get_starting_sector()) * 4 + self.StartingCHS[2] - def get_ending_cylinder(self): + def get_ending_cylinder(self) -> int: """Get Ending Cylinder.""" return (self.EndingCHS[1] - self.get_ending_sector()) * 4 + self.EndingCHS[2] - def get_starting_lba(self): + def get_starting_lba(self) -> int: """Get Starting LBA (Logical Block Addressing).""" return self.StartingLBA - def get_size_in_sectors(self): + def get_size_in_sectors(self) -> int: """Get Size in Sectors.""" return self.SizeInSectors From fb081ec233c08548b40b5bc96d182464f68e9795 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 00:36:03 +0900 Subject: [PATCH 155/404] Add Windows DRIVER_OBJECT, DEVICE_OBJECT method --- .../symbols/windows/extensions/__init__.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index dc0de1dda..c1972cb7f 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -351,17 +351,38 @@ class EX_FAST_REF(objects.StructType): class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel device objects.""" - def get_device_name(self) -> str: - header = self.get_object_header() - return header.NameInfo.Name.String # type: ignore + def get_device_name(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: + """Get device's name from the object header.""" + try: + header = self.get_object_header() + return header.NameInfo.Name.String # type: ignore + except(ValueError): + return renderers.UnparsableValue() + def get_attached_devices(self) -> interfaces.objects.ObjectInterface: + """Enumerate the device's attaches""" + device = self.AttachedDevice.dereference() + while device: + yield device + device = device.AttachedDevice.dereference() class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" - def get_driver_name(self) -> str: - header = self.get_object_header() - return header.NameInfo.Name.String # type: ignore + def get_driver_name(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: + """Get driver's name from the object header.""" + try: + header = self.get_object_header() + return header.NameInfo.Name.String # type: ignore + except(ValueError): + return renderers.UnparsableValue() + + def get_devices(self) -> interfaces.objects.ObjectInterface: + """Enumerate the driver's device objects""" + device = self.DeviceObject.dereference() + while device: + yield device + device = device.NextDevice.dereference() def is_valid(self) -> bool: """Determine if the object is valid.""" From 6b91927bb4f389cde1eb6f9bf339706e42cc1e3b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 00:51:35 +0900 Subject: [PATCH 156/404] Initialize DeviceTree plugin --- .../framework/plugins/windows/devicetree.py | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index e69de29bb..67bfcfbbf 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -0,0 +1,145 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterator, List, Tuple + +from volatility3.framework import constants, renderers, exceptions, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import driverscan + +DEVICE_CODES = { + 0x00000027 : "FILE_DEVICE_8042_PORT", + 0x00000032 : "FILE_DEVICE_ACPI", + 0x00000029 : "FILE_DEVICE_BATTERY", + 0x00000001 : "FILE_DEVICE_BEEP", + 0x0000002a : "FILE_DEVICE_BUS_EXTENDER", + 0x00000002 : "FILE_DEVICE_CD_ROM", + 0x00000003 : "FILE_DEVICE_CD_ROM_FILE_SYSTEM", + 0x00000030 : "FILE_DEVICE_CHANGER", + 0x00000004 : "FILE_DEVICE_CONTROLLER", + 0x00000005 : "FILE_DEVICE_DATALINK", + 0x00000006 : "FILE_DEVICE_DFS", + 0x00000035 : "FILE_DEVICE_DFS_FILE_SYSTEM", + 0x00000036 : "FILE_DEVICE_DFS_VOLUME", + 0x00000007 : "FILE_DEVICE_DISK", + 0x00000008 : "FILE_DEVICE_DISK_FILE_SYSTEM", + 0x00000033 : "FILE_DEVICE_DVD", + 0x00000009 : "FILE_DEVICE_FILE_SYSTEM", + 0x0000003a : "FILE_DEVICE_FIPS", + 0x00000034 : "FILE_DEVICE_FULLSCREEN_VIDEO", + 0x0000000a : "FILE_DEVICE_INPORT_PORT", + 0x0000000b : "FILE_DEVICE_KEYBOARD", + 0x0000002f : "FILE_DEVICE_KS", + 0x00000039 : "FILE_DEVICE_KSEC", + 0x0000000c : "FILE_DEVICE_MAILSLOT", + 0x0000002d : "FILE_DEVICE_MASS_STORAGE", + 0x0000000d : "FILE_DEVICE_MIDI_IN", + 0x0000000e : "FILE_DEVICE_MIDI_OUT", + 0x0000002b : "FILE_DEVICE_MODEM", + 0x0000000f : "FILE_DEVICE_MOUSE", + 0x00000010 : "FILE_DEVICE_MULTI_UNC_PROVIDER", + 0x00000011 : "FILE_DEVICE_NAMED_PIPE", + 0x00000012 : "FILE_DEVICE_NETWORK", + 0x00000013 : "FILE_DEVICE_NETWORK_BROWSER", + 0x00000014 : "FILE_DEVICE_NETWORK_FILE_SYSTEM", + 0x00000028 : "FILE_DEVICE_NETWORK_REDIRECTOR", + 0x00000015 : "FILE_DEVICE_NULL", + 0x00000016 : "FILE_DEVICE_PARALLEL_PORT", + 0x00000017 : "FILE_DEVICE_PHYSICAL_NETCARD", + 0x00000018 : "FILE_DEVICE_PRINTER", + 0x00000019 : "FILE_DEVICE_SCANNER", + 0x0000001c : "FILE_DEVICE_SCREEN", + 0x00000037 : "FILE_DEVICE_SERENUM", + 0x0000001a : "FILE_DEVICE_SERIAL_MOUSE_PORT", + 0x0000001b : "FILE_DEVICE_SERIAL_PORT", + 0x00000031 : "FILE_DEVICE_SMARTCARD", + 0x0000002e : "FILE_DEVICE_SMB", + 0x0000001d : "FILE_DEVICE_SOUND", + 0x0000001e : "FILE_DEVICE_STREAMS", + 0x0000001f : "FILE_DEVICE_TAPE", + 0x00000020 : "FILE_DEVICE_TAPE_FILE_SYSTEM", + 0x00000038 : "FILE_DEVICE_TERMSRV", + 0x00000021 : "FILE_DEVICE_TRANSPORT", + 0x00000022 : "FILE_DEVICE_UNKNOWN", + 0x0000002c : "FILE_DEVICE_VDM", + 0x00000023 : "FILE_DEVICE_VIDEO", + 0x00000024 : "FILE_DEVICE_VIRTUAL_DISK", + 0x00000025 : "FILE_DEVICE_WAVE_IN", + 0x00000026 : "FILE_DEVICE_WAVE_OUT", +} + +vollog = logging.getLogger(__name__) + +class DeviceTree(interfaces.plugins.PluginInterface): + """Listing tree based on drivers and attached devices in a particular windows memory image.""" + + _required_framework_version = (2, 0, 1) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name = "kernel", description = "Windows kernel", + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = "driverscan", plugin = driverscan.DriverScan, version = (1, 0, 0)), + ] + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config["kernel"]] + + # Scan the Layer for drivers + for driver in driverscan.DriverScan.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): + try: + driver_name = driver.get_driver_name() + + yield (0, ( + format_hints.Hex(driver.vol.offset), + "DRV", + driver_name, + renderers.NotApplicableValue(), + renderers.NotApplicableValue() + )) + + # Scan to get the device information of driver. + for device in driver.get_devices(): + device_name = device.get_device_name() + device_type = DEVICE_CODES.get(device.DeviceType, "UNKNOWN") + + yield (1, ( + format_hints.Hex(driver.vol.offset), + "DEV", + driver_name, + device_name, + device_type + )) + + # Scan to get the attached devices information of device. + for level, attached_device in enumerate(device.get_attached_devices(), start=2): + device_name = attached_device.get_device_name() + + attached_device_name = "Unparsable Value" if isinstance(device_name, renderers.UnparsableValue) else device_name + name = "{} - {}".format(attached_device_name, attached_device.DriverObject.DriverName.get_string()) + + attached_device_type = DEVICE_CODES.get(attached_device.DeviceType, "UNKNOWN") + + yield (level, ( + format_hints.Hex(driver.vol.offset), + "ATT", + driver_name, + name, + attached_device_type + )) + + except(exceptions.PagedInvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in drivers and devices: {format_hints.Hex(driver.vol.offset)}") + continue + + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid([ + ("Offset", format_hints.Hex), ("Type", str), ("DriverName", str), ("DeviceName", str), ("DeviceType", str), + ], self._generator()) From 8801a8974b72c19527d85587e312dbf3433d2dc9 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 27 Mar 2022 21:19:04 +0530 Subject: [PATCH 157/404] Added Caption To make it look organized in the left side of the readthedocs. --- doc/source/index.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index 3b5a5d2a8..50eaab694 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -10,6 +10,7 @@ Volatility 3 is Open Source. Here are some guidelines for using Volatility 3 effectively: .. toctree:: + :caption: Documentation basics development @@ -18,10 +19,10 @@ Here are some guidelines for using Volatility 3 effectively: volshell glossary -Python Packages -=============== .. toctree:: + :caption: Python Packages + volatility3 Indices and tables From 26251f28c23a7b1ef361ef4d083ce1c2df95e4e5 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 27 Mar 2022 21:33:35 +0530 Subject: [PATCH 158/404] Structure for Getting started added --- doc/source/index.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index 50eaab694..0d35b02ba 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -25,6 +25,15 @@ Here are some guidelines for using Volatility 3 effectively: volatility3 + +.. toctree:: + :caption: Getting Started + + FAQ + Installation + Linux + Windows + Indices and tables ================== From 95a11c366965b03c4f69b6b75687e600d2dd231f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 03:25:16 +0900 Subject: [PATCH 159/404] Core: Bump the development version to 2.0.3 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index badec946b..46d5be577 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_PATCH = 3 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From ff9c5cea4f021485bee51c8ae7fd5b5f3b5b93d8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 03:31:29 +0900 Subject: [PATCH 160/404] Modify return type hint of method for get driver's and device's --- .../framework/symbols/windows/extensions/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index c1972cb7f..2266305fa 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -7,9 +7,10 @@ import datetime import functools import logging import math -from typing import Iterable, Iterator, List, Optional, Tuple, Union +from typing import Generator, Iterable, Iterator, List, Optional, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols +from volatility3.framework.interfaces.objects import ObjectInterface from volatility3.framework.layers import intel from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic @@ -359,7 +360,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): except(ValueError): return renderers.UnparsableValue() - def get_attached_devices(self) -> interfaces.objects.ObjectInterface: + def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the device's attaches""" device = self.AttachedDevice.dereference() while device: @@ -377,7 +378,7 @@ class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): except(ValueError): return renderers.UnparsableValue() - def get_devices(self) -> interfaces.objects.ObjectInterface: + def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" device = self.DeviceObject.dereference() while device: From 2f3bc8cf0d58263c64357ca06ffe0d494648f668 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 03:34:24 +0900 Subject: [PATCH 161/404] Revert method for get driver's and device's name --- .../framework/plugins/windows/devicetree.py | 5 ++--- .../symbols/windows/extensions/__init__.py | 18 ++++++------------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index 67bfcfbbf..add654795 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -77,7 +77,7 @@ vollog = logging.getLogger(__name__) class DeviceTree(interfaces.plugins.PluginInterface): """Listing tree based on drivers and attached devices in a particular windows memory image.""" - _required_framework_version = (2, 0, 1) + _required_framework_version = (2, 0, 3) _version = (1, 0, 0) @classmethod @@ -121,8 +121,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): for level, attached_device in enumerate(device.get_attached_devices(), start=2): device_name = attached_device.get_device_name() - attached_device_name = "Unparsable Value" if isinstance(device_name, renderers.UnparsableValue) else device_name - name = "{} - {}".format(attached_device_name, attached_device.DriverObject.DriverName.get_string()) + name = "{} - {}".format(device_name, attached_device.DriverObject.DriverName.get_string()) attached_device_type = DEVICE_CODES.get(attached_device.DeviceType, "UNKNOWN") diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 2266305fa..b7b53f6e9 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -352,13 +352,10 @@ class EX_FAST_REF(objects.StructType): class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel device objects.""" - def get_device_name(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: + def get_device_name(self) -> str: """Get device's name from the object header.""" - try: - header = self.get_object_header() - return header.NameInfo.Name.String # type: ignore - except(ValueError): - return renderers.UnparsableValue() + header = self.get_object_header() + return header.NameInfo.Name.String # type: ignore def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the device's attaches""" @@ -370,13 +367,10 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" - def get_driver_name(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: + def get_driver_name(self) -> str: """Get driver's name from the object header.""" - try: - header = self.get_object_header() - return header.NameInfo.Name.String # type: ignore - except(ValueError): - return renderers.UnparsableValue() + header = self.get_object_header() + return header.NameInfo.Name.String # type: ignore def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" From 8c29ba9fa5ccb733780378c28ba9c3d5fa856b6c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 03:36:41 +0900 Subject: [PATCH 162/404] Modify code comment of get_attached_devices method --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b7b53f6e9..7d083fbba 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -358,7 +358,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): return header.NameInfo.Name.String # type: ignore def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: - """Enumerate the device's attaches""" + """Enumerate the attached device's objects""" device = self.AttachedDevice.dereference() while device: yield device From 8104ee5fcb6393f6fdfc663a44d164dada15f77d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 03:43:36 +0900 Subject: [PATCH 163/404] Prettier of TreeGrid column --- volatility3/framework/plugins/windows/devicetree.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index add654795..bd1a1ce36 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -140,5 +140,9 @@ class DeviceTree(interfaces.plugins.PluginInterface): def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([ - ("Offset", format_hints.Hex), ("Type", str), ("DriverName", str), ("DeviceName", str), ("DeviceType", str), + ("Offset", format_hints.Hex), + ("Type", str), + ("DriverName", str), + ("DeviceName", str), + ("DeviceType", str), ], self._generator()) From 7b3fa278e058f296940bd1713b908beab8177664 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 16:03:25 +0900 Subject: [PATCH 164/404] Move handling of ValueError, PagedInvalidAddressException to _generator --- .../framework/plugins/windows/devicetree.py | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index bd1a1ce36..8e92de0cc 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -94,19 +94,31 @@ class DeviceTree(interfaces.plugins.PluginInterface): # Scan the Layer for drivers for driver in driverscan.DriverScan.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): try: - driver_name = driver.get_driver_name() + try: + driver_name = driver.get_driver_name() + except (ValueError, exceptions.PagedInvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, + f"Failed to get Driver name : {driver.vol.offset:x}") + driver_name = renderers.UnparsableValue() yield (0, ( format_hints.Hex(driver.vol.offset), "DRV", driver_name, renderers.NotApplicableValue(), + renderers.NotApplicableValue(), renderers.NotApplicableValue() )) # Scan to get the device information of driver. for device in driver.get_devices(): - device_name = device.get_device_name() + try: + device_name = device.get_device_name() + except (ValueError, exceptions.PagedInvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, + f"Failed to get Device name : {device.vol.offset:x}") + device_name = renderers.UnparsableValue() + device_type = DEVICE_CODES.get(device.DeviceType, "UNKNOWN") yield (1, ( @@ -114,35 +126,42 @@ class DeviceTree(interfaces.plugins.PluginInterface): "DEV", driver_name, device_name, + renderers.NotApplicableValue(), device_type )) # Scan to get the attached devices information of device. for level, attached_device in enumerate(device.get_attached_devices(), start=2): - device_name = attached_device.get_device_name() - - name = "{} - {}".format(device_name, attached_device.DriverObject.DriverName.get_string()) + try: + device_name = attached_device.get_device_name() + except (ValueError, exceptions.PagedInvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, + f"Failed to get Attached Device Name: {attached_device.vol.offset:x}") + device_name = renderers.UnparsableValue() + attached_device_driver_name = attached_device.DriverObject.DriverName.get_string() attached_device_type = DEVICE_CODES.get(attached_device.DeviceType, "UNKNOWN") yield (level, ( format_hints.Hex(driver.vol.offset), "ATT", driver_name, - name, + device_name, + attached_device_driver_name, attached_device_type )) except(exceptions.PagedInvalidAddressException): - vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in drivers and devices: {format_hints.Hex(driver.vol.offset)}") + vollog.log(constants.LOGLEVEL_VVVV, + f"Invalid address identified in drivers and devices: {driver.vol.offset:x}") continue - def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([ ("Offset", format_hints.Hex), ("Type", str), ("DriverName", str), ("DeviceName", str), + ("DriverNameOfAttDevice", str), ("DeviceType", str), ], self._generator()) From 655ab9305d86bbb39ddacfdcad721685df092beb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 20:39:11 +0900 Subject: [PATCH 165/404] Fix typo error for licenses, docs --- doc/source/simple-plugin.rst | 2 +- volatility3/framework/plugins/mac/kauth_scopes.py | 2 +- volatility3/framework/plugins/mac/kevents.py | 2 +- volatility3/framework/plugins/mac/vfsevents.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 9360ccf40..ffc7b263f 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -196,7 +196,7 @@ The plugin then defaults the ``BaseDllName`` and ``FullDllName`` variables to an which is a way of indicating to the user interface that the value couldn't be read for some reason (but that it isn't fatal). There are currently four different reasons a value may be unreadable: -* **Unreadble**: values which are empty because the data cannot be read +* **Unredble**: values which are empty because the data cannot be read * **Unparsable**: values which are empty because the data cannot be interpreted correctly * **NotApplicable**: values which are empty because they don't make sense for this particular entry * **NotAvailable**: values which cannot be provided now (but might in a future run, via new symbols or an updated plugin) diff --git a/volatility3/framework/plugins/mac/kauth_scopes.py b/volatility3/framework/plugins/mac/kauth_scopes.py index 910de35fd..f1a2ad345 100644 --- a/volatility3/framework/plugins/mac/kauth_scopes.py +++ b/volatility3/framework/plugins/mac/kauth_scopes.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 16fa51fa4..6f82c75cd 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # diff --git a/volatility3/framework/plugins/mac/vfsevents.py b/volatility3/framework/plugins/mac/vfsevents.py index 38f4172ce..bc5668495 100644 --- a/volatility3/framework/plugins/mac/vfsevents.py +++ b/volatility3/framework/plugins/mac/vfsevents.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # From 0fcc0d8b36bd20a1814f0cac6d29c37b1629de95 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 20:42:13 +0900 Subject: [PATCH 166/404] Fix typo error of docs (Unreadable) --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index ffc7b263f..8446b0ef5 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -196,7 +196,7 @@ The plugin then defaults the ``BaseDllName`` and ``FullDllName`` variables to an which is a way of indicating to the user interface that the value couldn't be read for some reason (but that it isn't fatal). There are currently four different reasons a value may be unreadable: -* **Unredble**: values which are empty because the data cannot be read +* **Unreadable**: values which are empty because the data cannot be read * **Unparsable**: values which are empty because the data cannot be interpreted correctly * **NotApplicable**: values which are empty because they don't make sense for this particular entry * **NotAvailable**: values which cannot be provided now (but might in a future run, via new symbols or an updated plugin) From f51914746428c3ce61b68838389ba0667c08064d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 28 Mar 2022 18:00:19 +0100 Subject: [PATCH 167/404] Windows: Fix the location of a netstat data file Very kindly pointed out by @Digitalisx. Closes #691 --- .../symbols/windows/{ => netscan}/netscan-win81-19935-x64.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename volatility3/framework/symbols/windows/{ => netscan}/netscan-win81-19935-x64.json (100%) diff --git a/volatility3/framework/symbols/windows/netscan-win81-19935-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win81-19935-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-win81-19935-x64.json rename to volatility3/framework/symbols/windows/netscan/netscan-win81-19935-x64.json From 0f6bb99d115fd4b629bbdd5085c354cea98d1d72 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 29 Mar 2022 17:41:37 +0530 Subject: [PATCH 168/404] Cross document linked for symbol table Received help from my friend to resolve issues with it Co-authored-by: Abhinandhan S Signed-off-by: Tejas <47889755+tejas15802@users.noreply.github.com> --- doc/source/Linux.rst | 6 ++++++ doc/source/conf.py | 4 +++- doc/source/symbol-tables.rst | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 doc/source/Linux.rst diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst new file mode 100644 index 000000000..280126cb0 --- /dev/null +++ b/doc/source/Linux.rst @@ -0,0 +1,6 @@ +Linux +===== + +How to create symbol tables + +- :ref:`symbol-tables:Mac or Linux symbol tables`. diff --git a/doc/source/conf.py b/doc/source/conf.py index 731a73d56..cadf6d3f2 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -111,9 +111,11 @@ needs_sphinx = '2.0' # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'sphinx.ext.viewcode' + 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.autosectionlabel' ] +autosectionlabel_prefix_document = True + try: import sphinx_autodoc_typehints diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 245dd9c67..36b283fff 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -38,7 +38,7 @@ following command: The :envvar:`PYTHONPATH` environment variable is not required if the Volatility library is installed in the system's library path or a virtual environment. -Mac/Linux symbol tables +Mac or Linux symbol tables ----------------------- For Mac/Linux systems, both use the same mechanism for identification. JSON files live under the symbol directories, From 02569f4e0658a11142eb248bccd0bb8c356a7342 Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Tue, 29 Mar 2022 11:57:24 -0500 Subject: [PATCH 169/404] refs #668 handle freed windows big pools more accurately. add --show_free option to the bigpools plugin --- .../framework/plugins/windows/bigpools.py | 24 ++++++++++++++----- .../symbols/windows/extensions/pool.py | 5 +++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/bigpools.py b/volatility3/framework/plugins/windows/bigpools.py index c81125f07..329ccdb4e 100644 --- a/volatility3/framework/plugins/windows/bigpools.py +++ b/volatility3/framework/plugins/windows/bigpools.py @@ -21,7 +21,7 @@ class BigPools(interfaces.plugins.PluginInterface): """List big page pools.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -32,7 +32,11 @@ class BigPools(interfaces.plugins.PluginInterface): requirements.StringRequirement(name = 'tags', description = "Comma separated list of pool tags to filter pools returned", optional = True, - default = None) + default = None), + requirements.BooleanRequirement(name = 'show_free', + description = 'Show freed regions (otherwise only show allocations in use)', + default = False, + optional = True) ] @classmethod @@ -40,7 +44,8 @@ class BigPools(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - tags: Optional[list] = None): + tags: Optional[list] = None, + show_free: bool = False): """Returns the big page pool objects from the kernel PoolBigPageTable array. Args: @@ -97,7 +102,7 @@ class BigPools(interfaces.plugins.PluginInterface): for big_pool in big_pools: if big_pool.is_valid(): - if tags is None or big_pool.get_key() in tags: + if (tags is None or big_pool.get_key() in tags) and (show_free or not big_pool.is_free()): yield big_pool def _generator(self) -> Iterator[Tuple[int, Tuple[int, str]]]: # , str, int]]]: @@ -110,13 +115,19 @@ class BigPools(interfaces.plugins.PluginInterface): for big_pool in self.list_big_pools(context = self.context, layer_name = kernel.layer_name, symbol_table = kernel.symbol_table_name, - tags = tags): + tags = tags, + show_free = self.config.get("show_free")): num_bytes = big_pool.get_number_of_bytes() if not isinstance(num_bytes, interfaces.renderers.BaseAbsentValue): num_bytes = format_hints.Hex(num_bytes) - yield (0, (format_hints.Hex(big_pool.Va), big_pool.get_key(), big_pool.get_pool_type(), num_bytes)) + if big_pool.is_free(): + status = "Free" + else: + status = "Allocated" + + yield (0, (format_hints.Hex(big_pool.Va), big_pool.get_key(), big_pool.get_pool_type(), num_bytes, status)) def run(self): return renderers.TreeGrid([ @@ -124,4 +135,5 @@ class BigPools(interfaces.plugins.PluginInterface): ('Tag', str), ('PoolType', str), ('NumberOfBytes', format_hints.Hex), + ('Status', str), ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index d50d6e47a..368765497 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -233,7 +233,10 @@ class POOL_TRACKER_BIG_PAGES(objects.StructType): def is_valid(self) -> bool: return self.Key > 0 - # return self.Va > 0x1 + + def is_free(self) -> bool: + """Returns if the allocation is freed (True) or in-use (False)""" + return self.Va & 1 == 1 def get_key(self) -> str: """Returns the Key value as a 4 character string""" From 7ac6a60ff9546e89919e33598142d19b30f8082c Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Wed, 30 Mar 2022 00:05:08 +0530 Subject: [PATCH 170/404] Updated linux page similar to vol2 wiki --- doc/source/Linux.rst | 73 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 280126cb0..decc0c246 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -1,6 +1,71 @@ -Linux -===== +Linux Tutorial +============== + +This guide gives you a brief introduction to how volatility3 works and some demonstration on suite of plugins available from + +Procedure to create symbol tables for linux +-------------------------------------------- + +To create symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. +You can also find some ISF files from this website `Linux ISF Server `_ Which is built and maintained by `kevthehermit `_. + +Using plugins +------------- + +The following is the syntax to run volatility tool. + +.. code-block:: shell-session + + $ python3 vol.py -f plugin_name plugin_option + +List of Plugins +---------------- + +Following are the list of linux plugins available for volatility3. More plugins will be available on future releases. +For plugin requests, Please create an issue with description of the plugin. + +.. code-block:: shell-session + + $ vol3 --help | grep -i linux + + banners.Banners Attempts to identify potential linux banners in an + linux.bash.Bash Recovers bash command history from memory. + linux.check_afinfo.Check_afinfo + linux.check_creds.Check_creds + linux.check_idt.Check_idt + linux.check_modules.Check_modules + linux.check_syscall.Check_syscall + linux.elfs.Elfs Lists all memory mapped ELF files for all processes. + linux.keyboard_notifiers.Keyboard_notifiers + linux.kmsg.Kmsg Kernel log buffer reader + linux.lsmod.Lsmod Lists loaded kernel modules. + linux.lsof.Lsof Lists all memory maps for all processes. + linux.malfind.Malfind + linux.proc.Maps Lists all memory maps for all processes. + linux.pslist.PsList + Lists the processes present in a particular linux + linux.pstree.PsTree + linux.tty_check.tty_check + + +Acquiring memory +---------------- + +Volatility does not provide the ability to acquire memory. We recommend using `Lime `_ for this purpose. +It supports 32 and 64 bit captures from native Intel hardware as well as virtual machine guests. +It also supports capture from Android devices. See below for example commands building and running LiME: + +.. code-block:: shell-session + + $ tar -xvzf lime-forensics-1.1-r14.tar.gz + $ cd lime-forensics-1.1-r14/src + $ make + .... + CC [M] /home/mhl/Downloads/src/tcp.o + CC [M] /home/mhl/Downloads/src/disk.o + .... + $ sudo insmod lime-3.2.0-23-generic.ko "path=/home/mhl/ubuntu.lime format=lime" + $ ls -alh /home/mhl/ubuntu.lime + -r--r--r-- 1 root root 2.0G Aug 17 19:37 /home/mhl/ubuntu.lime -How to create symbol tables -- :ref:`symbol-tables:Mac or Linux symbol tables`. From 0d4c4b58081b808f248f5fe51516a7b5a7081cf2 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Wed, 30 Mar 2022 08:38:06 +0530 Subject: [PATCH 171/404] Making changes as per review --- doc/source/Linux.rst | 82 +++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 46 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index decc0c246..6ee797c76 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -3,55 +3,10 @@ Linux Tutorial This guide gives you a brief introduction to how volatility3 works and some demonstration on suite of plugins available from -Procedure to create symbol tables for linux --------------------------------------------- - -To create symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. -You can also find some ISF files from this website `Linux ISF Server `_ Which is built and maintained by `kevthehermit `_. - -Using plugins -------------- - -The following is the syntax to run volatility tool. - -.. code-block:: shell-session - - $ python3 vol.py -f plugin_name plugin_option - -List of Plugins ----------------- - -Following are the list of linux plugins available for volatility3. More plugins will be available on future releases. -For plugin requests, Please create an issue with description of the plugin. - -.. code-block:: shell-session - - $ vol3 --help | grep -i linux - - banners.Banners Attempts to identify potential linux banners in an - linux.bash.Bash Recovers bash command history from memory. - linux.check_afinfo.Check_afinfo - linux.check_creds.Check_creds - linux.check_idt.Check_idt - linux.check_modules.Check_modules - linux.check_syscall.Check_syscall - linux.elfs.Elfs Lists all memory mapped ELF files for all processes. - linux.keyboard_notifiers.Keyboard_notifiers - linux.kmsg.Kmsg Kernel log buffer reader - linux.lsmod.Lsmod Lists loaded kernel modules. - linux.lsof.Lsof Lists all memory maps for all processes. - linux.malfind.Malfind - linux.proc.Maps Lists all memory maps for all processes. - linux.pslist.PsList - Lists the processes present in a particular linux - linux.pstree.PsTree - linux.tty_check.tty_check - - Acquiring memory ---------------- -Volatility does not provide the ability to acquire memory. We recommend using `Lime `_ for this purpose. +Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `Lime `_ for this purpose. It supports 32 and 64 bit captures from native Intel hardware as well as virtual machine guests. It also supports capture from Android devices. See below for example commands building and running LiME: @@ -68,4 +23,39 @@ It also supports capture from Android devices. See below for example commands bu $ ls -alh /home/mhl/ubuntu.lime -r--r--r-- 1 root root 2.0G Aug 17 19:37 /home/mhl/ubuntu.lime +Procedure to create symbol tables for linux +-------------------------------------------- + +To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. +We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. + + +Using plugins +------------- + +The following is the syntax to run volatility tool. + +.. code-block:: shell-session + + $ python3 vol.py -f plugin_name plugin_option + +Listing plugins +--------------- + +Following are the list of linux plugins available for volatility3. More plugins will be available on future releases. +For plugin requests, Please create an issue with description of the plugin. + +.. code-block:: shell-session + + $ vol3 --help | grep -i linux. | head -n 5 + banners.Banners Attempts to identify potential linux banners in an + linux.bash.Bash Recovers bash command history from memory. + linux.check_afinfo.Check_afinfo + linux.check_creds.Check_creds + linux.check_idt.Check_idt + + + + + From f4e1533628dcfe5bf45e775259afa68af3e3983e Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sat, 2 Apr 2022 17:28:36 +0530 Subject: [PATCH 172/404] Order changed --- doc/source/Linux.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 6ee797c76..e7f81ff75 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -30,15 +30,6 @@ To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symb We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. -Using plugins -------------- - -The following is the syntax to run volatility tool. - -.. code-block:: shell-session - - $ python3 vol.py -f plugin_name plugin_option - Listing plugins --------------- @@ -55,6 +46,15 @@ For plugin requests, Please create an issue with description of the plugin. linux.check_idt.Check_idt +Using plugins +------------- + +The following is the syntax to run volatility tool. + +.. code-block:: shell-session + + $ python3 vol.py -f plugin_name plugin_option + From b6830a1f3172588e310b149da1bad28377903ad3 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sat, 2 Apr 2022 17:38:46 +0530 Subject: [PATCH 173/404] Additional context for proceudre to create symbol --- doc/source/Linux.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index e7f81ff75..afbccec49 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -28,6 +28,7 @@ Procedure to create symbol tables for linux To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. +After creating the file or downloading the file from the ISF server, please place the file under the directory ``volatility3/symbols/linux``. Make a directory linux under symbols. Listing plugins From 20f1fc39f243e61d9e6fba9c1472a671791462a8 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sat, 2 Apr 2022 18:46:16 +0530 Subject: [PATCH 174/404] Example 1 Added --- doc/source/Linux.rst | 128 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index afbccec49..2913a93b2 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -57,6 +57,134 @@ The following is the syntax to run volatility tool. $ python3 vol.py -f plugin_name plugin_option +Example +------- + +Example 1 +~~~~~~~~~ + +In this example we will be using memory dump from Insomni'hack teaser 2020 CTF. Challenge name Getdents, you can find the memory dump +in the link `here `_ . We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. +I'd like to say thanks to `stuxnet `_ for providing this memory dump and `writeup `_. +.. code-block:: shell-session + $ python3 vol.py -f memory.vmem banners + + Volatility 3 Framework 2.0.3 + + Progress: 100.00 PDB scanning finished + Offset Banner + + 0x141c1390 Linux version 4.15.0-42-generic (buildd@lgw01-amd64-023) (gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)) #45-Ubuntu SMP Thu Nov 15 19:32:57 UTC 2018 (Ubuntu 4.15.0-42.45-generic 4.15.18) + 0x63a00160 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x6455c4d4 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x6e1e055f Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x7fde0010 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + + +This above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for ISF file from the ISF server. +If you do not find the ISF file then, please follow the instructions on :ref:`Linux:Procedure to create symbol tables for linux`. After that place the ISF file under ``volatility3/symbols/linux`` directory. + +.. tip:: Use the banner text which is most repeated to search from ISF Server. + + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.pslist + + Volatility 3 Framework 2.0.3 Stacking attempts finished + + PID PPID COMM + + 1 0 systemd + 2 0 kthreadd + 3 2 kworker/0:0 + 4 2 kworker/0:0H + 5 2 kworker/u256:0 + 6 2 mm_percpu_wq + 7 2 ksoftirqd/0 + 8 2 rcu_sched + 9 2 rcu_bh + 10 2 migration/0 + 11 2 watchdog/0 + 12 2 cpuhp/0 + 13 2 kdevtmpfs + 14 2 netns + 15 2 rcu_tasks_kthre + 16 2 kauditd + ..... + +``linux.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.pstree + Volatility 3 Framework 2.0.3 + Progress: 100.00 Stacking attempts finished + PID PPID COMM + + 1 0 systemd + * 636 1 polkitd + * 514 1 acpid + * 1411 1 pulseaudio + * 517 1 rsyslogd + * 637 1 cups-browsed + * 903 1 whoopsie + * 522 1 ModemManager + * 525 1 cron + * 526 1 avahi-daemon + ** 542 526 avahi-daemon + * 657 1 unattended-upgr + * 914 1 kerneloops + * 532 1 dbus-daemon + * 1429 1 ibus-x11 + * 929 1 kerneloops + * 1572 1 gsd-printer + * 933 1 upowerd + * 1071 1 rtkit-daemon + * 692 1 gdm3 + ** 1234 692 gdm-session-wor + *** 1255 1234 gdm-x-session + **** 1257 1255 Xorg + **** 1266 1255 gnome-session-b + ***** 1537 1266 gsd-clipboard + ***** 1539 1266 gsd-color + ***** 1542 1266 gsd-datetime + ***** 2950 1266 deja-dup-monito + ***** 1546 1266 gsd-housekeepin + ***** 1548 1266 gsd-keyboard + ***** 1550 1266 gsd-media-keys + +``linux.pstree`` helps us to display the parent child relation of processes. + +Now to find the commands ran in bash shell. Lets use ``linux.bash``. + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.bash + + Volatility 3 Framework 2.0.3 + Progress: 100.00 Stacking attempts finished + PID Process CommandTime Command + + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 AWAVH�� + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 rub + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 uname -a + 1733 bash 2020-01-16 14:00:36.000000 uname -a + 1733 bash 2020-01-16 14:00:36.000000 sudo apt autoclean + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:41.000000 chmod +x meterpreter + 1733 bash 2020-01-16 14:00:42.000000 sudo ./meterpreter From ada212da9232a0fff1cdc21bfc2ed7c1793aa20f Mon Sep 17 00:00:00 2001 From: TEJENDRA SARADHI <47889755+tejas15802@users.noreply.github.com> Date: Sun, 3 Apr 2022 06:13:53 +0530 Subject: [PATCH 175/404] lime to LiME Fix inconsistency --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 2913a93b2..294f8ff63 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -6,7 +6,7 @@ This guide gives you a brief introduction to how volatility3 works and some demo Acquiring memory ---------------- -Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `Lime `_ for this purpose. +Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `LiME `_ for this purpose. It supports 32 and 64 bit captures from native Intel hardware as well as virtual machine guests. It also supports capture from Android devices. See below for example commands building and running LiME: From bbffe9f620924c10de7a05efe1a41692aad8d26d Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Wed, 6 Apr 2022 21:08:07 +0530 Subject: [PATCH 176/404] Windows page added and few commands in example1 --- doc/source/Windows.rst | 88 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 doc/source/Windows.rst diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst new file mode 100644 index 000000000..0d8c92c41 --- /dev/null +++ b/doc/source/Windows.rst @@ -0,0 +1,88 @@ +Windows Tutorial +================ + +This guide gives you a brief introduction to how volatility3 works and some demonstration on suite of plugins available from + +Acquiring memory +---------------- + +Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `WinPmem `_ for this purpose. + +Listing Plugins +--------------- + + $ vol3 --help | grep windows | head -n 5 + windows.bigpools.BigPools + windows.cmdline.CmdLine + windows.crashinfo.Crashinfo + windows.dlllist.DllList + Lists the loaded modules in a particular windows + +Using plugins +------------- + +The following is the syntax to run volatility tool. + +.. code-block:: shell-session + + $ python3 vol.py -f plugin_name plugin_option + + +Example +------- + +Example 1 +~~~~~~~~~ + +In this example we will be using memory dump from PragyanCTF'22. The dump is available `here `_. +We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. + +In windows memory forensics using volatility3, most of the times we do not require creating a ISF file. + +.. code-block:: shell-session + + $ vol3 -f MemDump.DMP windows.pslist | head -n 10 + + Volatility 3 Framework 2.0.2 PDB scanning finished + + PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output + + 4 0 System 0xfa8000cbc040 85 492 N/A False 2022-02-07 16:30:12.000000 N/A Disabled + 276 4 smss.exe 0xfa8001e04040 2 29 N/A False 2022-02-07 16:30:12.000000 N/A Disabled + 352 336 csrss.exe 0xfa8002110b30 9 375 0 False 2022-02-07 16:30:13.000000 N/A Disabled + 404 336 wininit.exe 0xfa800219f060 3 74 0 False 2022-02-07 16:30:13.000000 N/A Disabled + 412 396 csrss.exe 0xfa80021c5b30 9 224 1 False 2022-02-07 16:30:13.000000 N/A Disabled + 468 396 winlogon.exe 0xfa8002284060 5 113 1 False 2022-02-07 16:30:14.000000 N/A Disabled + +``windows.pslist`` helps us list the processes running while the memory dump was taken. + +.. code-block:: shell-session + + $ vol3 -f MemDump.DMP windows.pstree | head -n 20 + Volatility 3 Framework 2.0.2 PDB scanning finished + + PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime + + 4 0 System 0xfa8000cbc040 85 492 N/A False 2022-02-07 16:30:12.000000 N/A + * 276 4 smss.exe 0xfa8001e04040 2 29 N/A False 2022-02-07 16:30:12.000000 N/A + 352 336 csrss.exe 0xfa8002110b30 9 375 0 False 2022-02-07 16:30:13.000000 N/A + 404 336 wininit.exe 0xfa800219f060 3 74 0 False 2022-02-07 16:30:13.000000 N/A + * 504 404 services.exe 0xfa80022ccb30 7 190 0 False 2022-02-07 16:30:14.000000 N/A + ** 960 504 svchost.exe 0xfa8001c17b30 39 1003 0 False 2022-02-07 16:30:14.000000 N/A + ** 1216 504 svchost.exe 0xfa80026e0b30 18 311 0 False 2022-02-07 16:30:15.000000 N/A + ** 1312 504 svchost.exe 0xfa8002740380 19 287 0 False 2022-02-07 16:30:15.000000 N/A + ** 1984 504 taskhost.exe 0xfa8002eb1b30 8 129 1 False 2022-02-07 16:30:27.000000 N/A + ** 804 504 svchost.exe 0xfa80024ca5f0 20 450 0 False 2022-02-07 16:30:14.000000 N/A + *** 100 804 audiodg.exe 0xfa80025b4b30 6 131 0 False 2022-02-07 16:30:14.000000 N/A + ** 1568 504 SearchIndexer. 0xfa800254b480 12 616 0 False 2022-02-07 16:30:32.000000 N/A + ** 744 504 svchost.exe 0xfa8002477b30 8 265 0 False 2022-02-07 16:30:14.000000 N/A + ** 1096 504 svchost.exe 0xfa800260db30 14 357 0 False 2022-02-07 16:30:14.000000 N/A + ** 616 504 svchost.exe 0xfa8002b86ab0 13 314 0 False 2022-02-07 16:32:16.000000 N/A + ** 624 504 svchost.exe 0xfa8002410630 10 350 0 False 2022-02-07 16:30:14.000000 N/A + +``windows.pstree`` helps us to display the parent child relation of processes. + + + + + From 39db890ebd7dd0ef191e96831133e46812ed112b Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Wed, 6 Apr 2022 21:10:41 +0530 Subject: [PATCH 177/404] Fix code block syntax highlight --- doc/source/Windows.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index 0d8c92c41..5a197a73e 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -11,6 +11,8 @@ Volatility does not provide the ability to acquire memory. In this tutorial we w Listing Plugins --------------- +.. code-block:: shell-session + $ vol3 --help | grep windows | head -n 5 windows.bigpools.BigPools windows.cmdline.CmdLine From 215a43478932c9fa2f6db52487aab4f3fc672576 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Wed, 6 Apr 2022 21:13:16 +0530 Subject: [PATCH 178/404] Update alias vol3 to python3 vol.py --- doc/source/Windows.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index 5a197a73e..b086cc57b 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -12,8 +12,8 @@ Listing Plugins --------------- .. code-block:: shell-session - - $ vol3 --help | grep windows | head -n 5 + + $ python3 vol.py --help | grep windows | head -n 5 windows.bigpools.BigPools windows.cmdline.CmdLine windows.crashinfo.Crashinfo @@ -43,7 +43,7 @@ In windows memory forensics using volatility3, most of the times we do not requi .. code-block:: shell-session - $ vol3 -f MemDump.DMP windows.pslist | head -n 10 + $ python3 vol.py -f MemDump.DMP windows.pslist | head -n 10 Volatility 3 Framework 2.0.2 PDB scanning finished @@ -60,7 +60,7 @@ In windows memory forensics using volatility3, most of the times we do not requi .. code-block:: shell-session - $ vol3 -f MemDump.DMP windows.pstree | head -n 20 + $ python3 vol.py -f MemDump.DMP windows.pstree | head -n 20 Volatility 3 Framework 2.0.2 PDB scanning finished PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime From 0673282539bd706e44de2fbfbf1ca0f244302a63 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 10 Apr 2022 23:36:34 +0100 Subject: [PATCH 179/404] Configuration: Change unsatisfied response for Modules --- volatility3/cli/__init__.py | 14 ++++++------- .../framework/configuration/requirements.py | 20 +++++++++++++++---- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 35ad84011..488892d37 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,6 @@ import os import sys import tempfile import traceback -from datetime import datetime from typing import Any, Dict, Type, Union from urllib import parse, request @@ -453,16 +452,17 @@ class CommandLine: print(f"Unsatisfied requirement {config_path}: {excp.unsatisfied[config_path].description}") - if symbols_failed: - print("\nA symbol table requirement was not fulfilled. Please verify that:\n" - "\tYou have the correct symbol file for the requirement\n" - "\tThe symbol file is under the correct directory or zip file\n" - "\tThe symbol file is named appropriately or contains the correct banner\n") if translation_failed: print("\nA translation layer requirement was not fulfilled. Please verify that:\n" "\tA file was provided to create this layer (by -f, --single-location or by config)\n" "\tThe file exists and is readable\n" - "\tThe necessary symbols are present and identified by volatility3") + "\tThe file is a valid memory image and was acquired cleanly") + if symbols_failed: + print("\nA symbol table requirement was not fulfilled. Please verify that:\n" + "\tThe associated translation layer requirement was fulfilled\n" + "\tYou have the correct symbol file for the requirement\n" + "\tThe symbol file is under the correct directory or zip file\n" + "\tThe symbol file is named appropriately or contains the correct banner\n") def populate_config(self, context: interfaces.context.ContextInterface, configurables_list: Dict[str, Type[interfaces.configuration.ConfigurableInterface]], diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index a0fb186ae..746b72226 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -10,7 +10,7 @@ expect to be in the context (such as particular layers or symboltables). """ import abc import logging -from typing import Any, ClassVar, List, Optional, Type, Dict, Tuple +from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type from volatility3.framework import constants, interfaces @@ -303,7 +303,8 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if not subreq.optional]): + [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if + not subreq.optional]): return None obj = self._construct_class(context, config_path, args) @@ -358,7 +359,8 @@ class SymbolTableRequirement(interfaces.configuration.ConstructableRequirementIn args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if not subreq.optional]): + [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if + not subreq.optional]): return None # Fill out the parameter for class creation @@ -462,6 +464,15 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa "TypeError - Module Requirement only accepts string labels: {}".format(repr(value))) return {config_path: self} + result = {} + for subreq in self._requirements: + req_unsatisfied = self._requirements[subreq].unsatisfied(context, config_path) + if req_unsatisfied: + result.update(req_unsatisfied) + if not result: + result = {config_path: self} + return result + ### NOTE: This validate method has side effects (the dependencies can change)!!! self._validate_class(context, interfaces.configuration.parent_path(config_path)) @@ -482,7 +493,8 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if not subreq.optional]): + [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if + not subreq.optional]): return None obj = self._construct_class(context, config_path, args) From fd81dba42064414f93e20c724580502be2412528 Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Wed, 20 Apr 2022 13:21:43 -0500 Subject: [PATCH 180/404] refs #668 bump the minor version and not the revision for an additive change --- volatility3/framework/plugins/windows/bigpools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/bigpools.py b/volatility3/framework/plugins/windows/bigpools.py index 329ccdb4e..f8bb332df 100644 --- a/volatility3/framework/plugins/windows/bigpools.py +++ b/volatility3/framework/plugins/windows/bigpools.py @@ -21,7 +21,7 @@ class BigPools(interfaces.plugins.PluginInterface): """List big page pools.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 3e70030d1743ee7b45d33fb1dbb65dee1675ef60 Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Wed, 20 Apr 2022 13:22:18 -0500 Subject: [PATCH 181/404] refs #668 by convention, use show-free instead of show_free --- volatility3/framework/plugins/windows/bigpools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/bigpools.py b/volatility3/framework/plugins/windows/bigpools.py index f8bb332df..9e120446f 100644 --- a/volatility3/framework/plugins/windows/bigpools.py +++ b/volatility3/framework/plugins/windows/bigpools.py @@ -33,7 +33,7 @@ class BigPools(interfaces.plugins.PluginInterface): description = "Comma separated list of pool tags to filter pools returned", optional = True, default = None), - requirements.BooleanRequirement(name = 'show_free', + requirements.BooleanRequirement(name = 'show-free', description = 'Show freed regions (otherwise only show allocations in use)', default = False, optional = True) @@ -116,7 +116,7 @@ class BigPools(interfaces.plugins.PluginInterface): layer_name = kernel.layer_name, symbol_table = kernel.symbol_table_name, tags = tags, - show_free = self.config.get("show_free")): + show_free = self.config.get("show-free")): num_bytes = big_pool.get_number_of_bytes() if not isinstance(num_bytes, interfaces.renderers.BaseAbsentValue): From 72ebf11fd36ff6d0d942181713529c25a02f55f5 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 24 Apr 2022 15:13:55 +0300 Subject: [PATCH 182/404] support CallbackListHead when CmpCallBackVector not present --- .../framework/plugins/windows/callbacks.py | 75 ++++++++++++++----- .../symbols/windows/callbacks-x64.json | 43 +++++++++++ .../symbols/windows/callbacks-x86.json | 43 +++++++++++ 3 files changed, 143 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 352dba448..d8e5aea86 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -111,30 +111,19 @@ class Callbacks(interfaces.plugins.PluginInterface): yield symbol_name, callback.Callback, None @classmethod - def list_registry_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - callback_table_name: str) -> Iterable[Tuple[str, int, None]]: - """Lists all registry callbacks. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - symbol_table: The name of the table containing the kernel symbols - callback_table_name: The nae of the table containing the callback symbols - - Yields: - A name, location and optional detail string + def _list_registry_callbacks_legacy(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, + callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + """ + Lists all registry callbacks from the old format via the CmpCallBackVector. """ kvo = context.layers[layer_name].config['kernel_virtual_offset'] ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) full_type_name = callback_table_name + constants.BANG + "_EX_CALLBACK_ROUTINE_BLOCK" - try: - symbol_offset = ntkrnlmp.get_symbol("CmpCallBackVector").address - symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address - except exceptions.SymbolError: - vollog.debug("Cannot find CmpCallBackVector or CmpCallBackCount") - return + symbol_offset = ntkrnlmp.get_symbol("CmpCallBackVector").address + symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address + callback_count = ntkrnlmp.object(object_type = "unsigned int", offset = symbol_count_offset) @@ -155,6 +144,56 @@ class Callbacks(interfaces.plugins.PluginInterface): if callback.Function != 0: yield "CmRegisterCallback", callback.Function, None + @classmethod + def _list_registry_callbacks_new(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, + callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + """ + Lists all registry callbacks via the CallbackListHead. + """ + + kvo = context.layers[layer_name].config['kernel_virtual_offset'] + ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + full_type_name = callback_table_name + constants.BANG + "_CM_CALLBACK_ENTRY" + + symbol_offset = ntkrnlmp.get_symbol("CallbackListHead").address + symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address + + callback_count = ntkrnlmp.object(object_type = "unsigned int", offset = symbol_count_offset) + + if callback_count == 0: + return + + callback_list = ntkrnlmp.object(object_type = "_LIST_ENTRY", offset = symbol_offset) + for callback in callback_list.to_list(full_type_name, "Link"): + yield "CmRegisterCallbackEx", callback.Function, f"Alltitude: {callback.Alltitude.String}" + + @classmethod + def list_registry_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, + callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + """Lists all registry callbacks. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + callback_table_name: The nae of the table containing the callback symbols + + Yields: + A name, location and optional detail string + """ + + kvo = context.layers[layer_name].config['kernel_virtual_offset'] + ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + full_type_name = callback_table_name + constants.BANG + "_EX_CALLBACK_ROUTINE_BLOCK" + + if ntkrnlmp.has_symbol("CmpCallBackVector") and ntkrnlmp.has_symbol("CmpCallBackCount"): + yield from cls._list_registry_callbacks_legacy(context, layer_name, symbol_table, callback_table_name) + elif ntkrnlmp.has_symbol("CallbackListHead") and ntkrnlmp.has_symbol("CmpCallBackCount"): + yield from cls._list_registry_callbacks_new(context, layer_name, symbol_table, callback_table_name) + else: + vollog.debug("Cannot find CmpCallBackVector or CmpCallBackCount or CallbackListHead") + return + @classmethod def list_bugcheck_reason_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, callback_table_name: str) -> Iterable[Tuple[str, int, str]]: diff --git a/volatility3/framework/symbols/windows/callbacks-x64.json b/volatility3/framework/symbols/windows/callbacks-x64.json index dbb6086df..5300d28f9 100644 --- a/volatility3/framework/symbols/windows/callbacks-x64.json +++ b/volatility3/framework/symbols/windows/callbacks-x64.json @@ -8,6 +8,12 @@ "signed": false, "endian": "little" }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, "unsigned char": { "kind": "char", "size": 1, @@ -137,6 +143,43 @@ }, "kind": "struct", "size": 64 + }, + "_CM_CALLBACK_ENTRY": { + "fields": { + "Link": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Cookie": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 24 + }, + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "Alltitude": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 64 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/callbacks-x86.json b/volatility3/framework/symbols/windows/callbacks-x86.json index cf0cb8b65..52baeb18f 100644 --- a/volatility3/framework/symbols/windows/callbacks-x86.json +++ b/volatility3/framework/symbols/windows/callbacks-x86.json @@ -8,6 +8,12 @@ "signed": false, "endian": "little" }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, "unsigned char": { "kind": "char", "size": 1, @@ -137,6 +143,43 @@ }, "kind": "struct", "size": 28 + }, + "_CM_CALLBACK_ENTRY": { + "fields": { + "Link": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Cookie": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 16 + }, + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 28 + }, + "Alltitude": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 40 } }, "metadata": { From bc04d22a1b7e2969f1ddd5ec6598439724c17bc7 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 24 Apr 2022 16:42:25 +0300 Subject: [PATCH 183/404] remove unused line --- volatility3/framework/plugins/windows/callbacks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index d8e5aea86..c10d97405 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -184,7 +184,6 @@ class Callbacks(interfaces.plugins.PluginInterface): kvo = context.layers[layer_name].config['kernel_virtual_offset'] ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) - full_type_name = callback_table_name + constants.BANG + "_EX_CALLBACK_ROUTINE_BLOCK" if ntkrnlmp.has_symbol("CmpCallBackVector") and ntkrnlmp.has_symbol("CmpCallBackCount"): yield from cls._list_registry_callbacks_legacy(context, layer_name, symbol_table, callback_table_name) From 0057f81269b382fdaa9f15922ec41f0cd1f31faa Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 25 Apr 2022 09:27:54 +0300 Subject: [PATCH 184/404] log which symbol does not exist --- volatility3/framework/plugins/windows/callbacks.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index c10d97405..5ee11f164 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -190,7 +190,14 @@ class Callbacks(interfaces.plugins.PluginInterface): elif ntkrnlmp.has_symbol("CallbackListHead") and ntkrnlmp.has_symbol("CmpCallBackCount"): yield from cls._list_registry_callbacks_new(context, layer_name, symbol_table, callback_table_name) else: - vollog.debug("Cannot find CmpCallBackVector or CmpCallBackCount or CallbackListHead") + symbols_to_check = ["CmpCallBackVector", "CmpCallBackCount", "CallbackListHead"] + vollog.debug("Failed to get registry callbacks!") + for symbol_name in symbols_to_check: + symbol_status = "does not exist" + if ntkrnlmp.has_symbol(symbol_name): + symbol_status = "exists" + vollog.debug(f"symbol {symbol_name} {symbol_status}.") + return @classmethod From 7308f4af0c4da19228b86c2b4aa8adb3773e952e Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 25 Apr 2022 11:21:05 +0300 Subject: [PATCH 185/404] minor improvments --- volatility3/framework/interfaces/objects.py | 19 ++++++++++--- volatility3/framework/interfaces/symbols.py | 14 ++++++++++ .../framework/symbols/windows/__init__.py | 27 +++++++++---------- .../symbols/windows/extensions/__init__.py | 8 ++---- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index e589abd15..c1fb29bb6 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -115,7 +115,15 @@ class ObjectInterface(metaclass = abc.ABCMeta): mask = context.layers[object_info.layer_name].address_mask normalized_offset = object_info.offset & mask - self._vol = collections.ChainMap({}, {'type_name': type_name, 'offset': normalized_offset}, object_info, kwargs) + vol_info_dict = {'type_name': type_name, 'offset': normalized_offset} + if constants.BANG in type_name: + table_name, struct_name = type_name.split(constants.BANG) + vol_info_dict["table_name"] = table_name + vol_info_dict["short_name"] = struct_name + else: + vol_info_dict["short_name"] = type_name + + self._vol = collections.ChainMap({}, vol_info_dict, object_info, kwargs) self._context = context def __getattr__(self, attr: str) -> Any: @@ -142,7 +150,7 @@ class ObjectInterface(metaclass = abc.ABCMeta): """ if constants.BANG not in self.vol.type_name: raise ValueError(f"Unable to determine table for symbol: {self.vol.type_name}") - table_name = self.vol.type_name[:self.vol.type_name.index(constants.BANG)] + table_name = self.vol.table_name if table_name not in self._context.symbol_space: raise KeyError(f"Symbol table not found in context's symbol_space for symbol: {self.vol.type_name}") return table_name @@ -156,7 +164,7 @@ class ObjectInterface(metaclass = abc.ABCMeta): """ # TODO: Carefully consider the implications of casting and how it should work if constants.BANG not in new_type_name: - symbol_table = self.vol['type_name'].split(constants.BANG)[0] + symbol_table = self.get_symbol_table_name() new_type_name = symbol_table + constants.BANG + new_type_name object_template = self._context.symbol_space.get_type(new_type_name) object_template = object_template.clone() @@ -169,6 +177,11 @@ class ObjectInterface(metaclass = abc.ABCMeta): size = object_template.size) return object_template(context = self._context, object_info = object_info) + def at_layer(self, new_layer_name) -> 'ObjectInterface': + """Returns the same object casted at a different layer. + """ + return self._context.object(self.vol.type_name, offset=self.vol.offset, layer_name=new_layer_name) + def has_member(self, member_name: str) -> bool: """Returns whether the object would contain a member called member_name. diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index b271de412..99690054d 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -167,6 +167,20 @@ class BaseSymbolTableInterface: """ raise NotImplementedError("Abstract method set_type_class not implemented yet.") + def try_set_type_class(self, name: str, clazz: Type[objects.ObjectInterface]) -> bool: + """Calls the set_type_class function but does not throw an exception. + Returns whether setting the type class was successfull. + Args: + name: The name of the type to override the class for + clazz: The actual class to override for the provided type name + """ + try: + self.set_type_class(name, clazz) + + return True + except ValueError: + return False + def get_type_class(self, name: str) -> Type[objects.ObjectInterface]: """Returns the class associated with a Symbol type.""" raise NotImplementedError("Abstract method get_type_class not implemented yet.") diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index f09dadedf..468d998c4 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -4,7 +4,7 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import extensions -from volatility3.framework.symbols.windows.extensions import registry, pool +from volatility3.framework.symbols.windows.extensions import registry, pool, pe class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -38,6 +38,11 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('_SHARED_CACHE_MAP', extensions.SHARED_CACHE_MAP) self.set_type_class('_VACB', extensions.VACB) self.set_type_class('_POOL_TRACKER_BIG_PAGES', pool.POOL_TRACKER_BIG_PAGES) + self.set_type_class('_IMAGE_DOS_HEADER', pe.IMAGE_DOS_HEADER) + self.set_type_class('_IMAGE_NT_HEADERS', pe.IMAGE_NT_HEADERS) + + # Might not exist in 32-bit operating systems. + self.try_set_type_class('_IMAGE_NT_HEADERS64', pe.IMAGE_NT_HEADERS) # This doesn't exist in very specific versions of windows try: @@ -49,19 +54,11 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): pass # these don't exist in windows XP - try: - self.set_type_class('_MMADDRESS_NODE', extensions.MMVAD_SHORT) - except ValueError: - pass - + self.try_set_type_class('_MMADDRESS_NODE', extensions.MMVAD_SHORT) + # these were introduced starting in windows 8 - try: - self.set_type_class('_MM_AVL_NODE', extensions.MMVAD_SHORT) - except ValueError: - pass - + self.try_set_type_class('_MM_AVL_NODE', extensions.MMVAD_SHORT) + # these were introduced starting in windows 7 - try: - self.set_type_class('_RTL_BALANCED_NODE', extensions.MMVAD_SHORT) - except ValueError: - pass + self.try_set_type_class('_RTL_BALANCED_NODE', extensions.MMVAD_SHORT) + \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 7d083fbba..2f0f2388c 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -574,13 +574,9 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): proc_layer = self._context.layers[proc_layer_name] if not proc_layer.is_valid(self.Peb): raise exceptions.InvalidAddressException(proc_layer_name, self.Peb, - f"Invalid address at {self.Peb:0x}") + f"Invalid Peb address at {self.Peb:0x}") - sym_table = self.vol.type_name.split(constants.BANG)[0] - peb = self._context.object(f"{sym_table}{constants.BANG}_PEB", - layer_name = proc_layer_name, - offset = self.Peb) - return peb + return self.at_layer(proc_layer_name).Peb def load_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: """Generator for DLLs in the order that they were loaded.""" From c8ab4eb814b79afd4a2553703d97a3749a2e1fd8 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 25 Apr 2022 11:25:45 +0300 Subject: [PATCH 186/404] if not table name is present table name is an empty string --- volatility3/framework/interfaces/objects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index c1fb29bb6..8c7167a78 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -121,6 +121,7 @@ class ObjectInterface(metaclass = abc.ABCMeta): vol_info_dict["table_name"] = table_name vol_info_dict["short_name"] = struct_name else: + vol_info_dict["table_name"] = "" vol_info_dict["short_name"] = type_name self._vol = collections.ChainMap({}, vol_info_dict, object_info, kwargs) From a8d70c065738b3c3691da3ec2a82a6d4a7ca24a6 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Tue, 26 Apr 2022 10:11:12 +0300 Subject: [PATCH 187/404] fix typo --- volatility3/framework/plugins/windows/callbacks.py | 2 +- volatility3/framework/symbols/windows/callbacks-x64.json | 2 +- volatility3/framework/symbols/windows/callbacks-x86.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 5ee11f164..dca17aff7 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -165,7 +165,7 @@ class Callbacks(interfaces.plugins.PluginInterface): callback_list = ntkrnlmp.object(object_type = "_LIST_ENTRY", offset = symbol_offset) for callback in callback_list.to_list(full_type_name, "Link"): - yield "CmRegisterCallbackEx", callback.Function, f"Alltitude: {callback.Alltitude.String}" + yield "CmRegisterCallbackEx", callback.Function, f"Altitude: {callback.Altitude.String}" @classmethod def list_registry_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, diff --git a/volatility3/framework/symbols/windows/callbacks-x64.json b/volatility3/framework/symbols/windows/callbacks-x64.json index 5300d28f9..87682cb92 100644 --- a/volatility3/framework/symbols/windows/callbacks-x64.json +++ b/volatility3/framework/symbols/windows/callbacks-x64.json @@ -170,7 +170,7 @@ }, "offset": 40 }, - "Alltitude": { + "Altitude": { "type": { "kind": "struct", "name": "nt_symbols!_UNICODE_STRING" diff --git a/volatility3/framework/symbols/windows/callbacks-x86.json b/volatility3/framework/symbols/windows/callbacks-x86.json index 52baeb18f..702b68a65 100644 --- a/volatility3/framework/symbols/windows/callbacks-x86.json +++ b/volatility3/framework/symbols/windows/callbacks-x86.json @@ -170,7 +170,7 @@ }, "offset": 28 }, - "Alltitude": { + "Altitude": { "type": { "kind": "struct", "name": "nt_symbols!_UNICODE_STRING" From 1154a2a9f59da058fd9beb19713446c1b84b8218 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Apr 2022 22:20:00 +0100 Subject: [PATCH 188/404] Symbols: Catch bad JSON files without metadata Fixes #719 --- volatility3/framework/symbols/intermed.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index c48fcc8a7..a6a7a0fae 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -11,13 +11,13 @@ import os import pathlib import zipfile from abc import ABCMeta -from typing import Any, Dict, Generator, Iterable, List, Optional, Type, Tuple, Mapping +from typing import Any, Dict, Generator, Iterable, List, Mapping, Optional, Tuple, Type from volatility3 import schemas, symbols from volatility3.framework import class_subclasses, constants, exceptions, interfaces, objects from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources -from volatility3.framework.symbols import native, metadata +from volatility3.framework.symbols import metadata, native vollog = logging.getLogger(__name__) @@ -113,6 +113,9 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): metadata = json_object.get('metadata', None) + if not metadata: + raise exceptions.SymbolSpaceError(f"Invalid ISF file attempted to be parsed: {isf_url}") + # Determine the delegate or throw an exception self._delegate = self._closest_version(metadata.get('format', "0.0.0"), self._versions)(context, config_path, name, json_object, native_types, @@ -540,7 +543,8 @@ class Version3Format(Version2Format): if 'type' in symbol: symbol_type = self._interdict_to_template(symbol['type']) - self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address, type = symbol_type) + self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address, + type = symbol_type) return self._symbol_cache[name] From 4e2dd7b24fb1c2452cbfdb685d0c21614da3edba Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Apr 2022 22:49:05 +0100 Subject: [PATCH 189/404] Core: Bump patch number for additive change to API --- API_CHANGES.md | 19 +++++++++++++++++++ volatility3/framework/constants/__init__.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index 4a65de04b..b3962f072 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,25 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.0.4 +===== +Add in the linux `task.get_threads` method added to rhe API. + +2.0.3 +===== +`DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` added to the API. + +2.0.2 +===== +Fix the behaviour of the offsets returned by the PDB scanner. + +2.0.0 +===== +Remove the `symbol_shift` mechanism, where symbol tables could alter their own symbols. +Symbols from a symbol table are now always the offset values. They can be added to a Module +and when symbols are requested from a Module they are shifted by the module's offset to get +an absolute offset. This can be done with `Module.get_absolute_symbol_address` or as part of +`Module.object_from_symbol(absolute = False, ...)`. 1.2.0 ===== diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 46d5be577..ddc96bf31 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 3 # Number of changes that do not change the interface +VERSION_PATCH = 4 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 5679135f1aeb5ed82421eee37f3a57f6c0f97c53 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 08:12:06 +1000 Subject: [PATCH 190/404] The exception message is not accurate In the hypothetical case that the framework has a lower version than the module requirement, the message would be, for instance: 'Framework interface version 2 is an older revision than the required version 2.2' instead of: 'Framework interface version 2.1 is an older revision than the required version 2.2' See https://github.com/volatilityfoundation/volatility3/pull/593#discussion_r769238045 --- volatility3/framework/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 2c9444130..176eb2242 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -51,7 +51,7 @@ def require_interface_version(*args) -> None: if args[1] > interface_version()[1]: raise RuntimeError( "Framework interface version {} is an older revision than the required version {}".format( - ".".join([str(x) for x in interface_version()[0:1]]), ".".join([str(x) for x in args[0:2]]))) + ".".join([str(x) for x in interface_version()[0:2]]), ".".join([str(x) for x in args[0:2]]))) class NonInheritable(object): From a59de1b918ec4bdbc4d7e5a3ead8f842ae2e84d5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 08:13:24 +1000 Subject: [PATCH 191/404] Updated to resolve merge conflict --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09a2e4820..2afaf84cc 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 1 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 4 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From e5494ce8c9fb370841a0ffe9c1098b84fef11318 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 08:52:25 +1000 Subject: [PATCH 192/404] Temporarely setting interface minor version to zero to allow to merge the upstream latest changes --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 2afaf84cc..abf537811 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 1 # Number of changes that only add to the interface +VERSION_MINOR = 0 # Number of changes that only add to the interface VERSION_PATCH = 4 # Number of changes that do not change the interface VERSION_SUFFIX = "" From c6fbb9ce14a68c82c07d38eddb70d61f6efd00c2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 08:58:05 +1000 Subject: [PATCH 193/404] Restoring framework minor version to meet the mountinfo plugin see f23b288ffcc15234eb25e88ce82df86f7285e0ea --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index abf537811..2afaf84cc 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 0 # Number of changes that only add to the interface +VERSION_MINOR = 1 # Number of changes that only add to the interface VERSION_PATCH = 4 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 0dc0b8ca4019b8401b5478e25949ee2647caebb1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 28 Apr 2022 00:12:41 +0100 Subject: [PATCH 194/404] Core: Bump the API correctly for an addition (to 2.1.0) --- API_CHANGES.md | 2 +- volatility3/framework/constants/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index b3962f072..03fcc010c 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,7 +4,7 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. -2.0.4 +2.1.0 ===== Add in the linux `task.get_threads` method added to rhe API. diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index ddc96bf31..5060906d5 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,8 +39,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 4 # Number of changes that do not change the interface +VERSION_MINOR = 1 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 73a42577d7cdbe42d8e501b25e2cb3682005be39 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 09:24:19 +1000 Subject: [PATCH 195/404] VERSION_PATCH has to be reset after VERSION_MINOR was increased. --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 21b3ac17b..5060906d5 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 1 # Number of changes that only add to the interface -VERSION_PATCH = 4 # Number of changes that do not change the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From a236bdd60047702dea18aed0170dcc2efc4a0cc2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 28 Apr 2022 08:26:52 +0900 Subject: [PATCH 196/404] Fix typo for API_CHANGES.md --- API_CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index 03fcc010c..4e1820eff 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -6,7 +6,7 @@ When an API feature or function is removed or changed, the major version is bump 2.1.0 ===== -Add in the linux `task.get_threads` method added to rhe API. +Add in the linux `task.get_threads` method added to the API. 2.0.3 ===== From 0a3c6297823dd0e49d19864efc9962f3bcef6075 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 10:14:49 +1000 Subject: [PATCH 197/404] Updated to use task's is_thread_group_leader function --- volatility3/framework/plugins/linux/pstree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 174e94ec4..a44310147 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -25,7 +25,7 @@ class PsTree(pslist.PsList): level = 0 proc = self._tasks.get(pid) while proc and proc.parent and proc.parent.pid not in seen: - if self.task_is_thread_group_leader(proc): + if proc.is_thread_group_leader: parent_pid = proc.parent.pid else: parent_pid = proc.tgid From 192759b5d5c8c2622059847fbd915faf8af66e50 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Apr 2022 10:17:39 +1000 Subject: [PATCH 198/404] Updated to use the new task::get_threads --- volatility3/framework/plugins/linux/pslist.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 4c55f853c..a196810aa 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -137,20 +137,10 @@ class PsList(interfaces.plugins.PluginInterface): if filter_func(task): continue - task_threads = [] - current_task = None - next_task = task.thread_group.next - while current_task is None or current_task.vol.offset != task.vol.offset: - current_task = linux.LinuxUtilities.container_of(next_task, "task_struct", "thread_group", vmlinux) - if current_task.is_thread_group_leader: - # Making sure the first task yielded is the Task Group Leader - yield current_task - elif include_threads: - task_threads.append(current_task) - next_task = current_task.thread_group.next + yield task - # yield the other task threads - yield from task_threads + if include_threads: + yield from task.get_threads() def run(self): pids = self.config.get('pid') From 5f879649162f571eb3d5931bbbffac58b5a9e47b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 28 Apr 2022 15:41:29 +0100 Subject: [PATCH 199/404] Linux: Make sure the pslist offset column says that it's virtual --- volatility3/framework/plugins/linux/pslist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index dd1832576..0a9090bd4 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -85,4 +85,4 @@ class PsList(interfaces.plugins.PluginInterface): yield task def run(self): - return renderers.TreeGrid([("OFFSET", format_hints.Hex), ("COMM", str), ("PID", int), ("PPID", int)], self._generator()) + return renderers.TreeGrid([("OFFSET (V)", format_hints.Hex), ("COMM", str), ("PID", int), ("PPID", int)], self._generator()) From b43d61ca036926047a13343eb401ad920cd5e62b Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Thu, 28 Apr 2022 15:42:10 +0000 Subject: [PATCH 200/404] Address feedback from ikelos --- volatility3/framework/plugins/linux/psaux.py | 30 ++++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 31777f458..089bb61c7 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -4,6 +4,7 @@ from typing import Optional +from volatility3.framework.configuration import requirements from volatility3.framework import symbols, exceptions, renderers, interfaces from volatility3.framework.objects import utility from volatility3.plugins.linux import pslist @@ -11,6 +12,19 @@ from volatility3.plugins.linux import pslist class PsAux(pslist.PsList): """ Lists processes with their command line arguments """ + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + description = 'Filter on specific process IDs', + element_type = int, + optional = True) + ] + def _get_command_line_args(self, task: interfaces.objects.ObjectInterface, name: str) -> Optional[str]: """ @@ -41,7 +55,7 @@ class PsAux(pslist.PsList): # get the size of the arguments with sanity checking size_to_read = task.mm.arg_end - task.mm.arg_start - if size_to_read < 1 or size_to_read > 4096: + if not (0 < size_to_read <= 4096): return renderers.UnreadableValue() # attempt to read it all as partial values are invalid and misleading @@ -65,13 +79,11 @@ class PsAux(pslist.PsList): return args - def _generator(self): + def _generator(self, tasks): """ Generates a listing of processes along with command line arguments """ - vmlinux = self.context.modules[self.config['kernel']] - # walk the process list and report the arguments - for task in self.list_tasks(self.context, vmlinux.name): + for task in tasks: pid = task.pid try: @@ -86,5 +98,11 @@ class PsAux(pslist.PsList): yield (0, (pid, ppid, name, args)) def run(self): - return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], self._generator()) + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + + return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], + self._generator( + pslist.PsList.list_tasks(self.context, + self.config['kernel'], + filter_func = filter_func))) From 3175e25420095f237fcc987c1efd970b8cfc3305 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Thu, 28 Apr 2022 16:11:51 +0000 Subject: [PATCH 201/404] Remove the inheritance from pslist --- volatility3/framework/plugins/linux/psaux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 089bb61c7..c62712907 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -8,10 +8,13 @@ from volatility3.framework.configuration import requirements from volatility3.framework import symbols, exceptions, renderers, interfaces from volatility3.framework.objects import utility from volatility3.plugins.linux import pslist +from volatility3.framework.interfaces import plugins -class PsAux(pslist.PsList): +class PsAux(plugins.PluginInterface): """ Lists processes with their command line arguments """ + _required_framework_version = (2, 0, 0) + @classmethod def get_requirements(cls): # Since we're calling the plugin, make sure we have the plugin's requirements From e1b8a0b5cc99bcc3257eb698bb77004148471ab4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 29 Apr 2022 06:42:37 +1000 Subject: [PATCH 202/404] container_of() is no longer needed here --- volatility3/framework/symbols/linux/__init__.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 4df97954d..8d7f00e06 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -266,13 +266,4 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): while list_start: list_struct = vmlinux.object(object_type = struct_name, offset = list_start.vol.offset) yield list_struct - list_start = getattr(list_struct, list_member) - - @classmethod - def container_of(cls, addr, type_name, member_name, vmlinux): - if not addr: - return - type_dec = vmlinux.get_type(type_name) - member_offset = type_dec.relative_child_offset(member_name) - container_addr = addr - member_offset - return vmlinux.object(object_type=type_name, offset=container_addr, absolute=True) \ No newline at end of file + list_start = getattr(list_struct, list_member) \ No newline at end of file From 9aba96a1970e5a469df529e17de836c0c2a9e19b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 29 Apr 2022 06:46:58 +1000 Subject: [PATCH 203/404] Remove unused import module --- volatility3/framework/plugins/linux/pslist.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index a196810aa..45d364224 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -6,7 +6,6 @@ from typing import Callable, Iterable, List, Any, Tuple from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility -from volatility3.framework.symbols import linux class PsList(interfaces.plugins.PluginInterface): From 818002aba49fe8b3ee852a627a6fc13277acb987 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 30 Apr 2022 15:13:34 +0100 Subject: [PATCH 204/404] Codebase: Fix LGTM issues --- volatility3/cli/text_renderer.py | 5 +---- volatility3/framework/configuration/requirements.py | 6 +++--- volatility3/framework/layers/qemu.py | 3 --- volatility3/framework/plugins/windows/ldrmodules.py | 1 - 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 1ddfcca84..08608a3d0 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -322,10 +322,7 @@ class PrettyTextRenderer(CLIRenderer): tab_width = 8 while line.find('\t') >= 0: i = line.find('\t') - if (tab_width > 0): - pad = " " * (tab_width - (i % tab_width)) - else: - pad = "" + pad = " " * (tab_width - (i % tab_width)) line = line.replace("\t", pad, 1) return line diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 746b72226..4edc6d17c 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -470,14 +470,14 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa if req_unsatisfied: result.update(req_unsatisfied) if not result: + vollog.log(constants.LOGLEVEL_V, f"IndexError - No configuration provided: {config_path}") result = {config_path: self} - return result ### NOTE: This validate method has side effects (the dependencies can change)!!! self._validate_class(context, interfaces.configuration.parent_path(config_path)) - vollog.log(constants.LOGLEVEL_V, f"IndexError - No configuration provided: {config_path}") - return {config_path: self} + + return result def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None: """Constructs the appropriate layer and adds it based on the class parameter.""" diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index df383a04a..f1ba1e468 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -3,7 +3,6 @@ # import functools import json -import math from typing import Optional, Dict, Any, Tuple, List, Set from volatility3.framework import interfaces, exceptions, constants @@ -131,8 +130,6 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): index = 8 section_info = dict() current_section_id = -1 - version_id = -1 - name = None while section_byte != self.QEVM_EOF and index <= base_layer.maximum_address: section_byte = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 42eeacd4d..e7c96e946 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -1,7 +1,6 @@ from volatility3.framework import interfaces, constants from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements -from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe From 78eb31014b0a97145582d8ae4beef598c2659b1f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 2 May 2022 00:00:33 +0900 Subject: [PATCH 205/404] Fix: get owning process method from _ETHREAD --- volatility3/framework/symbols/windows/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 7d083fbba..fd5e075da 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -448,9 +448,9 @@ class KMUTANT(objects.StructType, pool.ExecutiveObject): class ETHREAD(objects.StructType): """A class for executive thread objects.""" - def owning_process(self, kernel_layer: str = None) -> interfaces.objects.ObjectInterface: + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" - return self.ThreadsProcess.dereference(kernel_layer) + return self.Tcb.Process.dereference().cast("_EPROCESS") def get_cross_thread_flags(self) -> str: dictCrossThreadFlags = { From 51b901960169b5681f4a1cbebf03c19997685232 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 2 May 2022 10:34:04 +0900 Subject: [PATCH 206/404] Bump: patch version 2.1.1 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 5060906d5..e08bc42bc 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 1 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 8bd7daf28fdbaf95dcfce1cfaa4e25517d24cd71 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 2 May 2022 09:33:41 +0300 Subject: [PATCH 207/404] fix minor improvments --- volatility3/framework/interfaces/objects.py | 10 +--------- volatility3/framework/interfaces/symbols.py | 2 +- volatility3/framework/symbols/windows/__init__.py | 8 ++++---- .../framework/symbols/windows/extensions/__init__.py | 6 +++++- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 8c7167a78..98ceca3d0 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -116,14 +116,6 @@ class ObjectInterface(metaclass = abc.ABCMeta): normalized_offset = object_info.offset & mask vol_info_dict = {'type_name': type_name, 'offset': normalized_offset} - if constants.BANG in type_name: - table_name, struct_name = type_name.split(constants.BANG) - vol_info_dict["table_name"] = table_name - vol_info_dict["short_name"] = struct_name - else: - vol_info_dict["table_name"] = "" - vol_info_dict["short_name"] = type_name - self._vol = collections.ChainMap({}, vol_info_dict, object_info, kwargs) self._context = context @@ -151,7 +143,7 @@ class ObjectInterface(metaclass = abc.ABCMeta): """ if constants.BANG not in self.vol.type_name: raise ValueError(f"Unable to determine table for symbol: {self.vol.type_name}") - table_name = self.vol.table_name + table_name = self.vol.type_name[:self.vol.type_name.index(constants.BANG)] if table_name not in self._context.symbol_space: raise KeyError(f"Symbol table not found in context's symbol_space for symbol: {self.vol.type_name}") return table_name diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index 99690054d..9f2cb9fc9 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -167,7 +167,7 @@ class BaseSymbolTableInterface: """ raise NotImplementedError("Abstract method set_type_class not implemented yet.") - def try_set_type_class(self, name: str, clazz: Type[objects.ObjectInterface]) -> bool: + def optional_set_type_class(self, name: str, clazz: Type[objects.ObjectInterface]) -> bool: """Calls the set_type_class function but does not throw an exception. Returns whether setting the type class was successfull. Args: diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index 468d998c4..b5129bb04 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -42,7 +42,7 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('_IMAGE_NT_HEADERS', pe.IMAGE_NT_HEADERS) # Might not exist in 32-bit operating systems. - self.try_set_type_class('_IMAGE_NT_HEADERS64', pe.IMAGE_NT_HEADERS) + self.optional_set_type_class('_IMAGE_NT_HEADERS64', pe.IMAGE_NT_HEADERS) # This doesn't exist in very specific versions of windows try: @@ -54,11 +54,11 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): pass # these don't exist in windows XP - self.try_set_type_class('_MMADDRESS_NODE', extensions.MMVAD_SHORT) + self.optional_set_type_class('_MMADDRESS_NODE', extensions.MMVAD_SHORT) # these were introduced starting in windows 8 - self.try_set_type_class('_MM_AVL_NODE', extensions.MMVAD_SHORT) + self.optional_set_type_class('_MM_AVL_NODE', extensions.MMVAD_SHORT) # these were introduced starting in windows 7 - self.try_set_type_class('_RTL_BALANCED_NODE', extensions.MMVAD_SHORT) + self.optional_set_type_class('_RTL_BALANCED_NODE', extensions.MMVAD_SHORT) \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 2f0f2388c..e7da0316d 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -576,7 +576,11 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): raise exceptions.InvalidAddressException(proc_layer_name, self.Peb, f"Invalid Peb address at {self.Peb:0x}") - return self.at_layer(proc_layer_name).Peb + sym_table = self.get_symbol_table_name() + peb = self._context.object(f"{sym_table}{constants.BANG}_PEB", + layer_name = proc_layer_name, + offset = self.Peb) + return peb def load_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: """Generator for DLLs in the order that they were loaded.""" From bb1e41f59be2444a4efeb7f81043f3ed214211fa Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 2 May 2022 09:36:50 +0300 Subject: [PATCH 208/404] remove at_layer --- volatility3/framework/interfaces/objects.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 98ceca3d0..2240c58c9 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -170,11 +170,6 @@ class ObjectInterface(metaclass = abc.ABCMeta): size = object_template.size) return object_template(context = self._context, object_info = object_info) - def at_layer(self, new_layer_name) -> 'ObjectInterface': - """Returns the same object casted at a different layer. - """ - return self._context.object(self.vol.type_name, offset=self.vol.offset, layer_name=new_layer_name) - def has_member(self, member_name: str) -> bool: """Returns whether the object would contain a member called member_name. From 14c11b24f6af44f627bc276c76b2b53d852724fa Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 2 May 2022 14:33:35 +0100 Subject: [PATCH 209/404] Symbols: Make _IMAGE_NT_HEADERS optional since not all Windows versions contain it --- volatility3/framework/symbols/windows/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index b5129bb04..899b89dc2 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -39,9 +39,9 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('_VACB', extensions.VACB) self.set_type_class('_POOL_TRACKER_BIG_PAGES', pool.POOL_TRACKER_BIG_PAGES) self.set_type_class('_IMAGE_DOS_HEADER', pe.IMAGE_DOS_HEADER) - self.set_type_class('_IMAGE_NT_HEADERS', pe.IMAGE_NT_HEADERS) - # Might not exist in 32-bit operating systems. + # Might not necessarily defined in every version of windows + self.optional_set_type_class('_IMAGE_NT_HEADERS', pe.IMAGE_NT_HEADERS) self.optional_set_type_class('_IMAGE_NT_HEADERS64', pe.IMAGE_NT_HEADERS) # This doesn't exist in very specific versions of windows From 6b7d5b63fe6d3fbef6fdcf339dc5c6b9426129e8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 6 May 2022 00:49:03 +0900 Subject: [PATCH 210/404] Add: venv environments, memory dump for .gitignore --- .gitignore | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitignore b/.gitignore index c6da33754..5986612ea 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,16 @@ config*.json # Pyinstaller files build dist + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Memory dump files +*.dmp +*.vmem From 16313c593989a9d5c42afa430194bfe248766c80 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 6 May 2022 01:18:31 +0900 Subject: [PATCH 211/404] Fix: typo for dlllist --- volatility3/framework/plugins/windows/dlllist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index b24f2c0ca..2fd7deeaf 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -65,7 +65,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): try: name = dll_entry.FullDllName.get_string() except exceptions.InvalidAddressException: - name = 'UnreadbleDLLName' + name = 'UnreadableDLLName' if layer_name is None: layer_name = dll_entry.vol.layer_name From 8b6194f8f4231ce8304366f2e985ca7e47f71e4e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 7 May 2022 18:16:43 +0900 Subject: [PATCH 212/404] Remove: environment .bak on .gitignore --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index 5986612ea..d26e17d91 100644 --- a/.gitignore +++ b/.gitignore @@ -34,8 +34,6 @@ dist env/ venv/ ENV/ -env.bak/ -venv.bak/ # Memory dump files *.dmp From 8f369180e4c1dcb8fafcc618b78450bcf36b7bb2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 8 May 2022 18:33:38 +0900 Subject: [PATCH 213/404] Fix: typo for documents --- doc/source/symbol-tables.rst | 2 +- doc/source/volshell.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 245dd9c67..4dea6077d 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -63,7 +63,7 @@ To determine the string for a particular memory image, use the `banners` plugin. try to locate that exact kernel debugging package for the operating system. Unfortunately each distribution provides its debugging packages under different package names and there are so many that the distribution may not keep all old versions of the debugging symbols, and therefore **it may not be possible to find the right symbols to analyze a linux -memory image with volatlity**. With Macs there are far fewer kernels and only one distribution, making it easier to +memory image with volatility**. With Macs there are far fewer kernels and only one distribution, making it easier to ensure that the right symbols can be found. Once a kernel with debugging symbols/appropriate DWARF file has been located, `dwarf2json `_ will convert it into an diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index de3c4398a..5a4b21ade 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -110,7 +110,7 @@ This means that pointers do not need to be explicitly dereferenced to access und Running plugins --------------- -It's possible to run any plugin by importing it appropriately and passing it to the `display_plugin_ouptut` or `dpo` +It's possible to run any plugin by importing it appropriately and passing it to the `display_plugin_output` or `dpo` method. In the following example we'll provide no additional parameters. Volatility will show us which parameters were required: From fde05cd1f2d7a35dd1949988f667d7f97d8e9459 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 9 May 2022 11:23:13 +0900 Subject: [PATCH 214/404] Fix: typo for cli exception message --- volatility3/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 488892d37..e3fb726a1 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -423,7 +423,7 @@ class CommandLine: detail = f"{excp}" caused_by = ["A required python module is not installed (install the module and re-run)"] else: - general = "Volatilty encountered an unexpected situation." + general = "Volatility encountered an unexpected situation." detail = "" caused_by = [ "Please re-run using with -vvv and file a bug with the output", f"at {constants.BUG_URL}" From f54edee36203d8537bf6716a577799bd9184bb1c Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 9 May 2022 10:56:40 +0300 Subject: [PATCH 215/404] removed svcscan import --- volatility3/framework/plugins/windows/callbacks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index dca17aff7..2195671df 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -11,7 +11,6 @@ from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.plugins.windows import ssdt -from volatility3.plugins.windows import svcscan vollog = logging.getLogger(__name__) @@ -28,7 +27,6 @@ class Callbacks(interfaces.plugins.PluginInterface): requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'ssdt', plugin = ssdt.SSDT, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'svcscan', plugin = svcscan.SvcScan, version = (1, 0, 0)) ] @staticmethod From 2c181fb9befd81079e575a554b3d4a625378a876 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 9 May 2022 19:38:25 +1000 Subject: [PATCH 216/404] Move path_root check up so path() and get_mnt_sb() can be avoided in the cases _do_get_path() fails. --- volatility3/framework/plugins/linux/mountinfo.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 6c0f8bcc3..3179eadd8 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -89,6 +89,10 @@ class MountInfo(plugins.PluginInterface): if not mnt_root: return None + path_root = cls._do_get_path(mnt, task.fs.root) + if path_root is None: + return None + mnt_root_path = mnt_root.path() superblock = mnt.get_mnt_sb() @@ -97,10 +101,6 @@ class MountInfo(plugins.PluginInterface): st_dev = f"{superblock.major}:{superblock.minor}" - path_root = cls._do_get_path(mnt, task.fs.root) - if path_root is None: - return None - mnt_opts: List[str] = [] mnt_opts.append(mnt.get_flags_access()) mnt_opts.extend(mnt.get_flags_opts()) From 62e00c087990c5094dcdd9ab23aeb559eee7f640 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 9 May 2022 20:18:11 +1000 Subject: [PATCH 217/404] Added smear protection in loops. Added/improved some docstrings. --- .../symbols/linux/extensions/__init__.py | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 43a6e0d1d..89c20fc0c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -202,14 +202,14 @@ class task_struct(generic.GenericIntelProcess): yield (start, end - start) def get_threads(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns a list of the task_struct based on the list_head + """Returns a list of the task_struct based on the list_head thread_node structure.""" task_symbol_table_name = self.get_symbol_table_name() # iterating through the thread_list from thread_group - # this allows iterating through pointers to grab the - # threads and using the thread_group offset to get the + # this allows iterating through pointers to grab the + # threads and using the thread_group offset to get the # corresponding task_struct for task in self.thread_group.to_list( f"{task_symbol_table_name}{constants.BANG}task_struct", @@ -425,12 +425,15 @@ class qstr(objects.StructType): class dentry(objects.StructType): def path(self) -> str: - """ Based on __dentry_path Linux kernel function""" + """Based on __dentry_path Linux kernel function""" reversed_path = [] + dentry_seen = set() current_dentry = self - while not current_dentry.is_root(): + while (not current_dentry.is_root() and + current_dentry.vol.offset not in dentry_seen): parent = current_dentry.d_parent reversed_path.append(current_dentry.d_name.name_as_str()) + dentry_seen.add(current_dentry.vol.offset) current_dentry = parent return "/" + "/".join(reversed(reversed_path)) @@ -455,11 +458,14 @@ class dentry(objects.StructType): if "ancestor_dentry" is an ancestor of "child_dentry", else None. """ + dentry_seen = set() current_dentry = self - while not current_dentry.is_root(): + while (not current_dentry.is_root() and + current_dentry.vol.offset not in dentry_seen): if current_dentry.d_parent == ancestor_dentry.vol.offset: return current_dentry + dentry_seen.add(current_dentry.vol.offset) current_dentry = current_dentry.d_parent return None @@ -632,32 +638,48 @@ class mount(objects.StructType): def get_dominating_id(self, root) -> int: """Get ID of closest dominating peer group having a representative under the given root.""" + mnt_seen = set() current_mnt = self.mnt_master - while current_mnt and current_mnt.vol.offset != 0: + while (current_mnt and + current_mnt.vol.offset != 0 and + current_mnt.vol.offset not in mnt_seen): peer = current_mnt.get_peer_under_root(self.mnt_ns, root) if peer and peer.vol.offset != 0: return peer.mnt_group_id + mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.mnt_master return 0 def get_peer_under_root(self, ns, root): - current = self - while True: - if current.mnt_ns == ns and current.is_path_reachable(current.mnt.mnt_root, root): - return current - current = current.next_peer() - if current.vol.offset == self.vol.offset: + """Return true if path is reachable from root. + It mimics the kernel function is_path_reachable(), ref: fs/namespace.c + """ + mnt_seen = set() + current_mnt = self + while current_mnt.vol.offset not in mnt_seen: + if current_mnt.mnt_ns == ns and current_mnt.is_path_reachable(current_mnt.mnt.mnt_root, root): + return current_mnt + + mnt_seen.add(current_mnt.vol.offset) + current_mnt = current_mnt.next_peer() + if current_mnt.vol.offset == self.vol.offset: break return None def is_path_reachable(self, current_dentry, root): - """Return true if path is reachable + """Return true if path is reachable. + It mimics the kernel function with same name, ref fs/namespace.c: """ + mnt_seen = set() current_mnt = self - while current_mnt.mnt.vol.offset != root.mnt and current_mnt.has_parent(): + while (current_mnt.mnt.vol.offset != root.mnt and + current_mnt.has_parent() and + current_mnt.vol.offset not in mnt_seen): + current_dentry = current_mnt.mnt_mountpoint + mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.mnt_parent return current_mnt.mnt.vol.offset == root.mnt and current_dentry.is_subdir(root.dentry) From f4dd582f158e8024e3fc5b4fba21a727f0913bfb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 12:29:36 +0900 Subject: [PATCH 218/404] Fix: ThreadsProcess for windows older version --- .../framework/symbols/windows/extensions/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b317f7693..ccfcb4290 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -450,7 +450,12 @@ class ETHREAD(objects.StructType): def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" - return self.Tcb.Process.dereference().cast("_EPROCESS") + if(self.has_member("ThreadsProcess")): + return self.ThreadsProcess.dereference().cast("_EPROCESS") + elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): + return self.Tcb.Process.dereference().cast("_EPROCESS") + else: + raise AttributeError("Unable to find the owning process of ethread") def get_cross_thread_flags(self) -> str: dictCrossThreadFlags = { From 1125be122e8d330cad156ba67a279bf83f22c2f0 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 12:31:56 +0900 Subject: [PATCH 219/404] Add: code comment for windows version --- volatility3/framework/symbols/windows/extensions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ccfcb4290..a1c347ed2 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -450,6 +450,7 @@ class ETHREAD(objects.StructType): def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" + if(self.has_member("ThreadsProcess")): return self.ThreadsProcess.dereference().cast("_EPROCESS") elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): From 3956f0ecc0406f32482123c0c1866755b8fd7cf7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 12:36:56 +0900 Subject: [PATCH 220/404] Add: code comment for windows version --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index a1c347ed2..805f8c26b 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -451,8 +451,10 @@ class ETHREAD(objects.StructType): def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" + # For Windows XPs if(self.has_member("ThreadsProcess")): return self.ThreadsProcess.dereference().cast("_EPROCESS") + # For Windows Vista and later versions elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): return self.Tcb.Process.dereference().cast("_EPROCESS") else: From 9fc6e5725c739e9339a5d4cb97dbb0b3ae3e08a4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 10 May 2022 17:24:32 +1000 Subject: [PATCH 221/404] Bump framework version to 2.2.0 --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/linux/mountinfo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 5060906d5..472a743e6 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 1 # Number of changes that only add to the interface +VERSION_MINOR = 2 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 3179eadd8..6f3cb712d 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -20,7 +20,7 @@ MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "m class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" - _required_framework_version = (2, 1, 0) + _required_framework_version = (2, 2, 0) _version = (1, 0, 0) From 690d8e3efe8ec08d4500b75ba60f14a281a52673 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 19:07:04 +0900 Subject: [PATCH 222/404] Fix: sync bump version --- volatility3/framework/constants/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index e08bc42bc..472a743e6 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,8 +39,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 1 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 2 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From a5bf5548b8d7e83ffd3b9065e968e321f6fcc964 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 19:08:29 +0900 Subject: [PATCH 223/404] Bump: patch version 2.2.1 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 472a743e6..44b98b95f 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 2 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From e5dfc47cc419d4d1ac929782008bc24765f46428 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 19:08:56 +0900 Subject: [PATCH 224/404] Fix: required framework version of psscan by bump --- volatility3/framework/plugins/windows/psscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index a0601aef1..cc030b4bf 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -22,7 +22,7 @@ vollog = logging.getLogger(__name__) class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for processes present in a particular windows memory image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 2, 1) _version = (1, 1, 0) @classmethod From 64cabc154679f629c113347798b3f1ea9ae6d44a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 20:53:03 +0900 Subject: [PATCH 225/404] Fix: typo, unification for API_CHANGES.md --- API_CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index 4e1820eff..e4b229dcf 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -10,11 +10,11 @@ Add in the linux `task.get_threads` method added to the API. 2.0.3 ===== -`DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` added to the API. +Add in the windows `DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` method added to the API. 2.0.2 ===== -Fix the behaviour of the offsets returned by the PDB scanner. +Fix the behavior of the offsets returned by the PDB scanner. 2.0.0 ===== From b9abb3f06c114ebac3191c785b041defe9c78cb8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 10 May 2022 20:53:18 +0900 Subject: [PATCH 226/404] Fix: typo for windows code comment --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index e7da0316d..69e8ba94e 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -719,7 +719,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): env = envar[:split_index] var = envar[split_index + 1:] - # Exlude parse problem with some types of env + # Exclude parse problem with some types of env if env and var: yield env, var From b81eb04fb26e1a84677a00e1992eff071bc85386 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 11 May 2022 07:39:45 +0900 Subject: [PATCH 227/404] Revert: british english by code review --- API_CHANGES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index e4b229dcf..274d1d8bb 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -6,15 +6,15 @@ When an API feature or function is removed or changed, the major version is bump 2.1.0 ===== -Add in the linux `task.get_threads` method added to the API. +Add in the linux `task.get_threads` method to the API. 2.0.3 ===== -Add in the windows `DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` method added to the API. +Add in the windows `DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` methods to the API. 2.0.2 ===== -Fix the behavior of the offsets returned by the PDB scanner. +Fix the behaviour of the offsets returned by the PDB scanner. 2.0.0 ===== From 51bce3e62049155b88feb6459b3a75bd1dcdffd7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 00:25:13 +0900 Subject: [PATCH 228/404] Fix: compare logic for key_path and top_key --- volatility3/plugins/windows/registry/certificates.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 96a7e3977..5bc2b18e3 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -6,7 +6,6 @@ from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey - class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" @@ -52,7 +51,7 @@ class Certificates(interfaces.plugins.PluginInterface): node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) - unique_key_offset = key_path.index(top_key) + len(top_key) + 1 + unique_key_offset = key_path.casefold().index(top_key.casefold()) + len(top_key) + 1 reg_section = key_path[unique_key_offset:key_path.index("\\", unique_key_offset)] key_hash = key_path[key_path.rindex("\\") + 1:] From fa6465dcfb05c3d3f63f3903d5d062eda3ddc131 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 00:26:39 +0900 Subject: [PATCH 229/404] Revert: blank line --- volatility3/plugins/windows/registry/certificates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 5bc2b18e3..91f17fb2d 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -6,6 +6,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey + class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" From baaedf21e51b33c9c3eee772296ebfd3b9dd9869 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 19:37:30 +0900 Subject: [PATCH 230/404] Add: plugin version, logger, dump options --- .../plugins/windows/registry/certificates.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 91f17fb2d..81b7d766f 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,16 +1,19 @@ +import logging import struct from typing import List, Iterator, Tuple -from volatility3.framework import interfaces, renderers +from volatility3.framework import constants, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey +vollog = logging.getLogger(__name__) class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -20,7 +23,11 @@ class Certificates(interfaces.plugins.PluginInterface): architectures = ["Intel32", "Intel64"]), requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)) + requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)), + requirements.BooleanRequirement(name = 'dump', + description = "Extract listed certificates", + default = False, + optional = True) ] def parse_data(self, data: bytes) -> Tuple[str, bytes]: @@ -48,21 +55,22 @@ class Certificates(interfaces.plugins.PluginInterface): try: # Walk it node_path = hive.get_key(top_key, return_list = True) - for (depth, is_key, last_write_time, key_path, volatility, - node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): + for (_, is_key, _, key_path, _, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = key_path.casefold().index(top_key.casefold()) + len(top_key) + 1 reg_section = key_path[unique_key_offset:key_path.index("\\", unique_key_offset)] key_hash = key_path[key_path.rindex("\\") + 1:] - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - with self.open("{} - {} - {}.crt".format(hex(hive.hive_offset), reg_section, - key_hash)) as file_data: - file_data.write(certificate_data) + if self.config['dump']: + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + with self.open("{} - {} - {}.crt".format(hex(hive.hive_offset), reg_section, + key_hash)) as file_data: + file_data.write(certificate_data) yield (0, (top_key, reg_section, key_hash, name)) except KeyError: # Key wasn't found in this hive, carry on + vollog.log(constants.LOGLEVEL_VVVV, "Key wasn't found in this hive") pass def run(self) -> renderers.TreeGrid: From dafd62d6cee5f8366e3071adfc80e1c910d301fa Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 19:52:57 +0900 Subject: [PATCH 231/404] Fix: dump options for depreated step --- .../plugins/windows/registry/certificates.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 81b7d766f..f261b5e36 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -41,6 +41,12 @@ class Certificates(interfaces.plugins.PluginInterface): elif ctype == 0x100000020: certificate_data = cvalue return (name, certificate_data) + + def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str) -> str: + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) + with self.open(dump_name) as file_data: + file_data.write(certificate_data) def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: for hive in hivelist.HiveList.list_hives(self.context, @@ -63,10 +69,11 @@ class Certificates(interfaces.plugins.PluginInterface): key_hash = key_path[key_path.rindex("\\") + 1:] if self.config['dump']: - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - with self.open("{} - {} - {}.crt".format(hex(hive.hive_offset), reg_section, - key_hash)) as file_data: - file_data.write(certificate_data) + self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) + else: + vollog.warning("Certificates plugin is no longer support automatically dumped, please use the dump option.") + self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) + yield (0, (top_key, reg_section, key_hash, name)) except KeyError: # Key wasn't found in this hive, carry on From f63f869506186d0396625d6232f9657ca1dad717 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 12 May 2022 20:23:42 +0900 Subject: [PATCH 232/404] Remove: return type of dump method --- volatility3/plugins/windows/registry/certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index f261b5e36..ee9cba6d3 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -42,7 +42,7 @@ class Certificates(interfaces.plugins.PluginInterface): certificate_data = cvalue return (name, certificate_data) - def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str) -> str: + def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str): if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) with self.open(dump_name) as file_data: From ba4a7d9262103ab3c650db47dba4f0d93c7fa6e6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 15 May 2022 00:17:46 +0900 Subject: [PATCH 233/404] Fix: JSON renderer for EOF format --- volatility3/cli/text_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 08608a3d0..ecb5179e0 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -345,7 +345,7 @@ class JsonRenderer(CLIRenderer): def output_result(self, outfd, result): """Outputs the JSON data to a file in a particular format""" - outfd.write(json.dumps(result, indent = 2, sort_keys = True)) + outfd.write("{}\n".format(json.dumps(result, indent = 2, sort_keys = True))) def render(self, grid: interfaces.renderers.TreeGrid): outfd = sys.stdout From 0e8958b8416350ac9c22961674532e62b72050ec Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 16 May 2022 15:11:07 +0900 Subject: [PATCH 234/404] Fix: classmethod, variable name, exceptions, etc --- .../plugins/windows/registry/certificates.py | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index ee9cba6d3..a27b3545a 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,8 +1,8 @@ import logging import struct -from typing import List, Iterator, Tuple +from typing import List, Iterator, Tuple, Type -from volatility3.framework import constants, interfaces, renderers +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey @@ -13,7 +13,6 @@ class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -42,11 +41,20 @@ class Certificates(interfaces.plugins.PluginInterface): certificate_data = cvalue return (name, certificate_data) - def dump_data(self, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str): - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) - with self.open(dump_name) as file_data: - file_data.write(certificate_data) + @classmethod + def dump_certificate(cls, certificate_data: bytes, hive_offset: int, + reg_section: str, key_hash: str, + open_method: Type[interfaces.plugins.FileHandlerInterface]) -> \ + interfaces.plugins.FileHandlerInterface: + try: + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) + with open_method(dump_name) as file_data: + file_data.write(certificate_data) + except exceptions.InvalidAddressException: + vollog.debug(f"Unable to certificate file at {hive_offset:#x}") + return None + def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: for hive in hivelist.HiveList.list_hives(self.context, @@ -61,7 +69,7 @@ class Certificates(interfaces.plugins.PluginInterface): try: # Walk it node_path = hive.get_key(top_key, return_list = True) - for (_, is_key, _, key_path, _, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): + for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = key_path.casefold().index(top_key.casefold()) + len(top_key) + 1 @@ -69,10 +77,9 @@ class Certificates(interfaces.plugins.PluginInterface): key_hash = key_path[key_path.rindex("\\") + 1:] if self.config['dump']: - self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) - else: - vollog.warning("Certificates plugin is no longer support automatically dumped, please use the dump option.") - self.dump_data(certificate_data, hive.hive_offset, reg_section, key_hash) + file_handle = self.dump_certificate(certificate_data, hive.hive_offset, reg_section, key_hash, self.open) + if file_handle: + file_handle.close() yield (0, (top_key, reg_section, key_hash, name)) except KeyError: From 5770d35a4afd718760ddeeb1c21606e6b5bd1e2e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 17 May 2022 00:18:06 +0900 Subject: [PATCH 235/404] Add: return file handle --- volatility3/plugins/windows/registry/certificates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index a27b3545a..b079844e7 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -49,8 +49,9 @@ class Certificates(interfaces.plugins.PluginInterface): try: if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) - with open_method(dump_name) as file_data: - file_data.write(certificate_data) + file_handle = open_method(dump_name) + file_handle.write(certificate_data) + return file_handle except exceptions.InvalidAddressException: vollog.debug(f"Unable to certificate file at {hive_offset:#x}") return None From 3846268bf7a8b3731a80a61959ca1aee227a112d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 17 May 2022 00:19:39 +0900 Subject: [PATCH 236/404] Add: optional return type --- volatility3/plugins/windows/registry/certificates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index b079844e7..d2fb61f02 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,6 +1,6 @@ import logging import struct -from typing import List, Iterator, Tuple, Type +from typing import List, Iterator, Optional, Tuple, Type from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -45,7 +45,7 @@ class Certificates(interfaces.plugins.PluginInterface): def dump_certificate(cls, certificate_data: bytes, hive_offset: int, reg_section: str, key_hash: str, open_method: Type[interfaces.plugins.FileHandlerInterface]) -> \ - interfaces.plugins.FileHandlerInterface: + Optional[interfaces.plugins.FileHandlerInterface]: try: if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) From 05ae20c78bd982e95ab3dd99d92a2efaf68be3b3 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 22 May 2022 11:49:18 +0300 Subject: [PATCH 237/404] fix off by in filelayer --- volatility3/framework/layers/physical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 73d46b211..5d5fd17a9 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -118,7 +118,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): with self._lock: orig = self._file.tell() self._file.seek(0, 2) - self._size = self._file.tell() + self._size = self._file.tell() - 1 self._file.seek(orig) return self._size From 98001e7dd72ac6e39f440191da019ec33a9e64c5 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 22 May 2022 23:58:38 +0900 Subject: [PATCH 238/404] Fix: typo for code comments --- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/symbols.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index ab81beb5e..85a7d32b7 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -321,7 +321,7 @@ class SizedModule(Module): The mapping should be sorted and should be quicker than reading the data We turn it into JSON to make a common string and use a - quick hash, because collissions are unlikely + quick hash, because collisions are unlikely """ layer = self._context.layers[self.layer_name] if not isinstance(layer, interfaces.layers.TranslationLayerInterface): diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index 9f2cb9fc9..1ad30cfdf 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -169,7 +169,7 @@ class BaseSymbolTableInterface: def optional_set_type_class(self, name: str, clazz: Type[objects.ObjectInterface]) -> bool: """Calls the set_type_class function but does not throw an exception. - Returns whether setting the type class was successfull. + Returns whether setting the type class was successful. Args: name: The name of the type to override the class for clazz: The actual class to override for the provided type name From 2a011487a91c9f8e71b86a80e0d186e03b84a5b4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 22 May 2022 23:23:02 +0100 Subject: [PATCH 239/404] Core: Old linux systems may not have mnt_namespace structures --- volatility3/framework/symbols/linux/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0c5ce395c..d59a95db5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,10 +1,10 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import List, Tuple, Iterator +from typing import Iterator, List, Tuple from volatility3 import framework -from volatility3.framework import exceptions, constants, interfaces, objects +from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -29,7 +29,9 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('files_struct', extensions.files_struct) self.set_type_class('vfsmount', extensions.vfsmount) self.set_type_class('kobject', extensions.kobject) - self.set_type_class('mnt_namespace', extensions.mnt_namespace) + + if 'mnt_namespace' in self.types: + self.set_type_class('mnt_namespace', extensions.mnt_namespace) if 'module' in self.types: self.set_type_class('module', extensions.module) @@ -267,4 +269,4 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): while list_start: list_struct = vmlinux.object(object_type = struct_name, offset = list_start.vol.offset) yield list_struct - list_start = getattr(list_struct, list_member) \ No newline at end of file + list_start = getattr(list_struct, list_member) From 786fd61fc9b6b2978e0de7595bdb22aca9e91843 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 9 Apr 2022 20:57:34 +0100 Subject: [PATCH 240/404] Layers: Add architecture to qemu layer --- volatility3/framework/layers/qemu.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index f1ba1e468..65f8eed9a 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -3,9 +3,9 @@ # import functools import json -from typing import Optional, Dict, Any, Tuple, List, Set +from typing import Any, Dict, List, Optional, Set, Tuple -from volatility3.framework import interfaces, exceptions, constants +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.layers import segmented from volatility3.framework.symbols import intermed @@ -39,6 +39,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): metadata: Optional[Dict[str, Any]] = None) -> None: self._qemu_table_name = intermed.IntermediateSymbolTable.create(context, config_path, 'generic', 'qemu') self._configuration = None + self._architecture = None self._compressed: Set[int] = set() self._current_segment_name = b'' super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) @@ -139,6 +140,9 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', offset = index, layer_name = self._base_layer) + self._architecture = self.context.object(self._qemu_table_name + constants.BANG + 'string', + offset = index + 4, layer_name = self._base_layer, + max_length = section_len) index += 4 + section_len elif section_byte == self.QEVM_SECTION_START or section_byte == self.QEVM_SECTION_FULL: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', From 302bb63645af3b9b20b2a361a73c06a2c27e3513 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 9 Apr 2022 21:46:47 +0100 Subject: [PATCH 241/404] Layers: Detect and compensate for QEVM pci-hole --- volatility3/framework/layers/qemu.py | 30 +++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 65f8eed9a..4e6252b2f 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -3,12 +3,17 @@ # import functools import json +import logging +import re +import struct from typing import Any, Dict, List, Optional, Set, Tuple from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.layers import segmented from volatility3.framework.symbols import intermed +vollog = logging.getLogger(__name__) + class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): """A Qemu suspend-to-disk translation layer.""" @@ -32,6 +37,13 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): SEGMENT_FLAG_XBZRLE = 0x40 SEGMENT_FLAG_HOOK = 0x80 + pci_hole_table = {re.compile(r"^pc-i440fx-\d\.\d$"): (0xc0000000, 0x100000000), + re.compile(r"^pc-1440fx-eoan$"): (0xe0000000, 0x100000000), + re.compile(r"^pc-q35$"): (0x80000000, 0x100000000), + re.compile(r"^microvm$"): (0xc0000000, 0x100000000), + re.compile(r"^xen$"): (0xf0000000, 0x100000000) + } + def __init__(self, context: interfaces.context.ContextInterface, config_path: str, @@ -42,6 +54,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._architecture = None self._compressed: Set[int] = set() self._current_segment_name = b'' + self._pci_hole_start = 0 + self._pci_hole_end = 0 super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) @classmethod @@ -77,9 +91,9 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): base_layer = self.context.layers[self._base_layer] while not done: - addr = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', - offset = index, - layer_name = self._base_layer) + # Use struct.unpack here for performance improvements + addr = struct.unpack('>Q', base_layer.read(index, 8))[0] + # Flags are stored in the n least significant bits, where n equals the bit-length of pagesize flags = addr & (page_size - 1) # addr equals the highest multiple of pagesize <= offset @@ -87,6 +101,9 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): addr = addr ^ (addr & (page_size - 1)) index += 8 + if addr > self._pci_hole_start: + addr += self._pci_hole_end - self._pci_hole_start + if flags & self.SEGMENT_FLAG_MEM_SIZE: namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, @@ -143,6 +160,13 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._architecture = self.context.object(self._qemu_table_name + constants.BANG + 'string', offset = index + 4, layer_name = self._base_layer, max_length = section_len) + for regex in self.pci_hole_table: + if regex.match(self._architecture): + self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM archicture detected as: {self._architecture}") + break + else: + vollog.debug(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") index += 4 + section_len elif section_byte == self.QEVM_SECTION_START or section_byte == self.QEVM_SECTION_FULL: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', From dea8e1dac9090b40823a941fdcfc1a828f80de9e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 18 Apr 2022 01:45:12 +0100 Subject: [PATCH 242/404] Layers: Add QEVM architecture fallback detection --- volatility3/framework/layers/qemu.py | 97 ++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 4e6252b2f..7835cad17 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -9,7 +9,7 @@ import struct from typing import Any, Dict, List, Optional, Set, Tuple from volatility3.framework import constants, exceptions, interfaces -from volatility3.framework.layers import segmented +from volatility3.framework.layers import scanners, segmented from volatility3.framework.symbols import intermed vollog = logging.getLogger(__name__) @@ -37,11 +37,32 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): SEGMENT_FLAG_XBZRLE = 0x40 SEGMENT_FLAG_HOOK = 0x80 - pci_hole_table = {re.compile(r"^pc-i440fx-\d\.\d$"): (0xc0000000, 0x100000000), - re.compile(r"^pc-1440fx-eoan$"): (0xe0000000, 0x100000000), - re.compile(r"^pc-q35$"): (0x80000000, 0x100000000), - re.compile(r"^microvm$"): (0xc0000000, 0x100000000), - re.compile(r"^xen$"): (0xf0000000, 0x100000000) + # See https://qemu.readthedocs.io/en/latest/devel/memory.html for more info + # + # At least the following values could occur for devices using > 3-4 GB RAM: + # +--------------------------------+--------------------------------+------------+-------------+ + # | Architecture | Reference Code | Hole Start | Hole End | + # +--------------------------------+--------------------------------+------------+-------------+ + # | PC i440FX + PIIX "New Default" | qemu/hw/i386/pc_piix.c:98 | 0xc0000000 | 0x100000000 | + # | PC i440FX + PIIX "Old Default" | qemu/hw/i386/pc_piix.c:98 | 0xe0000000 | 0x100000000 | + # | PC Q35 + ICH9 | qemu/hw/i386/pc_q35.c:141 | 0x80000000 | 0x100000000 | + # | MicroVM | qemu/hw/i386/microvm.c:291 | 0xc0000000 | 0x100000000 | + # | Xen | qemu/hw/i386/xen/xen-hvm.c:248 | 0xf0000000 | 0x100000000 | + # +--------------------------------+--------------------------------+------------+-------------+ + # + # For now, we assume that the parameter max-ram-below-4g is not set, since this parameter influences the size + # and location of the memory gap. Deviating hole sizes could eventually be detected for Linux by e.g. scanning + # for dmesg entries with a regex like rb'\[mem (0x[0-9a-f]{4,10})-0x[0-9a-f]{4,10}\] available for PCI devices' + + debian_re = r"artful|eoan" + + pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), + re.compile(r"^pc-i440fx-[01].\d$"): (0xe0000000, 0xe0000000, 0x100000000), + re.compile(r"^pc-q35-\d.\d$"): (0xe0000000, 0x80000000, 0x100000000), + re.compile(r"^microvm$"): (0xe0000000, 0xc0000000, 0x100000000), + re.compile(r"^xen$"): (0xe0000000, 0xf0000000, 0x100000000), + re.compile(r"^pc-i440fx-" + debian_re + r"$"): (0xe0000000, 0xc0000000, 0x100000000), + re.compile(r"^pc-q35-" + debian_re + r"$"): (0xe0000000, 0x80000000, 0x100000000), } def __init__(self, @@ -65,6 +86,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): raise exceptions.LayerException(name, 'No QEMU magic bytes') if header[4:] != b'\x00\x00\x00\x03': raise exceptions.LayerException(name, 'Unsupported QEMU version found') + vollog.debug("QEVM header found") def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, name: str) -> Any: """Reads the JSON configuration from the end of the file""" @@ -160,13 +182,6 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._architecture = self.context.object(self._qemu_table_name + constants.BANG + 'string', offset = index + 4, layer_name = self._base_layer, max_length = section_len) - for regex in self.pci_hole_table: - if regex.match(self._architecture): - self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] - vollog.log(constants.LOGLEVEL_VVVV, f"QEVM archicture detected as: {self._architecture}") - break - else: - vollog.debug(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") index += 4 + section_len elif section_byte == self.QEVM_SECTION_START or section_byte == self.QEVM_SECTION_FULL: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', @@ -217,6 +232,62 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): else: raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}') + # If no architecture has been set, try to determine it using fallback mechanisms + if not self._architecture: + self._architecture = self._fallback_determine_architecture() + if self._architecture is None: + vollog.log(constants.LOGLEVEL_VV, f"QEVM architecture could not be determined") + + # Once all segments have been read, determine the PCI hole if any + for regex in self.pci_hole_table: + if regex.match(self._architecture): + self._pci_hole_minimum, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] + if self.maximum_address < self._pci_hole_minimum: + # The PCI hole isn't present because we're below the minimum value + self._pci_hole_start, self._pci_hole_end = 0, 0 + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") + break + else: + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") + + def _fallback_determine_architecture(self) -> str: + architecture_pattern = rb'pc-(i440fx|q35)-([0-9]{1,2}.[0-9]{1,2}(?:.[0-9]{1,2})?)' + base_layer = self.context.layers[self._base_layer] + + vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") + + res = scanners.RegExScanner(architecture_pattern) + for offset in base_layer.scan(context = self.context, scanner = res): + line = base_layer.read(offset, 64) + regex_results = re.search(architecture_pattern, line) + architecture = "pc-" + regex_results.groups()[0].decode() + return architecture + + # If that does not work, look in configuration JSON for devices specific to a certain architecture + architecture = None + for device in self._configuration.get('devices', []): + device_name = device.get('vmsd_name', '').lower() + if 'i440fx' in device_name or 'piix' in device_name: + architecture = 'pc-i440fx-2.0' + break + elif 'ich9' in device_name: + architecture = 'pc-q35-1.0' + break + if architecture: + return architecture + + # Still haven't found architecture, switch to fallback-method + architecture_pattern = rb'Standard PC \((i440FX|Q35)' + res = scanners.RegExScanner(architecture_pattern) + for offset in base_layer.scan(context = self.context, scanner = res): + line = base_layer.read(offset, 64) + regex_results = re.search(architecture_pattern, line) + architecture = "pc-" + regex_results.groups()[0].decode().lower() + return architecture + + vollog.warning("Could not determine QEMU target architecture!") + return None + def extract_data(self, index, name, version_id): if name == 'ram': if version_id != 4: From 1fca57ffc3603ab383332076f05e54f973c21452 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 20 Apr 2022 21:43:30 +0100 Subject: [PATCH 243/404] Layers: QEVM more fixes for minimum addresses --- volatility3/framework/layers/qemu.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 7835cad17..23c29dc39 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -54,15 +54,15 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): # and location of the memory gap. Deviating hole sizes could eventually be detected for Linux by e.g. scanning # for dmesg entries with a regex like rb'\[mem (0x[0-9a-f]{4,10})-0x[0-9a-f]{4,10}\] available for PCI devices' - debian_re = r"artful|eoan" + distro_re = r"(artful|eoan|rhel[\d\.]+)" pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), - re.compile(r"^pc-i440fx-[01].\d$"): (0xe0000000, 0xe0000000, 0x100000000), - re.compile(r"^pc-q35-\d.\d$"): (0xe0000000, 0x80000000, 0x100000000), - re.compile(r"^microvm$"): (0xe0000000, 0xc0000000, 0x100000000), - re.compile(r"^xen$"): (0xe0000000, 0xf0000000, 0x100000000), - re.compile(r"^pc-i440fx-" + debian_re + r"$"): (0xe0000000, 0xc0000000, 0x100000000), - re.compile(r"^pc-q35-" + debian_re + r"$"): (0xe0000000, 0x80000000, 0x100000000), + re.compile(r"^pc-i440fx-[01]\.\d$"): (0xe0000000, 0xe0000000, 0x100000000), + re.compile(r"^pc-q35-\d\.\d$"): (0xb0000000, 0x80000000, 0x100000000), + re.compile(r"^microvm$"): (0xc0000000, 0xc0000000, 0x100000000), + re.compile(r"^xen$"): (0xf0000000, 0xf0000000, 0x100000000), + re.compile(r"^pc-i440fx-" + distro_re + r"$"): (0xe0000000, 0xe0000000, 0x100000000), + re.compile(r"^pc-q35-" + distro_re + r"$"): (0xb0000000, 0x80000000, 0x100000000), } def __init__(self, @@ -252,6 +252,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): def _fallback_determine_architecture(self) -> str: architecture_pattern = rb'pc-(i440fx|q35)-([0-9]{1,2}.[0-9]{1,2}(?:.[0-9]{1,2})?)' + old_suffix = "-1.0" base_layer = self.context.layers[self._base_layer] vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") @@ -260,7 +261,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for offset in base_layer.scan(context = self.context, scanner = res): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) - architecture = "pc-" + regex_results.groups()[0].decode() + architecture = "pc-" + regex_results.groups()[0].decode() + old_suffix return architecture # If that does not work, look in configuration JSON for devices specific to a certain architecture @@ -268,10 +269,10 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for device in self._configuration.get('devices', []): device_name = device.get('vmsd_name', '').lower() if 'i440fx' in device_name or 'piix' in device_name: - architecture = 'pc-i440fx-2.0' + architecture = 'pc-i440fx' + old_suffix break elif 'ich9' in device_name: - architecture = 'pc-q35-1.0' + architecture = 'pc-q35' + old_suffix break if architecture: return architecture @@ -282,7 +283,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for offset in base_layer.scan(context = self.context, scanner = res): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) - architecture = "pc-" + regex_results.groups()[0].decode().lower() + architecture = "pc-" + regex_results.groups()[0].decode().lower() + old_suffix return architecture vollog.warning("Could not determine QEMU target architecture!") From afec05106127a8de73c22e7cc7f87ab3dd67ef9d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 21 Apr 2022 10:29:26 +0100 Subject: [PATCH 244/404] Layers: Make QEVM changes based on @cstation 's feedback --- volatility3/framework/layers/qemu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 23c29dc39..16f18cca1 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -123,7 +123,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): addr = addr ^ (addr & (page_size - 1)) index += 8 - if addr > self._pci_hole_start: + if addr >= self._pci_hole_start: addr += self._pci_hole_end - self._pci_hole_start if flags & self.SEGMENT_FLAG_MEM_SIZE: @@ -252,7 +252,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): def _fallback_determine_architecture(self) -> str: architecture_pattern = rb'pc-(i440fx|q35)-([0-9]{1,2}.[0-9]{1,2}(?:.[0-9]{1,2})?)' - old_suffix = "-1.0" + old_suffix = "-2.0" base_layer = self.context.layers[self._base_layer] vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") @@ -261,7 +261,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for offset in base_layer.scan(context = self.context, scanner = res): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) - architecture = "pc-" + regex_results.groups()[0].decode() + old_suffix + architecture = regex_results.group().decode() return architecture # If that does not work, look in configuration JSON for devices specific to a certain architecture From fb861eb6dfc528ec8f4c2a3f71a0995bdada0cba Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 21 Apr 2022 10:33:29 +0100 Subject: [PATCH 245/404] Layers: Simplification of QEVM fallback regex --- volatility3/framework/layers/qemu.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 16f18cca1..11bec2dd3 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -61,7 +61,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): re.compile(r"^pc-q35-\d\.\d$"): (0xb0000000, 0x80000000, 0x100000000), re.compile(r"^microvm$"): (0xc0000000, 0xc0000000, 0x100000000), re.compile(r"^xen$"): (0xf0000000, 0xf0000000, 0x100000000), - re.compile(r"^pc-i440fx-" + distro_re + r"$"): (0xe0000000, 0xe0000000, 0x100000000), + re.compile(r"^pc-i440fx-" + distro_re + r"$"): (0xe0000000, 0xc0000000, 0x100000000), re.compile(r"^pc-q35-" + distro_re + r"$"): (0xb0000000, 0x80000000, 0x100000000), } @@ -251,7 +251,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") def _fallback_determine_architecture(self) -> str: - architecture_pattern = rb'pc-(i440fx|q35)-([0-9]{1,2}.[0-9]{1,2}(?:.[0-9]{1,2})?)' + architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|[\w\d\.]+)' old_suffix = "-2.0" base_layer = self.context.layers[self._base_layer] From 21b0cb56a5746410f8ff9c96fba9d0e0ee86730f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 21 Apr 2022 23:50:13 +0100 Subject: [PATCH 246/404] Layers: Shift when we calculate the QEVM PCI hole --- volatility3/framework/layers/qemu.py | 39 +++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 11bec2dd3..270405645 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -170,7 +170,28 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): index = 8 section_info = dict() current_section_id = -1 + version_id = -1 + name = None + arch_detected = False while section_byte != self.QEVM_EOF and index <= base_layer.maximum_address: + if index > 20 and not arch_detected: + # We're past where the QEVM_CONFIGURATION might be, so set the values + # If no architecture has been set, try to determine it using fallback mechanisms + if not self._architecture: + self._architecture = self._fallback_determine_architecture() + if self._architecture is None: + vollog.log(constants.LOGLEVEL_VV, f"QEVM architecture could not be determined") + + # Once all segments have been read, determine the PCI hole if any + for regex in self.pci_hole_table: + if regex.match(self._architecture): + _, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") + break + else: + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") + arch_detected = True + section_byte = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) @@ -232,24 +253,6 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): else: raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}') - # If no architecture has been set, try to determine it using fallback mechanisms - if not self._architecture: - self._architecture = self._fallback_determine_architecture() - if self._architecture is None: - vollog.log(constants.LOGLEVEL_VV, f"QEVM architecture could not be determined") - - # Once all segments have been read, determine the PCI hole if any - for regex in self.pci_hole_table: - if regex.match(self._architecture): - self._pci_hole_minimum, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] - if self.maximum_address < self._pci_hole_minimum: - # The PCI hole isn't present because we're below the minimum value - self._pci_hole_start, self._pci_hole_end = 0, 0 - vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") - break - else: - vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") - def _fallback_determine_architecture(self) -> str: architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|[\w\d\.]+)' old_suffix = "-2.0" From 4eacdd9bea6be1be5506273abba5a4cc7715e7ff Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 25 Apr 2022 00:09:07 +0100 Subject: [PATCH 247/404] Layers: use QEVM size to turn off pci hole if needed --- volatility3/framework/layers/qemu.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 270405645..c755b8a9b 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -77,6 +77,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._current_segment_name = b'' self._pci_hole_start = 0 self._pci_hole_end = 0 + self._pci_hole_minimum = 0 super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) @classmethod @@ -110,6 +111,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): done = None segments = [] + size_array = {} base_layer = self.context.layers[self._base_layer] while not done: @@ -131,14 +133,21 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): offset = index, layer_name = self._base_layer) while namelen != 0: - # if base_layer.read(index + 1, namelen) == b'pc.ram': - # total_size = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', - # offset = index + 1 + namelen, - # layer_name = self._base_layer) + total_size = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', + offset = index + 1 + namelen, + layer_name = self._base_layer) + size_array[base_layer.read(index + 1, namelen)] = total_size index += 1 + namelen + 8 namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) + if size_array.get(b'pc.ram', + max([x[0] for x in self.pci_hole_table.values()]) + 1) <= self._pci_hole_minimum: + # Turns off the pci_hole if it's not supposed to be there + vollog.debug( + f"QEVM tunrning off PCI hole due to small image size: {size_array.get(b'pc.ram'):x} < {self._pci_hole_minimum:x}") + self._pci_hole_start, self._pci_hole_end = 0, 0 + if flags & (self.SEGMENT_FLAG_COMPRESS | self.SEGMENT_FLAG_PAGE): if not (flags & self.SEGMENT_FLAG_CONTINUE): namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', @@ -185,7 +194,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): # Once all segments have been read, determine the PCI hole if any for regex in self.pci_hole_table: if regex.match(self._architecture): - _, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] + self._pci_hole_minimum, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") break else: From dd8fcce2ef683aa4bad211ee9273a865a59e84e1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Apr 2022 22:30:14 +0100 Subject: [PATCH 248/404] Layers: QEMU improvements suggested by @cstation --- volatility3/framework/layers/qemu.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index c755b8a9b..e3d70ba6e 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -54,7 +54,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): # and location of the memory gap. Deviating hole sizes could eventually be detected for Linux by e.g. scanning # for dmesg entries with a regex like rb'\[mem (0x[0-9a-f]{4,10})-0x[0-9a-f]{4,10}\] available for PCI devices' - distro_re = r"(artful|eoan|rhel[\d\.]+)" + distro_re = r"(\w+[\d\.]?)" pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), re.compile(r"^pc-i440fx-[01]\.\d$"): (0xe0000000, 0xe0000000, 0x100000000), @@ -141,8 +141,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) - if size_array.get(b'pc.ram', - max([x[0] for x in self.pci_hole_table.values()]) + 1) <= self._pci_hole_minimum: + highest_possible_maximum = max([x[0] for x in self.pci_hole_table.values()]) + 1 + if size_array.get(b'pc.ram', highest_possible_maximum) < self._pci_hole_minimum: # Turns off the pci_hole if it's not supposed to be there vollog.debug( f"QEVM tunrning off PCI hole due to small image size: {size_array.get(b'pc.ram'):x} < {self._pci_hole_minimum:x}") @@ -264,7 +264,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): def _fallback_determine_architecture(self) -> str: architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|[\w\d\.]+)' - old_suffix = "-2.0" + default_suffix = "-2.0" base_layer = self.context.layers[self._base_layer] vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") @@ -281,10 +281,10 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for device in self._configuration.get('devices', []): device_name = device.get('vmsd_name', '').lower() if 'i440fx' in device_name or 'piix' in device_name: - architecture = 'pc-i440fx' + old_suffix + architecture = 'pc-i440fx' + default_suffix break elif 'ich9' in device_name: - architecture = 'pc-q35' + old_suffix + architecture = 'pc-q35' + default_suffix break if architecture: return architecture @@ -295,7 +295,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): for offset in base_layer.scan(context = self.context, scanner = res): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) - architecture = "pc-" + regex_results.groups()[0].decode().lower() + old_suffix + architecture = "pc-" + regex_results.groups()[0].decode().lower() + default_suffix return architecture vollog.warning("Could not determine QEMU target architecture!") From 7ce95117484e13ab0ba6a4a50051b56252cca978 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 23 May 2022 01:20:46 +0100 Subject: [PATCH 249/404] Layers: QEMU recommendations from @cstation --- volatility3/framework/layers/qemu.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index e3d70ba6e..985c4534a 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -54,7 +54,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): # and location of the memory gap. Deviating hole sizes could eventually be detected for Linux by e.g. scanning # for dmesg entries with a regex like rb'\[mem (0x[0-9a-f]{4,10})-0x[0-9a-f]{4,10}\] available for PCI devices' - distro_re = r"(\w+[\d\.]?)" + distro_re = r"(\w+[\d{1,2}\.]*)" pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), re.compile(r"^pc-i440fx-[01]\.\d$"): (0xe0000000, 0xe0000000, 0x100000000), @@ -145,7 +145,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if size_array.get(b'pc.ram', highest_possible_maximum) < self._pci_hole_minimum: # Turns off the pci_hole if it's not supposed to be there vollog.debug( - f"QEVM tunrning off PCI hole due to small image size: {size_array.get(b'pc.ram'):x} < {self._pci_hole_minimum:x}") + f"QEVM tunrning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}") self._pci_hole_start, self._pci_hole_end = 0, 0 if flags & (self.SEGMENT_FLAG_COMPRESS | self.SEGMENT_FLAG_PAGE): @@ -179,8 +179,6 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): index = 8 section_info = dict() current_section_id = -1 - version_id = -1 - name = None arch_detected = False while section_byte != self.QEVM_EOF and index <= base_layer.maximum_address: if index > 20 and not arch_detected: @@ -263,7 +261,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}') def _fallback_determine_architecture(self) -> str: - architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|[\w\d\.]+)' + architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|\w+[\d{1,2}\.]*)' default_suffix = "-2.0" base_layer = self.context.layers[self._base_layer] @@ -287,6 +285,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): architecture = 'pc-q35' + default_suffix break if architecture: + vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}') return architecture # Still haven't found architecture, switch to fallback-method @@ -296,6 +295,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): line = base_layer.read(offset, 64) regex_results = re.search(architecture_pattern, line) architecture = "pc-" + regex_results.groups()[0].decode().lower() + default_suffix + vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}') return architecture vollog.warning("Could not determine QEMU target architecture!") From 6276b984008868161638971c8e7bf0eeddc1ea1a Mon Sep 17 00:00:00 2001 From: ikelos Date: Mon, 23 May 2022 01:48:07 +0100 Subject: [PATCH 250/404] Update volatility3/framework/layers/qemu.py Fix typo courtesy of @digitalisx Co-authored-by: Donghyun Kim --- volatility3/framework/layers/qemu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 985c4534a..907116e99 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -145,7 +145,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if size_array.get(b'pc.ram', highest_possible_maximum) < self._pci_hole_minimum: # Turns off the pci_hole if it's not supposed to be there vollog.debug( - f"QEVM tunrning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}") + f"QEVM turning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}") self._pci_hole_start, self._pci_hole_end = 0, 0 if flags & (self.SEGMENT_FLAG_COMPRESS | self.SEGMENT_FLAG_PAGE): From a3c63fbdf893324df2754b999e51db6655f3b06a Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 23 May 2022 09:25:30 +0300 Subject: [PATCH 251/404] rename variable --- volatility3/framework/layers/physical.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 5d5fd17a9..728fdcf31 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -113,14 +113,15 @@ class FileLayer(interfaces.layers.DataLayerInterface): def maximum_address(self) -> int: """Returns the largest available address in the space.""" # Zero based, so we return the size of the file minus 1 - if self._size: - return self._size + if self._maximum_address + return self._maximum_address with self._lock: orig = self._file.tell() self._file.seek(0, 2) - self._size = self._file.tell() - 1 + self._size = self._file.tell() self._file.seek(orig) - return self._size + self._maximum_address = self._size - 1 + return self._maximum_address @property def minimum_address(self) -> int: From be82c1639c051f929cbc17c6aa8e7250d6711f8b Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 23 May 2022 09:40:25 +0300 Subject: [PATCH 252/404] fix missing --- volatility3/framework/layers/physical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 728fdcf31..5d757f482 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -113,7 +113,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): def maximum_address(self) -> int: """Returns the largest available address in the space.""" # Zero based, so we return the size of the file minus 1 - if self._maximum_address + if self._maximum_address: return self._maximum_address with self._lock: orig = self._file.tell() From e15fa0ebad1a46fd990d31181d2dbe8f6b5b994d Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 23 May 2022 09:43:49 +0300 Subject: [PATCH 253/404] declared in constructor --- volatility3/framework/layers/physical.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 5d757f482..5cf0b776d 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -88,6 +88,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): self._accessor = resources.ResourceAccessor() self._file_: Optional[IO[Any]] = None self._size: Optional[int] = None + self._maximum_address: Optional[int] = None # Construct the lock now (shared if made before threading) in case we ever need it self._lock: Union[DummyLock, threading.Lock] = DummyLock() if constants.PARALLELISM == constants.Parallelism.Threading: From 946d2302bbc92e781b1281632c5ea5a669228bd0 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 23 May 2022 17:44:52 +0300 Subject: [PATCH 254/404] log resource cache usage --- volatility3/framework/layers/resources.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index ac25b5cc2..8a0e96208 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -171,6 +171,8 @@ class ResourceAccessor(object): cache_file.write(block) block = fp.read(block_size) cache_file.close() + else: + vollog.debug(f"Using already cached file at: {temp_filename}") # Re-open the cache with a different mode # Since we don't want people thinking they're able to save to the cache file, # open it in read mode only and allow breakages to happen if they wanted to write From 4dd8114dcb565daddbd105809252b5517f87a5a1 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 29 May 2022 11:28:40 +0300 Subject: [PATCH 255/404] check return value from is_valid --- .../framework/symbols/windows/extensions/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 69e8ba94e..e9264e0a0 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -746,7 +746,10 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): trans_layer = self._context.layers[layer] try: - trans_layer.is_valid(self.vol.offset) + is_valid = trans_layer.is_valid(self.vol.offset) + if not is_valid: + return + link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: return @@ -762,7 +765,9 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): obj_offset = link.vol.offset - relative_offset try: - trans_layer.is_valid(obj_offset) + is_valid = trans_layer.is_valid(obj_offset) + if not is_valid: + return except exceptions.InvalidAddressException: return From cdbe41dbf5a2a43714d3bc3579746a057055e07a Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 29 May 2022 12:20:24 +0300 Subject: [PATCH 256/404] removed redundant try catch --- .../framework/symbols/windows/extensions/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index e9264e0a0..b5ee272a0 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -764,11 +764,7 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): while link.vol.offset not in seen: obj_offset = link.vol.offset - relative_offset - try: - is_valid = trans_layer.is_valid(obj_offset) - if not is_valid: - return - except exceptions.InvalidAddressException: + if not trans_layer.is_valid(obj_offset): return obj = self._context.object(symbol_type, From b9694e109a03f9589f124cb3d88c4ec4e58cc719 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 02:48:46 +0900 Subject: [PATCH 257/404] Add: JSON EOF for config file --- volatility3/cli/__init__.py | 1 + volatility3/cli/volshell/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index e3fb726a1..8851e2b18 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -332,6 +332,7 @@ class CommandLine: parser.error(f"Cannot write configuration: file {args.save_config} already exists") with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + f.write("\n") except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 30fe75e06..769e958fd 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -246,6 +246,7 @@ class VolShell(cli.CommandLine): parser.error(f"Cannot write configuration: file {args.save_config} already exists") with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + f.write("\n") except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") From bd332261dede65118114e6484c4d8ce446d3b165 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 05:39:37 +0900 Subject: [PATCH 258/404] Fix: __del__ to __exit__ --- volatility3/framework/layers/physical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 5cf0b776d..0633637ca 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -191,7 +191,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): """Closes the file handle.""" self._file.close() - def __del__(self) -> None: + def __exit__(self) -> None: self.destroy() @classmethod From a4e162c7f1dd8597cd0b9a0a8e175573c7fb8c62 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 07:24:45 +0900 Subject: [PATCH 259/404] Fix: minor for better code --- volatility3/framework/plugins/mac/kauth_listeners.py | 2 +- volatility3/framework/plugins/windows/skeleton_key_check.py | 2 +- volatility3/framework/symbols/metadata.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/mac/kauth_listeners.py b/volatility3/framework/plugins/mac/kauth_listeners.py index 7002d88e2..fba6a8e0a 100644 --- a/volatility3/framework/plugins/mac/kauth_listeners.py +++ b/volatility3/framework/plugins/mac/kauth_listeners.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index cd4a5baec..4a1b48c9a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -9,7 +9,7 @@ # For a thorough walkthrough on how the R&D was performed to develop this plugin, # please see our blogpost here: # -# +# https://volatility-labs.blogspot.com/2021/10/memory-forensics-r-illustrated.html import io import logging diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 7cde686ee..350bb0a53 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -38,4 +38,4 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): class LinuxMetadata(interfaces.symbols.MetadataInterface): - """Class to handle the etadata from a Linux symbol table.""" + """Class to handle the metadata from a Linux symbol table.""" From 1ef8c5167722aaed65e163be3ab1d1f06c6117bb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 08:29:13 +0900 Subject: [PATCH 260/404] Fix: minor code for improve --- volatility3/framework/plugins/linux/check_syscall.py | 2 +- volatility3/framework/plugins/linux/mountinfo.py | 1 - volatility3/framework/plugins/windows/ssdt.py | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 87d252cd5..50fd05fa5 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -80,7 +80,7 @@ class Check_syscall(plugins.PluginInterface): def _get_table_info_disassembly(self, ptr_sz, vmlinux): """Find the size of the system call table by disassembling functions - that immediately reference it in their first isntruction This is in the + that immediately reference it in their first instruction This is in the form 'cmp reg,NR_syscalls'.""" table_size = 0 diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 6f3cb712d..551d128ad 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -1,7 +1,6 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -# Author: Gustavo Moreira import logging from collections import namedtuple diff --git a/volatility3/framework/plugins/windows/ssdt.py b/volatility3/framework/plugins/windows/ssdt.py index 0d921535d..78fd72630 100644 --- a/volatility3/framework/plugins/windows/ssdt.py +++ b/volatility3/framework/plugins/windows/ssdt.py @@ -95,10 +95,10 @@ class SSDT(plugins.PluginInterface): if is_kernel_64: array_subtype = "long" - def kvo_calulator(func: int) -> int: + def kvo_calculator(func: int) -> int: return kvo + service_table_address + (func >> 4) - find_address = kvo_calulator + find_address = kvo_calculator else: array_subtype = "unsigned long" From 8b128b05f834c210ce607ab40d386718b5b363b5 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 30 May 2022 22:17:33 +0900 Subject: [PATCH 261/404] Fix: typo for code comments --- volatility3/framework/objects/templates.py | 2 +- volatility3/framework/plugins/linux/psaux.py | 2 +- volatility3/framework/plugins/linux/pstree.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/objects/templates.py b/volatility3/framework/objects/templates.py index b544d117f..56754d255 100644 --- a/volatility3/framework/objects/templates.py +++ b/volatility3/framework/objects/templates.py @@ -63,7 +63,7 @@ class ObjectTemplate(interfaces.objects.Template): object_info: interfaces.objects.ObjectInformation) -> interfaces.objects.ObjectInterface: """Constructs the object. - Returns: an object adhereing to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface` + Returns: an object adhering to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface` """ arguments: Dict[str, Any] = {} for arg in self.vol: diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index c62712907..ed91c66f2 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -40,7 +40,7 @@ class PsAux(plugins.PluginInterface): name: string name of the process (from task.comm) """ - # kernel theads never have an mm as they do not have userland mappings + # kernel threads never have an mm as they do not have userland mappings try: mm = task.mm except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index a44310147..3ad5f3e19 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -19,7 +19,7 @@ class PsTree(pslist.PsList): """Finds how deep the PID is in the tasks hierarchy. Args: - pid: PID to find the level in the hierachy + pid: PID to find the level in the hierarchy """ seen = set([pid]) level = 0 From bb80d7067e99d55748e1067e6d11e22c2ffe5e4d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 31 May 2022 23:24:49 +0900 Subject: [PATCH 262/404] Fix: typo of timeliner parameter --- volatility3/framework/plugins/timeliner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 8785f62e1..c1d29062d 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -101,7 +101,7 @@ class Timeliner(interfaces.plugins.PluginInterface): return [sortable(timestamp) for timestamp in data[2:]] - def _generator(self, runable_plugins: List[TimeLinerInterface]) -> Optional[Iterable[Tuple[int, Tuple]]]: + def _generator(self, runnable_plugins: List[TimeLinerInterface]) -> Optional[Iterable[Tuple[int, Tuple]]]: """Takes a timeline, sorts it and output the data from each relevant row from each plugin.""" # Generate the results for each plugin @@ -115,9 +115,9 @@ class Timeliner(interfaces.plugins.PluginInterface): file_data = None fp = None - for plugin in runable_plugins: + for plugin in runnable_plugins: plugin_name = plugin.__class__.__name__ - self._progress_callback((runable_plugins.index(plugin) * 100) // len(runable_plugins), + self._progress_callback((runnable_plugins.index(plugin) * 100) // len(runnable_plugins), f"Running plugin {plugin_name}...") try: vollog.log(logging.INFO, f"Running {plugin_name}") From 0abd2e53abefd0856c83bfad2cf61ab500868d38 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 5 Jun 2022 10:56:42 +0100 Subject: [PATCH 263/404] Pyinstaller: Fix path need to current directory to be correct --- vol.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vol.spec b/vol.spec index 42b69af3f..666526dde 100644 --- a/vol.spec +++ b/vol.spec @@ -26,7 +26,7 @@ except ImportError: # Volatility must be findable in sys.path in order for collect_submodules to work # This adds the current working directory, which should usually do the trick -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.abspath(SPEC))) vol_analysis = Analysis(['vol.py'], pathex = [], From db3408bdaa978de5b23eecd2d4411df8a6de1f16 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 5 Jun 2022 22:23:30 +0100 Subject: [PATCH 264/404] Windows: Extend the pdb support to modules --- .../framework/symbols/windows/pdbutil.py | 75 +++++++++++++++---- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 585e96b6d..41037d464 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -10,10 +10,10 @@ import os import re import struct from typing import Any, Dict, Generator, List, Optional, Tuple, Union -from urllib import request, parse +from urllib import parse, request from volatility3 import symbols -from volatility3.framework import constants, interfaces, exceptions +from volatility3.framework import constants, contexts, exceptions, interfaces from volatility3.framework.configuration.requirements import SymbolTableRequirement from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbconv @@ -24,7 +24,7 @@ vollog = logging.getLogger(__name__) class PDBUtility(interfaces.configuration.VersionableInterface): """Class to handle and manage all getting symbols based on MZ header""" - _version = (1, 0, 0) + _version = (1, 0, 1) _required_framework_version = (2, 0, 0) @classmethod @@ -131,14 +131,14 @@ class PDBUtility(interfaces.configuration.VersionableInterface): # Check it is actually the MZ header if mz_sig != b"MZ": return None - + nt_header_start, = struct.unpack(" str: + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: """Creates symbol table for a module in the specified layer_name. Searches the memory section of the loaded module for its PDB GUID @@ -307,6 +307,19 @@ class PDBUtility(interfaces.configuration.VersionableInterface): Returns: The name of the constructed and loaded symbol table """ + _, symbol_table_name = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size) + return symbol_table_name + + @classmethod + def _modtable_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None, + create_module: bool = False) -> Tuple[Optional[str], Optional[str]]: + + if module_offset is None: + module_offset = context.layers[layer_name].minimum_address + if module_size is None: + module_size = context.layers[layer_name].maximum_address - module_offset guids = list( cls.pdbname_scan(context, @@ -323,12 +336,46 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - return cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) + module_name = guid["pdb_name"].strip('.pdb') + + symbol_table_name = cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) + + new_module_name = None + if create_module: + new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], + symbol_table_name = symbol_table_name) + new_module_name = new_module.name + + return new_module_name, symbol_table_name + + @classmethod + def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: + """Creates a module in the specified layer_name based on a pdb name. + + Searches the memory section of the loaded module for its PDB GUID + and loads the associated symbol table into the symbol space. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + config_path: The config path where to find symbol files + layer_name: The name of the layer on which to operate + module_offset: This memory dump's module image offset + module_size: The size of the module for this dump + + Returns: + The name of the constructed and loaded symbol table + """ + + module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size, create_module = True) + + return module_name class PdbSignatureScanner(interfaces.layers.ScannerInterface): From 21d916be0a08eccc91bbd4884f458ae6ff489b95 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 6 Jun 2022 14:53:52 +0100 Subject: [PATCH 265/404] Pyinstaller: Support pyinstaller 5 and later --- volatility3/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/__init__.py b/volatility3/__init__.py index db52aa9b0..b6da6e01e 100644 --- a/volatility3/__init__.py +++ b/volatility3/__init__.py @@ -37,9 +37,9 @@ class WarningFindSpec(abc.MetaPathFinder): first.""" if fullname.startswith("volatility3.framework.plugins."): warning = "Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins" - # Pyinstaller uses walk_packages to import, but needs to read the modules to figure out dependencies - # As such, we only print the warning when directly imported rather than from within walk_packages - if inspect.stack()[-2].function != 'walk_packages': + # Pyinstaller uses walk_packages/_collect_submodules to import, but needs to read the modules to figure out dependencies + # As such, we only print the warning when directly imported rather than from within walk_packages/_collect_submodules + if inspect.stack()[-2].function in ['walk_packages', '_collect_submodules']: raise Warning(warning) From aa06ed6e674761c8ec1238daeff9a64603aff392 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 12 Jun 2022 20:32:06 +0900 Subject: [PATCH 266/404] Add: new options for vol-cli.rst --- doc/source/vol-cli.rst | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 9db29c818..902787c9c 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -9,7 +9,11 @@ Synopsis **volatility** [-h] [-c CONFIG] [--parallelism [{processes,threads,off}]] [-e EXTEND] [-p PLUGIN_DIRS] [-s SYMBOL_DIRS] [-v] [-l LOG] [-o OUTPUT_DIR] [-q] [-r RENDERER] [-f FILE] - [--write-config] [--single-location SINGLE_LOCATION] + [--write-config] [--save-config SAVE_CONFIG] + [--clear-cache] [--cache-path CACHE_PATH] + [--offline] + [--single-location SINGLE_LOCATION] + [--stackers [STACKERS ...]] [--single-swap-locations SINGLE_SWAP_LOCATIONS] ... @@ -105,11 +109,31 @@ Options other plugins, but there's no guarantee that plugins use the same configuration options. +--save-config + This flag specifies that volatility should write or overwrite a file + called config.json in the current directory. The file will contain + the necessary JSON configuration to recreate the environment that the + plugin was previously run in. This configuration *may* be accepted by + other plugins, but there's no guarantee that plugins use the same + configuration options. + +--clear-cache + Clears out all short-term cached items. + +--cache-path + Change the default path ({constants.CACHE_PATH}) used to store the cache. + +--offline + Do not search online for additional JSON files. + --single-location SINGLE_LOCATION This specifies a URL which will be downloaded if necessary, and built upon by the automagic and, since most plugins require a single memory image, can be considered the input for the program. +--stackers STACKERS + + --single-swap-locations SINGLE_SWAP_LOCATIONS A comma-separated list of swap files to be considered as part of the memory image specified by the single-location or file parameters. From 2d14e4e012d6745862fafa1a857414102a412eb1 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 12 Jun 2022 20:50:03 +0900 Subject: [PATCH 267/404] Add: descriptions of new options --- doc/source/vol-cli.rst | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 902787c9c..b9e16623d 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -102,12 +102,8 @@ Options attempt to build upon, and can be considered the input for the program. --write-config - This flag specifies that volatility should write or overwrite a file - called config.json in the current directory. The file will contain - the necessary JSON configuration to recreate the environment that the - plugin was previously run in. This configuration *may* be accepted by - other plugins, but there's no guarantee that plugins use the same - configuration options. + *Deprecated* + Use of `--write-config` has been deprecated, replaced by `--save-config` --save-config This flag specifies that volatility should write or overwrite a file @@ -121,19 +117,18 @@ Options Clears out all short-term cached items. --cache-path - Change the default path ({constants.CACHE_PATH}) used to store the cache. + Change the default path used to store the cache. --offline Do not search online for additional JSON files. + Run offline mode (defaults to false) and for + remote windows symbol tables, linux/mac banner repositories. --single-location SINGLE_LOCATION This specifies a URL which will be downloaded if necessary, and built upon by the automagic and, since most plugins require a single memory image, can be considered the input for the program. ---stackers STACKERS - - --single-swap-locations SINGLE_SWAP_LOCATIONS A comma-separated list of swap files to be considered as part of the memory image specified by the single-location or file parameters. From 3ef505641eb2f7d3d76174effb18cba69434298f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 12 Jun 2022 20:55:15 +0900 Subject: [PATCH 268/404] Add: stacker descriptions --- doc/source/vol-cli.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index b9e16623d..cc6f7fe6a 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -129,6 +129,9 @@ Options upon by the automagic and, since most plugins require a single memory image, can be considered the input for the program. +--stackers STACKERS + Creates the list of stackers to use based on the config option. + --single-swap-locations SINGLE_SWAP_LOCATIONS A comma-separated list of swap files to be considered as part of the memory image specified by the single-location or file parameters. From e1f3f65202d7eb23901a4c9639ad1523f4429369 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 14 Jun 2022 21:03:33 +0900 Subject: [PATCH 269/404] Fix: typo for code comment, requirements name --- volatility3/framework/interfaces/configuration.py | 2 +- volatility3/framework/interfaces/layers.py | 4 ++-- volatility3/framework/plugins/mac/kevents.py | 2 +- volatility3/framework/plugins/windows/modscan.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index c39dba680..e271ef6d4 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -523,7 +523,7 @@ class ConstructableRequirementInterface(RequirementInterface): must happen after the class configuration value has been provided). These values are then provided to the object's constructor by name as arguments (as well as the standard `context` and `config_path` - arguments. + arguments). """ def __init__(self, *args, **kwargs) -> None: diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index a42282c39..7ff110c6e 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -307,7 +307,7 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla while length > 0: chunk_size = min(length, scanner.chunk_size + scanner.overlap) yield [(layer_name, mapped_offset, chunk_size)], offset + chunk_size - # It we've got more than the scanner's chunk_size, only move up by the chunk_size + # If we've got more than the scanner's chunk_size, only move up by the chunk_size if chunk_size > scanner.chunk_size: chunk_size -= scanner.overlap length -= chunk_size @@ -517,7 +517,7 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): yield output, chunk_position output = [] chunk_position = chunk_start - # Take from chunk_position as far as far as the block can go, + # Take from chunk_position as far as the block can go, # or as much left of a scanner chunk as we can chunk_size = min(block_end - chunk_position, scanner.chunk_size + scanner.overlap - (chunk_position - chunk_start)) diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 6f82c75cd..4a82d81cd 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -74,7 +74,7 @@ class Kevents(interfaces.plugins.PluginInterface): @classmethod def _walk_klist_array(cls, kernel, fdp, array_pointer_member, array_size_member): """ - Convience wrapper for walking an array of lists of kernel events + Convenience wrapper for walking an array of lists of kernel events Handles invalid address references """ try: diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index b661d71d7..e352c21fe 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -25,7 +25,7 @@ class ModScan(interfaces.plugins.PluginInterface): return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'poolerscanner', + requirements.VersionRequirement(name = 'poolscanner', component = poolscanner.PoolScanner, version = (1, 0, 0)), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), From dd92955a99249fe9e8863cb2754229e01a917d73 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 16 Jun 2022 05:40:26 +0900 Subject: [PATCH 270/404] Remove: unreachable code --- volatility3/cli/volshell/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 769e958fd..5eeef77cf 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -257,7 +257,6 @@ class VolShell(cli.CommandLine): constructed.run() except exceptions.VolatilityException as excp: self.process_exceptions(excp) - parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") def main(): From 9f525dfa733dd65769458540d3996917a1daaa96 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 22 Jun 2022 19:57:47 +0900 Subject: [PATCH 271/404] Refactor: simplify comparision --- volatility3/framework/automagic/pdbscan.py | 2 +- volatility3/framework/plugins/windows/ldrmodules.py | 12 ++++++------ volatility3/framework/plugins/windows/vadinfo.py | 2 +- .../framework/symbols/linux/extensions/__init__.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 5db66a3d0..cedbc4919 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -148,7 +148,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vollog.debug("Kernel base determination - optimized scan virtual layer") valid_kernel = self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, True, False, progress_callback) - if valid_kernel != None: + if valid_kernel is not None: return valid_kernel vollog.debug("Kernel base determination - slow scan virtual layer") diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index e7c96e946..284d1afc2 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -71,14 +71,14 @@ class LdrModules(interfaces.plugins.PluginInterface): mem_mod = mem_order_mod.get(base, None) yield (0, [int(proc.UniqueProcessId), - str(proc.ImageFileName.cast("string", + str(proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, errors = 'replace')), - format_hints.Hex(base), - load_mod != None, - init_mod != None, - mem_mod != None, - mapped_files[base]]) + format_hints.Hex(base), + load_mod is not None, + init_mod is not None, + mem_mod is not None, + mapped_files[base]]) def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 9fa1458d1..e357b150a 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -132,7 +132,7 @@ class VadInfo(interfaces.plugins.PluginInterface): vollog.debug("Unable to find the starting/ending VPN member") return None - if maxsize > 0 and (vad_end - vad_start) > maxsize: + if 0 < maxsize < (vad_end - vad_start): vollog.debug(f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit") return None diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6792ab19c..73f31115a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -409,7 +409,7 @@ class vm_area_struct(objects.StructType): fname = linux.LinuxUtilities.path_for_file(context, task, self.vm_file) elif self.vm_start <= task.mm.start_brk and self.vm_end >= task.mm.brk: fname = "[heap]" - elif self.vm_start <= task.mm.start_stack and self.vm_end >= task.mm.start_stack: + elif self.vm_start <= task.mm.start_stack <= self.vm_end: fname = "[stack]" elif self.vm_mm.context.has_member("vdso") and self.vm_start == self.vm_mm.context.vdso: fname = "[vdso]" From 7ba27a75ca9cbecbe796f6475340718e6bce0dd0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 22 Jun 2022 14:49:57 +0100 Subject: [PATCH 272/404] Documentation: Improve the simple-plugin example --- doc/source/simple-plugin.rst | 54 ++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 8446b0ef5..904c586c7 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -30,6 +30,9 @@ to be able to run properly. Any that are defined as optional need not necessari :: + _version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + @classmethod def get_requirements(cls): return [requirements.TranslationLayerRequirement(name = 'primary', @@ -37,13 +40,13 @@ to be able to run properly. Any that are defined as optional need not necessari architectures = ["Intel32", "Intel64"]), requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (1, 0, 0)), requirements.ListRequirement(name = 'pid', element_type = int, description = "Process IDs to include (all other processes are excluded)", - optional = True)] + optional = True), + requirements.PluginRequirement(name = 'pslist', + plugin = pslist.PsList, + version = (1, 0, 0))] This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how @@ -91,29 +94,44 @@ name of the :py:class:`SymbolTable Date: Wed, 22 Jun 2022 15:12:40 +0100 Subject: [PATCH 273/404] Documentation: Update the documentation to the latest framework --- doc/source/simple-plugin.rst | 103 ++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 904c586c7..543451b88 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -35,11 +35,8 @@ to be able to run properly. Any that are defined as optional need not necessari @classmethod def get_requirements(cls): - return [requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.ListRequirement(name = 'pid', element_type = int, description = "Process IDs to include (all other processes are excluded)", @@ -54,45 +51,73 @@ to instantiate the plugin). At the moment these requirements are fairly straigh :: - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), -This requirement indicates that the plugin will operate on a single -:py:class:`TranslationLayer `. The name of the -loaded layer will appear in the plugin's configuration under the name ``primary``. Requirement values can be -accessed within the plugin through the plugin's `config` attribute (for example ``self.config['pid']``). +This requirement specifies the need for a particular submodule. Each module requires a +:py:class:`TranslationLayer ` and a +:py:class:`SymbolTable `, which are fulfilled by two +subrequirements: a +:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` and a +:py:class:`~volatility3.framework.configuration.requirements.SymbolTableRequirement`. At the moment, the automagic +only fills `ModuleRequirements` with kernels, and so has relatively few parameters. It requires the architecture for +the underlying TranslationLayer, and the offset of the module within that layer. -.. note:: The name itself is dynamic depending on the other layers already present in the Context. Always use the value - from the configuration rather than attempting to guess what the layer will be called. +The name of the module will be stored in the ``kernel`` configuration option, and the module object itself +can be accessed from the ``context.modules`` collection. This requirement is a Complex Requirement and therefore will +not be requested directly from the user. -Finally, this defines that the translation layer must be on the Intel Architecture. At the moment, this acts as a filter, -failing to be satisfied by memory images that do not match the architecture required. -Most plugins will only operate on a single layer, but it is entirely possible for a plugin to request two different -layers, for example a plugin that carries out some form of difference or statistics against multiple memory images. +.. note:: -This requirement (and the next two) are known as Complex Requirements, and user interfaces will likely not directly -request a value for this from a user. The value stored in the configuration tree for a -:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` is -the string name of a layer present in the context's memory that satisfies the requirement. + In previous versions of volatility 3, there was no `ModuleRequirement`, and instead two requirements were defined + a :py:class:`TranslationLayer ` and a `SymbolTableRequirement`. These still exist, and can be used, most plugins just + define a single `ModuleRequirement` for the kernel, which the automagic will populate. The `ModuleRequirement` has + two automatic sub-requirements, a `TranslationLayerRequirement` and a `SymbolTableRequirement`, but the module also + includes the offset of the module, and will allow future expansion to specify specific modules when application + level plugins become more common. Below are how the requirements would be specified: -:: + :: - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), -This requirement specifies the need for a particular -:py:class:`SymbolTable ` -to be loaded. This gets populated by various -:py:class:`Automagic ` as the nearest sibling to a particular -:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`. -This means that if the :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` -is satisfied and the :py:class:`Automagic ` can determine -the appropriate :py:class:`SymbolTable `, the -name of the :py:class:`SymbolTable ` will be stored in the configuration. + This requirement indicates that the plugin will operate on a single + :py:class:`TranslationLayer `. The name of the + loaded layer will appear in the plugin's configuration under the name ``primary``. Requirement values can be + accessed within the plugin through the plugin's `config` attribute (for example ``self.config['pid']``). -This requirement is also a Complex Requirement and therefore will not be requested directly from the user. + .. note:: The name itself is dynamic depending on the other layers already present in the Context. Always use the value + from the configuration rather than attempting to guess what the layer will be called. + + Finally, this defines that the translation layer must be on the Intel Architecture. At the moment, this acts as a filter, + failing to be satisfied by memory images that do not match the architecture required. + + Most plugins will only operate on a single layer, but it is entirely possible for a plugin to request two different + layers, for example a plugin that carries out some form of difference or statistics against multiple memory images. + + This requirement (and the next two) are known as Complex Requirements, and user interfaces will likely not directly + request a value for this from a user. The value stored in the configuration tree for a + :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` is + the string name of a layer present in the context's memory that satisfies the requirement. + + :: + + requirements.SymbolTableRequirement(name = "nt_symbols", + description = "Windows kernel symbols"), + + This requirement specifies the need for a particular + :py:class:`SymbolTable ` + to be loaded. This gets populated by various + :py:class:`Automagic ` as the nearest sibling to a particular + :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`. + This means that if the :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` + is satisfied and the :py:class:`Automagic ` can determine + the appropriate :py:class:`SymbolTable `, the + name of the :py:class:`SymbolTable ` will be stored in the configuration. + + This requirement is also a Complex Requirement and therefore will not be requested directly from the user. :: @@ -147,6 +172,7 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] return renderers.TreeGrid([("PID", int), ("Process", str), @@ -155,8 +181,8 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. ("Name", str), ("Path", str)], self._generator(pslist.PsList.list_processes(self.context, - self.config['primary'], - self.config['nt_symbols'], + kernel.layer_name, + kernel.symbol_table_name, filter_func = filter_func))) In this instance, the plugin constructs a filter (using the PsList plugin's *classmethod* for creating filters). @@ -175,7 +201,8 @@ the :py:class:`~volatility3.plugins.windows.pslist.PsList` plugin. That plugin so that other plugins can call it. As such, it takes all the necessary parameters rather than accessing them from a configuration. Since it must be portable code, it takes a context, as well as the layer name, symbol table and optionally a filter. In this instance we unconditionally -pass it the values from the configuration for the ``primary`` and ``nt_symbols`` requirements. This will generate a list +pass it the values from the configuration for the layer and symbol table from the kernel module object, constructed from +the ``kernel`` configuration requirement. This will generate a list of :py:class:`~volatility3.framework.symbols.windows.extensions.EPROCESS` objects, as provided by the :py:class:`~volatility.plugins.windows.pslist.PsList` plugin, and is not covered here but is used as an example for how to share code across plugins (both as the provider and the consumer of the shared code). From fd524a6b314750bd07779f65257d010ed635b1f6 Mon Sep 17 00:00:00 2001 From: ikelos Date: Wed, 22 Jun 2022 17:08:18 +0100 Subject: [PATCH 274/404] Update doc/source/simple-plugin.rst Yep, that seems fine. Co-authored-by: Donghyun Kim --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 543451b88..d03f7c7d6 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -43,7 +43,7 @@ to be able to run properly. Any that are defined as optional need not necessari optional = True), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, - version = (1, 0, 0))] + version = (2, 0, 0))] This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how From a386de72f5a22d176ecad730e83f804e2f62c633 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 22 Jun 2022 17:12:24 +0100 Subject: [PATCH 275/404] Documentation: Fix pslist plugin requirement --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index d03f7c7d6..1c7b91205 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -134,7 +134,7 @@ being defined within the configuration tree at all. requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, - version = (1, 0, 0)) + version = (2, 0, 0))] This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements on that plugin. The version is specified in terms of Semantic Versioning meaning that, to be compatible, the major From aed87346cdd362fb59fce772cbd62b0dded51bf5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 23 Jun 2022 09:23:31 +0100 Subject: [PATCH 276/404] Core: Add support to templates to get child templates --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/interfaces/objects.py | 11 +++++++++++ volatility3/framework/objects/__init__.py | 17 +++++++++++++++++ volatility3/framework/objects/templates.py | 7 +++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 472a743e6..f08819f29 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 2 # Number of changes that only add to the interface +VERSION_MINOR = 3 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 2240c58c9..3cc23e759 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -241,6 +241,13 @@ class ObjectInterface(metaclass = abc.ABCMeta): the child member.""" raise KeyError(f"Template does not contain any children: {template.vol.type_name}") + @classmethod + @abc.abstractmethod + def child_template(cls, template: 'Template', child: str) -> interfaces.objects.Template: + """Returns the template of the child member from the parent.""" + raise KeyError(f"Template does not contain any children: {template.vol.type_name}") + + @classmethod @abc.abstractmethod def has_member(cls, template: 'Template', member_name: str) -> bool: @@ -305,6 +312,10 @@ class Template: """Returns the relative offset of the `child` member from its parent offset.""" + @abc.abstractmethod + def child_template(self, child: str) -> interfaces.objects.Template: + """Returns the `child` member template from its parent.""" + @abc.abstractmethod def replace_child(self, old_child: 'Template', new_child: 'Template') -> None: """Replaces `old_child` with `new_child` in the list of children.""" diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index e0f927ec9..feb49a089 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -602,6 +602,14 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): return 0 raise IndexError(f"Member not present in array template: {child}") + @classmethod + def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: + """Returns the template of the child member.""" + if 'subtype' in template.vol and child == 'subtype'@ + return template.vol.subtype + raise IndexError(f"Member not present in array template: {child}") + + @overload def __getitem__(self, i: int) -> interfaces.objects.Template: ... @@ -715,6 +723,15 @@ class AggregateType(interfaces.objects.ObjectInterface): raise IndexError(f"Member not present in template: {child}") return retlist[0] + @classmethod + def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: + """Returns the template of a child to its parent.""" + retlist = template.vol.members.get(child, None) + if retlist is None: + raise IndexError(f"Member not present in template: {child}") + return retlist[1] + + @classmethod def has_member(cls, template: interfaces.objects.Template, member_name: str) -> bool: """Returns whether the object would contain a member called diff --git a/volatility3/framework/objects/templates.py b/volatility3/framework/objects/templates.py index 56754d255..e8b523373 100644 --- a/volatility3/framework/objects/templates.py +++ b/volatility3/framework/objects/templates.py @@ -48,6 +48,12 @@ class ObjectTemplate(interfaces.objects.Template): plateProxy`)""" return self.vol.object_class.VolTemplateProxy.relative_child_offset(self, child) + def child_template(self, child: str) -> interfaces.objects.Template: + """Returns the template of a child of the templated object (see + :class:`~volatility3.framework.interfaces.objects.ObjectInterface.VolTem + plateProxy`)""" + return self.vol.object_class.VolTemplateProxy.child_template(self, child) + def replace_child(self, old_child: interfaces.objects.Template, new_child: interfaces.objects.Template) -> None: """Replaces `old_child` for `new_child` in the templated object's child list (see :class:`~volatility3.framework.interfaces.objects.ObjectInterf @@ -99,6 +105,7 @@ class ReferenceTemplate(interfaces.objects.Template): size: ClassVar[Any] = property(_unresolved) replace_child: ClassVar[Any] = _unresolved relative_child_offset: ClassVar[Any] = _unresolved + child_template: ClassVar[Any] = _unresolved has_member: ClassVar[Any] = _unresolved def __call__(self, context: interfaces.context.ContextInterface, object_info: interfaces.objects.ObjectInformation): From 6982650c188a7c8fccbbee3c1d7d1f47f309df28 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Jun 2022 15:38:48 +0100 Subject: [PATCH 277/404] Volshell: Fixes use of old config variables Closes #780 --- volatility3/cli/volshell/linux.py | 4 ++-- volatility3/cli/volshell/mac.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 97a488743..0f2a90c7e 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -56,13 +56,13 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['vmlinux'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: - symbol_table = self.config['vmlinux'] + symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) @property diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 305f80505..6744f3394 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -56,7 +56,7 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['darwin'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): From e8e6bacb194933de3402a182ffc3dd070256e32b Mon Sep 17 00:00:00 2001 From: ikelos Date: Thu, 30 Jun 2022 09:57:09 +0100 Subject: [PATCH 278/404] Update volatility3/framework/objects/__init__.py Fix typo Co-authored-by: Donghyun Kim --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index feb49a089..62e6de553 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -605,7 +605,7 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): @classmethod def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: """Returns the template of the child member.""" - if 'subtype' in template.vol and child == 'subtype'@ + if 'subtype' in template.vol and child == 'subtype': return template.vol.subtype raise IndexError(f"Member not present in array template: {child}") From 951a0f5d508b8db4985d54751ea93ca78e57b191 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 30 Jun 2022 11:43:38 +0100 Subject: [PATCH 279/404] Documentation: Clarify that the code is just an example Clarifies for #773 and #776 --- doc/source/simple-plugin.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 1c7b91205..e2143f1b7 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -6,6 +6,12 @@ This guide will step through how to construct a simple plugin using Volatility 3 The example plugin we'll use is :py:class:`~volatility3.plugins.windows.dlllist.DllList`, which features the main traits of a normal plugin, and reuses other plugins appropriately. +.. note:: + + This document will not include the complete code necessary for a + working plugin (such as imports, etc) since it's designed to focus on the necessary componets for writing a plugin. + For complete and functioning plugins, the ``framework/plugins`` directory should be consulted. + Inherit from PluginInterface ---------------------------- From 6d7095fa3bf01aa4f2a9fceb1887cf28ed463e58 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 2 Jul 2022 19:30:00 +0900 Subject: [PATCH 280/404] Add: exceptions code --- .../plugins/windows/registry/certificates.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index d2fb61f02..e2fe662fc 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -17,10 +17,8 @@ class Certificates(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)), requirements.BooleanRequirement(name = 'dump', @@ -58,10 +56,12 @@ class Certificates(interfaces.plugins.PluginInterface): def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: + kernel = self.context.modules[self.config['kernel']] + for hive in hivelist.HiveList.list_hives(self.context, base_config_path = self.config_path, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols']): + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name): for top_key in [ "Microsoft\\SystemCertificates", @@ -87,6 +87,12 @@ class Certificates(interfaces.plugins.PluginInterface): # Key wasn't found in this hive, carry on vollog.log(constants.LOGLEVEL_VVVV, "Key wasn't found in this hive") pass + except exceptions.SwappedInvalidAddressException as exp: + vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)") + pass + except exceptions.PagedInvalidAddressException as exp: + vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)") + pass def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), From c40aecdfdacfde5ac17b658b581aa85595272b85 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 2 Jul 2022 19:38:12 +0900 Subject: [PATCH 281/404] Remove: invalid exceptions code --- volatility3/plugins/windows/registry/certificates.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index e2fe662fc..e873fd1d6 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -90,9 +90,6 @@ class Certificates(interfaces.plugins.PluginInterface): except exceptions.SwappedInvalidAddressException as exp: vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)") pass - except exceptions.PagedInvalidAddressException as exp: - vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)") - pass def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), From 69c50e3c6511bf8dc5dbbd80d908ab82f1b926ff Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 3 Jul 2022 19:57:16 +0530 Subject: [PATCH 282/404] last command added to example 1 --- doc/source/Windows.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index b086cc57b..3e6844f22 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -84,6 +84,22 @@ In windows memory forensics using volatility3, most of the times we do not requi ``windows.pstree`` helps us to display the parent child relation of processes. +.. code-block:: shell-session + + $ python3 vol.py -f MemDump.DMP windows.hashdump + Volatility 3 Framework 2.0.3 + Progress: 100.00 PDB scanning finished + User rid lmhash nthash + + Administrator 500 aad3b435b51404eeaad3b435b51404ee 31d6cfe0d16ae931b73c59d7e0c089c0 + Guest 501 aad3b435b51404eeaad3b435b51404ee 31d6cfe0d16ae931b73c59d7e0c089c0 + Frank Reynolds 1000 aad3b435b51404eeaad3b435b51404ee a88d1e18706d3aa676e01e5943d15911 + HomeGroupUser$ 1002 aad3b435b51404eeaad3b435b51404ee af10ecac6ea817d2bb56e3e5c33ce1cd + Dennis 1003 aad3b435b51404eeaad3b435b51404ee cf96684bbc7877920adaa9663698bf54 + +``windows.hashdump`` helps us to list the hashes of the users in the system. + + From dcc774787cf333ac574ae1dd402f752616e946a4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Jul 2022 22:11:24 +0100 Subject: [PATCH 283/404] Core: Convert try/except/pass to contextlib.supress --- volatility3/framework/automagic/pdbscan.py | 13 ++++---- volatility3/framework/interfaces/objects.py | 5 ++- volatility3/framework/layers/crash.py | 5 ++- volatility3/framework/layers/registry.py | 21 ++++++------ volatility3/framework/layers/resources.py | 4 +-- volatility3/framework/layers/vmware.py | 26 +++++++-------- .../framework/plugins/linux/check_syscall.py | 8 ++--- .../framework/plugins/windows/dlllist.py | 13 ++++---- .../framework/plugins/windows/envars.py | 21 ++++-------- .../framework/plugins/windows/mftscan.py | 7 ++-- .../plugins/windows/registry/userassist.py | 18 ++++------- volatility3/framework/renderers/conversion.py | 6 ++-- .../symbols/mac/extensions/__init__.py | 32 +++++++------------ .../framework/symbols/windows/__init__.py | 14 ++++---- .../symbols/windows/extensions/__init__.py | 21 ++++-------- .../symbols/windows/extensions/pool.py | 28 +++++++--------- .../symbols/windows/extensions/registry.py | 6 ++-- 17 files changed, 97 insertions(+), 151 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index cedbc4919..5cbdbfe0e 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -7,10 +7,11 @@ from loaded PE files. This module contains a standalone scanner, and also a :class:`~volatility3.framework.interfaces.layers.ScannerInterface` based scanner for use within the framework by calling :func:`~volatility3.framework.interfaces.layers.DataLayerInterface.scan`. """ +import contextlib import logging import math import os -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union, Callable +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.configuration import requirements @@ -139,7 +140,8 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]: + def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ + ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): # Rule out kernels that couldn't find a suitable MZ header @@ -159,7 +161,8 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - def test_physical_kernel(physical_layer_name:str , virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]: + def test_physical_kernel(physical_layer_name: str, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ + ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): # Rule out kernels that couldn't find a suitable MZ header @@ -274,7 +277,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] virtual_layer_name = vlayer.name - try: + with contextlib.suppress(exceptions.InvalidAddressException): if vlayer.read(address, 0x2) == b'MZ': res = list( PDBUtility.pdbname_scan(ctx = context, @@ -286,8 +289,6 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): end = address + self.max_pdb_size)) if res: valid_kernel = (virtual_layer_name, address, res[0]) - except exceptions.InvalidAddressException: - pass return valid_kernel # List of methods to be run, in order, to determine the valid kernels diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 2240c58c9..0f8e742fb 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -6,6 +6,7 @@ interpreted values of data from a layer.""" import abc import collections import collections.abc +import contextlib import logging from typing import Any, Dict, List, Mapping, Optional @@ -187,11 +188,9 @@ class ObjectInterface(metaclass = abc.ABCMeta): """ if self.has_member(member_name): # noinspection PyBroadException - try: + with contextlib.suppress(Exception): _ = getattr(self, member_name) return True - except Exception: - pass return False def has_valid_members(self, member_names: List[str]) -> bool: diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index c690c8d8f..6194501ee 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -1,6 +1,7 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib import logging import struct from typing import Tuple, Optional @@ -202,11 +203,9 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: for layer in [WindowsCrashDump32Layer, WindowsCrashDump64Layer]: - try: + with contextlib.suppress(WindowsCrashDumpFormatException): layer.check_header(context.layers[layer_name]) new_name = context.layers.free_layer_name(layer.__name__) context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name return layer(context, new_name, new_name) - except WindowsCrashDumpFormatException: - pass return None diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 55a6e5186..ec7aed217 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import logging from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union @@ -92,11 +92,9 @@ class RegistryHive(linear.LinearlyMappedLayer): @property def root_cell_offset(self) -> int: """Returns the offset for the root cell in this hive.""" - try: + with contextlib.suppress(InvalidAddressException): if self._base_block.Signature.cast("string", max_length = 4, encoding = "latin-1") == 'regf': return self._base_block.RootCell - except InvalidAddressException: - pass return 0x20 def get_cell(self, cell_offset: int) -> 'objects.StructType': @@ -201,11 +199,11 @@ class RegistryHive(linear.LinearlyMappedLayer): if offset & 0x7fffffff > self._get_hive_maxaddr(volatile): vollog.log(constants.LOGLEVEL_VVV, "Layer {} couldn't translate offset {}, greater than {} in {} store of {}".format( - self.name, - hex(offset & 0x7fffffff), - hex(self._get_hive_maxaddr(volatile)), - "volative" if volatile else "non-volatile", - self.get_name())) + self.name, + hex(offset & 0x7fffffff), + hex(self._get_hive_maxaddr(volatile)), + "volative" if volatile else "non-volatile", + self.get_name())) raise RegistryInvalidIndex(self.name, "Mapping request for value greater than maxaddr") storage = self.hive.Storage[volatile] @@ -252,14 +250,13 @@ class RegistryHive(linear.LinearlyMappedLayer): def is_valid(self, offset: int, length: int = 1) -> bool: """Returns a boolean based on whether the offset is valid or not.""" - try: + with contextlib.suppress(exceptions.InvalidAddressException): # Pass this to the lower layers for now return all([ self.context.layers[layer].is_valid(offset, length) for (_, _, offset, length, layer) in self.mapping(offset, length) ]) - except exceptions.InvalidAddressException: - return False + return False @property def minimum_address(self) -> int: diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 8a0e96208..dca215c85 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -184,14 +184,12 @@ class ResourceAccessor(object): stop = False while not stop: detected = None - try: + with contextlib.suppress(AttributeError, IOError): # Detect the content detected = magic.detect_from_fobj(curfile) IMPORTED_MAGIC = True # This is because python-magic and file provide a magic module # Only file's python has magic.detect_from_fobj - except (AttributeError, IOError): - pass if detected: if detected.mime_type == 'application/x-xz': diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 85e961b24..ae4a7d55e 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -1,14 +1,14 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import logging import struct from typing import Any, Dict, List, Optional -from volatility3.framework import interfaces, constants, exceptions +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.layers import physical, segmented, resources +from volatility3.framework.layers import physical, resources, segmented from volatility3.framework.symbols import native vollog = logging.getLogger(__name__) @@ -87,13 +87,13 @@ class VmwareLayer(segmented.SegmentedLayer): offset = offset + name_len + 2 + (index * index_len), layer_name = self._meta_layer)) data_len = flags & 0x3f - + if data_len in [62, 63]: # Handle special data sizes that indicate a longer data stream data_len = 4 if version == 0 else 8 # Read the size of the data data_size = self._context.object(self._choose_type(data_len), - layer_name = self._meta_layer, - offset = offset + 2 + name_len + (indices_len * index_len)) + layer_name = self._meta_layer, + offset = offset + 2 + name_len + (indices_len * index_len)) # Skip two bytes of padding (as it seems?) # Read the actual data data = self._context.object("vmware!bytes", @@ -113,9 +113,9 @@ class VmwareLayer(segmented.SegmentedLayer): if tags[("regionsCount", ())][1] == 0: raise VmwareFormatException(self.name, "VMware VMEM is not split into regions") for region in range(tags[("regionsCount", ())][1]): - offset = tags[("regionPPN", (region, ))][1] * self._page_size - mapped_offset = tags[("regionPageNum", (region, ))][1] * self._page_size - length = tags[("regionSize", (region, ))][1] * self._page_size + offset = tags[("regionPPN", (region,))][1] * self._page_size + mapped_offset = tags[("regionPageNum", (region,))][1] * self._page_size + length = tags[("regionSize", (region,))][1] * self._page_size self._segments.append((offset, mapped_offset, length, length)) @property @@ -153,23 +153,19 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): current_layer_name) vmss_success = False - try: + with contextlib.suppress(IOError): _ = resources.ResourceAccessor().open(vmss).read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmss_success = True - except IOError: - pass vmsn_success = False if not vmss_success: - try: + with contextlib.suppress(IOError): _ = resources.ResourceAccessor().open(vmsn).read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmsn context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmsn_success = True - except IOError: - pass vollog.log(constants.LOGLEVEL_VVVV, f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})") diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 50fd05fa5..6ec5fd354 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -3,11 +3,11 @@ # """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" +import contextlib import logging from typing import List -from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers, constants +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.renderers import format_hints @@ -40,11 +40,9 @@ class Check_syscall(plugins.PluginInterface): symbol_list = [] for sn in vmlinux.symbols: - try: + with contextlib.suppress(exceptions.SymbolError): # When requesting the symbol from the module, a full resolve is performed symbol_list.append((vmlinux.get_symbol(sn).address, sn)) - except exceptions.SymbolError: - pass sorted_symbols = sorted(symbol_list) sym_address = 0 diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 2fd7deeaf..cb7626dfa 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -1,18 +1,19 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib import datetime import logging import ntpath from typing import List, Optional, Type -from volatility3.framework import exceptions, renderers, interfaces, constants +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins import timeliner -from volatility3.plugins.windows import pslist, info +from volatility3.plugins.windows import info, pslist vollog = logging.getLogger(__name__) @@ -28,7 +29,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), requirements.ListRequirement(name = 'pid', @@ -107,12 +108,10 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for entry in proc.load_order_modules(): BaseDllName = FullDllName = renderers.UnreadableValue() - try: + with contextlib.suppress(exceptions.InvalidAddressException): BaseDllName = entry.BaseDllName.get_string() # We assume that if the BaseDllName points to an invalid buffer, so will FullDllName FullDllName = entry.FullDllName.get_string() - except exceptions.InvalidAddressException: - pass if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index 9791fa580..e9015280a 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -1,9 +1,10 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +import contextlib import logging from typing import List -from volatility3.framework import renderers, interfaces, objects, exceptions, constants +from volatility3.framework import constants, exceptions, interfaces, objects, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.plugins.windows import pslist @@ -23,7 +24,7 @@ class Envars(interfaces.plugins.PluginInterface): # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -61,13 +62,11 @@ class Envars(interfaces.plugins.PluginInterface): key = hive.get_key('CurrentControlSet\\Control\\Session Manager\\Environment') sys = True except KeyError: - try: + with contextlib.suppress(KeyError): key = hive.get_key('ControlSet001\\Control\\Session Manager\\Environment') sys = True - except KeyError: - pass if sys: - try: + with contextlib.suppress(KeyError): for node in key.get_values(): try: value_node_name = node.get_name() @@ -78,17 +77,13 @@ class Envars(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVV, "Error while parsing global environment variables keys (some keys might be excluded)") continue - except KeyError: - pass ## The user-specific variables - try: + with contextlib.suppress(KeyError): key = hive.get_key('Environment') ntuser = True - except KeyError: - pass if ntuser: - try: + with contextlib.suppress(KeyError): for node in key.get_values(): try: value_node_name = node.get_name() @@ -99,8 +94,6 @@ class Envars(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVV, "Error while parsing user environment variables keys (some keys might be excluded)") continue - except KeyError: - pass ## The volatile user variables try: diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 654e26db7..c96fd9522 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -1,7 +1,7 @@ # This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import datetime import logging @@ -56,7 +56,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): - try: + with contextlib.suppress(exceptions.PagedInvalidAddressException): mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset @@ -131,9 +131,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) - except exceptions.PagedInvalidAddressException: - pass - def generate_timeline(self): for row in self._generator(): _depth, row_data = row diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index a788f058f..30b5db695 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -3,17 +3,18 @@ # import codecs +import contextlib import datetime import json import logging import os -from typing import Any, List, Tuple, Generator +from typing import Any, Generator, List, Tuple -from volatility3.framework import exceptions, renderers, constants, interfaces +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers.physical import BufferDataLayer from volatility3.framework.layers.registry import RegistryHive -from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.plugins.windows.registry import hivelist @@ -38,7 +39,7 @@ class UserAssist(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.IntRequirement(name = 'offset', description = "Hive Offset", default = None, optional = True), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) ] @@ -126,11 +127,9 @@ class UserAssist(interfaces.plugins.PluginInterface): hive_name = hive.hive.cast(kernel.symbol_table_name + constants.BANG + "_CMHIVE").get_name() if self._win7 is None: - try: + with contextlib.suppress(exceptions.SymbolError): self._win7 = self._win7_or_later() - except exceptions.SymbolError: # self._win7 will be None and only registry value rawdata will be output - pass self._determine_userassist_type() @@ -163,7 +162,6 @@ class UserAssist(interfaces.plugins.PluginInterface): # output any subkeys under Count for subkey in countkey.get_subkeys(): - subkey_name = subkey.get_name() result = (1, ( renderers.format_hints.Hex(hive.hive_offset), @@ -185,10 +183,8 @@ class UserAssist(interfaces.plugins.PluginInterface): for value in countkey.get_values(): value_name = value.get_name() - try: + with contextlib.suppress(UnicodeDecodeError): value_name = codecs.encode(value_name, "rot_13") - except UnicodeDecodeError: - pass if self._win7: guid = value_name.split("\\")[0] diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index 996cf03a5..3ce49bbde 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import datetime import ipaddress import socket @@ -27,10 +27,8 @@ def unixtime_to_datetime(unixtime: int) -> Union[interfaces.renderers.BaseAbsent ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = renderers.UnparsableValue() if unixtime > 0: - try: + with contextlib.suppress(ValueError): ret = datetime.datetime.utcfromtimestamp(unixtime) - except ValueError: - pass return ret diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 94045d2e7..a66bfb534 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -1,19 +1,18 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib +import logging from typing import Generator, Iterable, Optional, Set, Tuple -import logging - -from volatility3.framework import constants, objects, renderers -from volatility3.framework import exceptions, interfaces +from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic vollog = logging.getLogger(__name__) + class proc(generic.GenericIntelProcess): def get_task(self): @@ -29,10 +28,8 @@ class proc(generic.GenericIntelProcess): if not isinstance(parent_layer, interfaces.layers.TranslationLayerInterface): raise TypeError("Parent layer is not a translation layer, unable to construct process layer") - try: + with contextlib.suppress(exceptions.InvalidAddressException): dtb = self.get_task().map.pmap.pm_cr3 - except exceptions.InvalidAddressException: - return None if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.p_pid}" @@ -41,10 +38,8 @@ class proc(generic.GenericIntelProcess): return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - try: + with contextlib.suppress(exceptions.InvalidAddressException): task = self.get_task() - except exceptions.InvalidAddressException: - return try: current_map = task.map.hdr.links.next @@ -55,9 +50,9 @@ class proc(generic.GenericIntelProcess): for i in range(task.map.hdr.nentries): if (not current_map or - current_map.vol.offset in seen or - not self._context.layers[task.vol.native_layer_name].is_valid(current_map.dereference().vol.offset, current_map.dereference().vol.size)): - + current_map.vol.offset in seen or + not self._context.layers[task.vol.native_layer_name].is_valid(current_map.dereference().vol.offset, + current_map.dereference().vol.size)): vollog.log(constants.LOGLEVEL_VVV, "Breaking process maps iteration due to invalid state.") break @@ -102,10 +97,8 @@ class fileglob(objects.StructType): if self.has_member("fg_type"): ret = self.fg_type elif self.fg_ops != 0: - try: + with contextlib.suppress(exceptions.InvalidAddressException): ret = self.fg_ops.fo_type - except exceptions.InvalidAddressException: - pass if ret: ret = str(ret.description).replace("DTYPE_", "") @@ -456,7 +449,7 @@ class queue_entry(objects.StructType): seen = set() for attr in ['next', 'prev']: - try: + with contextlib.suppress(exceptions.InvalidAddressException): n = getattr(self, attr).dereference().cast(type_name) while n is not None and n.vol.offset != list_head: @@ -473,9 +466,6 @@ class queue_entry(objects.StructType): n = getattr(n.member(attr = member_name), attr).dereference().cast(type_name) - except exceptions.InvalidAddressException: - pass - class ifnet(objects.StructType): diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index 899b89dc2..cfac87e2c 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -1,10 +1,11 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import extensions -from volatility3.framework.symbols.windows.extensions import registry, pool, pe +from volatility3.framework.symbols.windows.extensions import pe, pool, registry class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -39,26 +40,23 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('_VACB', extensions.VACB) self.set_type_class('_POOL_TRACKER_BIG_PAGES', pool.POOL_TRACKER_BIG_PAGES) self.set_type_class('_IMAGE_DOS_HEADER', pe.IMAGE_DOS_HEADER) - + # Might not necessarily defined in every version of windows self.optional_set_type_class('_IMAGE_NT_HEADERS', pe.IMAGE_NT_HEADERS) self.optional_set_type_class('_IMAGE_NT_HEADERS64', pe.IMAGE_NT_HEADERS) # This doesn't exist in very specific versions of windows - try: + with contextlib.suppress(ValueError): if self.get_type("_POOL_TRACKER_BIG_PAGES").has_member("PoolType"): self.set_type_class('_POOL_HEADER', pool.POOL_HEADER_VISTA) else: self.set_type_class('_POOL_HEADER', pool.POOL_HEADER) - except ValueError: - pass # these don't exist in windows XP self.optional_set_type_class('_MMADDRESS_NODE', extensions.MMVAD_SHORT) - + # these were introduced starting in windows 8 self.optional_set_type_class('_MM_AVL_NODE', extensions.MMVAD_SHORT) - + # these were introduced starting in windows 7 self.optional_set_type_class('_RTL_BALANCED_NODE', extensions.MMVAD_SHORT) - \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b5ee272a0..7be9c4791 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -3,6 +3,7 @@ # import collections.abc +import contextlib import datetime import functools import logging @@ -305,7 +306,7 @@ class MMVAD(MMVAD_SHORT): file_name = renderers.NotApplicableValue() - try: + with contextlib.suppress(exceptions.InvalidAddressException): # this is for xp and 2003 if self.has_member("ControlArea"): filename_obj = self.ControlArea.FilePointer.FileName @@ -318,9 +319,6 @@ class MMVAD(MMVAD_SHORT): if filename_obj.Length > 0: file_name = filename_obj.get_string() - except exceptions.InvalidAddressException: - pass - return file_name @@ -364,6 +362,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): yield device device = device.AttachedDevice.dereference() + class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" @@ -374,7 +373,7 @@ class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" - device = self.DeviceObject.dereference() + device = self.DeviceObject.dereference() while device: yield device device = device.NextDevice.dereference() @@ -413,15 +412,11 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. if self._context.layers[self.vol.native_layer_name].is_valid(self.DeviceObject): - try: + with contextlib.suppress(ValueError): name = f"\\Device\\{self.DeviceObject.get_device_name()}" - except ValueError: - pass - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): name += self.FileName.String - except (TypeError, exceptions.InvalidAddressException): - pass return name @@ -1114,12 +1109,10 @@ class SHARED_CACHE_MAP(objects.StructType): iterval = 0 while (iterval < full_blocks) and (full_blocks <= 4): vacb_obj = self.InitialVacbs[iterval] - try: + with contextlib.suppress(exceptions.InvalidAddressException): # Make sure that the SharedCacheMap member of the VACB points back to the parent object. if vacb_obj.SharedCacheMap == self.vol.offset: self.save_vacb(vacb_obj, vacb_list) - except exceptions.InvalidAddressException: - pass iterval += 1 # We also have to account for the spill over data that is not found in the full blocks. diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index 368765497..79ea60027 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -1,12 +1,14 @@ +import contextlib import functools import logging import struct -from typing import Optional, Tuple, List, Dict, Union +from typing import Dict, List, Optional, Tuple, Union -from volatility3.framework import objects, interfaces, constants, symbols, exceptions, renderers -from volatility3.framework.renderers import conversion from volatility3.plugins.windows.poolscanner import PoolConstraint +from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols +from volatility3.framework.renderers import conversion + vollog = logging.getLogger(__name__) @@ -138,7 +140,7 @@ class POOL_HEADER(objects.StructType): if addr - optional_headers_length >= padding_length > addr: continue - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): mem_object = self._context.object(symbol_table_name + constants.BANG + type_name, layer_name = self.vol.layer_name, offset = addr + body_offset + start_offset, @@ -147,15 +149,13 @@ class POOL_HEADER(objects.StructType): if mem_object.is_valid(): yield mem_object - except (TypeError, exceptions.InvalidAddressException): - pass - # use the bottom up approach for windows 7 and earlier else: type_size = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + type_name).size if constraint.additional_structures: for additional_structure in constraint.additional_structures: - type_size += self._context.symbol_space.get_type(symbol_table_name + constants.BANG + additional_structure).size + type_size += self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + additional_structure).size rounded_size = conversion.round(type_size, alignment, up = True) @@ -164,11 +164,9 @@ class POOL_HEADER(objects.StructType): offset = self.vol.offset + self.BlockSize * alignment - rounded_size, native_layer_name = native_layer_name) - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): if mem_object.is_valid(): yield mem_object - except (TypeError, exceptions.InvalidAddressException): - pass @classmethod @functools.lru_cache() @@ -177,20 +175,18 @@ class POOL_HEADER(objects.StructType): headers = [] sizes = [] for header in [ - 'CREATOR_INFO', 'NAME_INFO', 'HANDLE_INFO', 'QUOTA_INFO', 'PROCESS_INFO', 'AUDIT_INFO', 'EXTENDED_INFO', - 'HANDLE_REVOCATION_INFO', 'PADDING_INFO' + 'CREATOR_INFO', 'NAME_INFO', 'HANDLE_INFO', 'QUOTA_INFO', 'PROCESS_INFO', 'AUDIT_INFO', 'EXTENDED_INFO', + 'HANDLE_REVOCATION_INFO', 'PADDING_INFO' ]: - try: + with contextlib.suppress(AttributeError, exceptions.SymbolError): type_name = f"{symbol_table_name}{constants.BANG}_OBJECT_HEADER_{header}" header_type = context.symbol_space.get_type(type_name) headers.append(header) sizes.append(header_type.size) - except (AttributeError, exceptions.SymbolError): # Some of these may not exist, for example: # if build < 9200: PADDING_INFO else: AUDIT_INFO # if build == 10586: HANDLE_REVOCATION_INFO else EXTENDED_INFO # based on what's present and what's not, this list should be the right order and the right length - pass return headers, sizes def is_free_pool(self): diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 47ff24506..c71fcf49b 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import enum import logging import struct @@ -75,12 +75,10 @@ class CMHIVE(objects.StructType): """ for attr in ["FileFullPath", "FileUserName", "HiveRootPath"]: - try: + with contextlib.suppress(AttributeError, exceptions.InvalidAddressException): name = getattr(self, attr) if name.Length > 0: return name.get_string() - except (AttributeError, exceptions.InvalidAddressException): - pass return None From 3679134f01abcd901f430be1292d99de21c093fc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Jul 2022 22:45:11 +0100 Subject: [PATCH 284/404] Core: Prevent circular dependency on imports --- volatility3/framework/interfaces/objects.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 3cc23e759..d1f442d69 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -243,11 +243,10 @@ class ObjectInterface(metaclass = abc.ABCMeta): @classmethod @abc.abstractmethod - def child_template(cls, template: 'Template', child: str) -> interfaces.objects.Template: + def child_template(cls, template: 'Template', child: str) -> 'interfaces.objects.Template': """Returns the template of the child member from the parent.""" raise KeyError(f"Template does not contain any children: {template.vol.type_name}") - @classmethod @abc.abstractmethod def has_member(cls, template: 'Template', member_name: str) -> bool: From b1b4d21bbcfd0095a10d363e99d8b4bcbfb1a015 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Jul 2022 22:46:31 +0100 Subject: [PATCH 285/404] Core: Prevent circular dependency on imports - take 2 --- volatility3/framework/interfaces/objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index d1f442d69..fcf3c8d6c 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -312,7 +312,7 @@ class Template: offset.""" @abc.abstractmethod - def child_template(self, child: str) -> interfaces.objects.Template: + def child_template(self, child: str) -> 'interfaces.objects.Template': """Returns the `child` member template from its parent.""" @abc.abstractmethod From ec78fe7d8dd015c8ebfc366a5937cebe2bf92b3e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 4 Jul 2022 15:49:03 +0900 Subject: [PATCH 286/404] Fix: try/except/pass to contextlib.supress by #782 --- volatility3/plugins/windows/registry/certificates.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index e873fd1d6..429db96a6 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,3 +1,4 @@ +import contextlib import logging import struct from typing import List, Iterator, Optional, Tuple, Type @@ -67,7 +68,7 @@ class Certificates(interfaces.plugins.PluginInterface): "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - try: + with contextlib.suppress(KeyError, exceptions.SwappedInvalidAddressException): # Walk it node_path = hive.get_key(top_key, return_list = True) for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): @@ -83,13 +84,6 @@ class Certificates(interfaces.plugins.PluginInterface): file_handle.close() yield (0, (top_key, reg_section, key_hash, name)) - except KeyError: - # Key wasn't found in this hive, carry on - vollog.log(constants.LOGLEVEL_VVVV, "Key wasn't found in this hive") - pass - except exceptions.SwappedInvalidAddressException as exp: - vollog.log(constants.LOGLEVEL_VVVV, f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)") - pass def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), From 2f25312a5c96376b58817772bde1371b424d3f49 Mon Sep 17 00:00:00 2001 From: Malware Utkonos Date: Mon, 4 Jul 2022 13:08:30 -0400 Subject: [PATCH 287/404] Refactor try to reduce size of clause to only what is needed. Based on feedback, memory_object.get_available_pages() might raise this type of exception, so it's still inside the try clause. --- .../framework/plugins/windows/dumpfiles.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 58166ee7f..2c3f8c2d5 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -63,29 +63,28 @@ class DumpFiles(interfaces.plugins.PluginInterface): :return: result status """ filedata = open_method(desired_file_name) - try: - # Description of these variables: - # memoffset: offset in the specified layer where the page begins - # fileoffset: write to this offset in the destination file - # datasize: size of the page + # Description of these variables: + # memoffset: offset in the specified layer where the page begins + # fileoffset: write to this offset in the destination file + # datasize: size of the page - # track number of bytes written so we don't write empty files to disk - bytes_written = 0 + # track number of bytes written so we don't write empty files to disk + bytes_written = 0 + try: for memoffset, fileoffset, datasize in memory_object.get_available_pages(): data = layer.read(memoffset, datasize, pad = True) bytes_written += len(data) filedata.seek(fileoffset) filedata.write(data) - - if not bytes_written: - vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") - return None - else: - vollog.debug(f"Stored {filedata.preferred_filename}") - return filedata except exceptions.InvalidAddressException: vollog.debug(f"Unable to dump file at {file_object.vol.offset:#x}") return None + if not bytes_written: + vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") + return None + vollog.debug(f"Stored {filedata.preferred_filename}") + + return filedata @classmethod def process_file_object(cls, context: interfaces.context.ContextInterface, primary_layer_name: str, From 772ae98eb1966b4b0fa7451673352c6f8f97095c Mon Sep 17 00:00:00 2001 From: Malware Utkonos Date: Mon, 4 Jul 2022 13:27:17 -0400 Subject: [PATCH 288/404] Style changes including yapf according to .style.yapf in package root --- .../framework/plugins/windows/dumpfiles.py | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 58166ee7f..26b637fc3 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -5,6 +5,7 @@ import logging import ntpath from typing import List, Tuple, Type, Optional, Generator + from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -32,8 +33,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.IntRequirement(name = 'pid', description = "Process ID to include (all other processes are excluded)", optional = True), @@ -98,12 +100,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): :param open_method: class for constructing output files :param file_obj: the FILE_OBJECT """ - # Filtering by these types of devices prevents us from processing other types of devices that # use the "File" object type, such as \Device\Tcp and \Device\NamedPipe. if file_obj.DeviceObject.DeviceType not in [FILE_DEVICE_DISK, FILE_DEVICE_NETWORK_FILE_SYSTEM]: - vollog.log(constants.LOGLEVEL_VVV, - f"The file object at {file_obj.vol.offset:#x} is not a file on disk") + vollog.log(constants.LOGLEVEL_VVV, f"The file object at {file_obj.vol.offset:#x} is not a file on disk") return # Depending on the type of object (DataSection, ImageSection, SharedCacheMap) we may need to @@ -120,7 +120,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): # layer to read from, # file extension to apply, # ) - dump_parameters = [] + dump_parameters = list() # The DataSectionObject and ImageSectionObject caches are handled in basically the same way. # We carve these "pages" from the memory_layer. @@ -131,8 +131,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if control_area.is_valid(): dump_parameters.append((control_area, memory_layer, extension)) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") # The SharedCacheMap is handled differently than the caches above. # We carve these "pages" from the primary_layer. @@ -142,8 +141,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if shared_cache_map.is_valid(): dump_parameters.append((shared_cache_map, primary_layer, "vacb")) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] @@ -151,7 +149,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): memory_object.vol.offset, cache_name, ntpath.basename(obj_name), extension) - file_handle = DumpFiles.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) + file_handle = cls.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) file_output = "Error dumping file" if file_handle: @@ -185,8 +183,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): try: object_table = proc.ObjectTable except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") continue for entry in handles_plugin.handles(object_table): @@ -218,12 +215,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue - for result in self.process_file_object(self.context, kernel.layer_name, self.open, - file_obj): + for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot extract file from VAD at {vad.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"Cannot extract file from VAD at {vad.vol.offset:#x}") elif offsets: # Now process any offsets explicitly requested by the user. @@ -234,10 +229,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not is_virtual: layer_name = self.context.layers[layer_name].config["memory_layer"] - file_obj = self.context.object( - kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", + file_obj = self.context.object(kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", layer_name = layer_name, - native_layer_name = kernel.layer_name, + native_layer_name = kernel.layer_name, offset = offset) for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) @@ -246,9 +240,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): def run(self): # a list of tuples (, ) where is the address and is True for virtual. - offsets = [] + offsets = list() # a list of processes matching the pid filter. all files for these process(es) will be dumped. - procs = [] + procs = list() kernel = self.context.modules[self.config['kernel']] if self.config.get("virtaddr", None) is not None: From b30cb5d96842178085967f9462487e1b08b3ec19 Mon Sep 17 00:00:00 2001 From: Malware Utkonos Date: Mon, 4 Jul 2022 13:47:23 -0400 Subject: [PATCH 289/404] Move debug logging based on feedback. --- volatility3/framework/plugins/windows/dumpfiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 2c3f8c2d5..7e1480cbe 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -82,8 +82,8 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not bytes_written: vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") return None - vollog.debug(f"Stored {filedata.preferred_filename}") + vollog.debug(f"Stored {filedata.preferred_filename}") return filedata @classmethod From 84d26ba4bdf46b0280b343b3bd3c3b7ee54238c8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 5 Jul 2022 11:08:41 +0100 Subject: [PATCH 290/404] Core: Add in API_CHANGES updates --- API_CHANGES.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/API_CHANGES.md b/API_CHANGES.md index 274d1d8bb..4d8733286 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,14 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.3.0 +===== +Add in `child_template` to template class + +2.2.0 +===== +Changes to linux core calls + 2.1.0 ===== Add in the linux `task.get_threads` method to the API. From 179d35d03dded64676339ceea37b04ef4a7107c5 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:15:48 +0530 Subject: [PATCH 291/404] volatility to volatiliy3 --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 294f8ff63..d8b30ff28 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -1,7 +1,7 @@ Linux Tutorial ============== -This guide gives you a brief introduction to how volatility3 works and some demonstration on suite of plugins available from +This guide gives you a brief introduction to how volatility3 works and some demonstration of several of the plugins available from Acquiring memory ---------------- From 92f308b5e7c7b8556072e411d66a5e089828e658 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:16:07 +0530 Subject: [PATCH 292/404] volatility3 specified --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index d8b30ff28..28fa4e0cd 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -6,7 +6,7 @@ This guide gives you a brief introduction to how volatility3 works and some demo Acquiring memory ---------------- -Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `LiME `_ for this purpose. +Volatility3 does not provide the ability to acquire memory. In this tutorial we will see how we can use `LiME `_ for this purpose. It supports 32 and 64 bit captures from native Intel hardware as well as virtual machine guests. It also supports capture from Android devices. See below for example commands building and running LiME: From b782e1d751d57fabea3e382ebdee55966eb8eeaf Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:27:36 +0530 Subject: [PATCH 293/404] path adjustments made to have relative and generic --- doc/source/Linux.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 28fa4e0cd..3c77a16b6 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -16,12 +16,12 @@ It also supports capture from Android devices. See below for example commands bu $ cd lime-forensics-1.1-r14/src $ make .... - CC [M] /home/mhl/Downloads/src/tcp.o - CC [M] /home/mhl/Downloads/src/disk.o + CC [M] lime-forensics-1.1-r14/src/tcp.o + CC [M] lime-forensics-1.1-r14/src/disk.o .... - $ sudo insmod lime-3.2.0-23-generic.ko "path=/home/mhl/ubuntu.lime format=lime" - $ ls -alh /home/mhl/ubuntu.lime - -r--r--r-- 1 root root 2.0G Aug 17 19:37 /home/mhl/ubuntu.lime + $ sudo insmod lime-3.2.0-23-generic.ko "path=/tmp/ubuntu.lime format=lime" + $ ls -alh /tmp/ubuntu.lime + -r--r--r-- 1 root root 2.0G Aug 17 19:37 /tmp/ubuntu.lime Procedure to create symbol tables for linux -------------------------------------------- From fc48e7b83f59320642952466e40e515b242aa0ec Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:33:33 +0530 Subject: [PATCH 294/404] regarding ISF server its moved to tips section --- doc/source/Linux.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 3c77a16b6..21b8659fa 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -27,8 +27,9 @@ Procedure to create symbol tables for linux -------------------------------------------- To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. -We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. -After creating the file or downloading the file from the ISF server, please place the file under the directory ``volatility3/symbols/linux``. Make a directory linux under symbols. + +.. tip:: We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. + After creating the file or downloading the file from the ISF server, please place the file under the directory ``volatility3/symbols/linux``. Make a directory linux under symbols. Listing plugins From 89f1116374a1bd2e57696d2109fc7a9475d43c5c Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:36:58 +0530 Subject: [PATCH 295/404] Sentence reframed and clarrified regarding sample plugin list --- doc/source/Linux.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 21b8659fa..82b42b251 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -35,7 +35,7 @@ To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symb Listing plugins --------------- -Following are the list of linux plugins available for volatility3. More plugins will be available on future releases. +Following are the sample of linux plugins available for volatility3. More plugins will be available on future releases. For plugin requests, Please create an issue with description of the plugin. .. code-block:: shell-session @@ -47,6 +47,8 @@ For plugin requests, Please create an issue with description of the plugin. linux.check_creds.Check_creds linux.check_idt.Check_idt +.. note:: Here the the command is piped to grep and head in-order to give you sample list of plugins. + Using plugins ------------- From 3da043c60192a706e82b115952065b84d892de9e Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:39:08 +0530 Subject: [PATCH 296/404] Command syntax angular bracket added --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 82b42b251..bf6b28be9 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -57,7 +57,7 @@ The following is the syntax to run volatility tool. .. code-block:: shell-session - $ python3 vol.py -f plugin_name plugin_option + $ python3 vol.py -f Example From ed361893c7e3b2a055d7d5c5785102da453e3f4c Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:40:43 +0530 Subject: [PATCH 297/404] command fix vol.py to python3 vol.py --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index bf6b28be9..8919a83ce 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -40,7 +40,7 @@ For plugin requests, Please create an issue with description of the plugin. .. code-block:: shell-session - $ vol3 --help | grep -i linux. | head -n 5 + $ python3 vol.py --help | grep -i linux. | head -n 5 banners.Banners Attempts to identify potential linux banners in an linux.bash.Bash Recovers bash command history from memory. linux.check_afinfo.Check_afinfo From 20b76830f7ccefcb6b2d01dd6c0423083891a9a1 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:42:49 +0530 Subject: [PATCH 298/404] Removed external link to memory dump --- doc/source/Linux.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 8919a83ce..773aa80e4 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -66,8 +66,7 @@ Example Example 1 ~~~~~~~~~ -In this example we will be using memory dump from Insomni'hack teaser 2020 CTF. Challenge name Getdents, you can find the memory dump -in the link `here `_ . We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. +In this example we will be using memory dump from Insomni'hack teaser 2020 CTF. Challenge name Getdents. We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. I'd like to say thanks to `stuxnet `_ for providing this memory dump and `writeup `_. From 0844929610893ca5e239037c1b1f2031316acbb2 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:45:13 +0530 Subject: [PATCH 299/404] Use same voltility3 version in documentation volatility3 2.0.1 --- doc/source/Linux.rst | 8 ++++---- doc/source/Windows.rst | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 773aa80e4..8132453ce 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -74,7 +74,7 @@ I'd like to say thanks to `stuxnet `_ for provid $ python3 vol.py -f memory.vmem banners - Volatility 3 Framework 2.0.3 + Volatility 3 Framework 2.0.1 Progress: 100.00 PDB scanning finished Offset Banner @@ -96,7 +96,7 @@ If you do not find the ISF file then, please follow the instructions on :ref:`Li $ python3 vol.py -f memory.vmem linux.pslist - Volatility 3 Framework 2.0.3 Stacking attempts finished + Volatility 3 Framework 2.0.1 Stacking attempts finished PID PPID COMM @@ -123,7 +123,7 @@ If you do not find the ISF file then, please follow the instructions on :ref:`Li .. code-block:: shell-session $ python3 vol.py -f memory.vmem linux.pstree - Volatility 3 Framework 2.0.3 + Volatility 3 Framework 2.0.1 Progress: 100.00 Stacking attempts finished PID PPID COMM @@ -167,7 +167,7 @@ Now to find the commands ran in bash shell. Lets use ``linux.bash``. $ python3 vol.py -f memory.vmem linux.bash - Volatility 3 Framework 2.0.3 + Volatility 3 Framework 2.0.1 Progress: 100.00 Stacking attempts finished PID Process CommandTime Command diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index 3e6844f22..e722be0a3 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -45,7 +45,7 @@ In windows memory forensics using volatility3, most of the times we do not requi $ python3 vol.py -f MemDump.DMP windows.pslist | head -n 10 - Volatility 3 Framework 2.0.2 PDB scanning finished + Volatility 3 Framework 2.0.1 PDB scanning finished PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output @@ -61,7 +61,7 @@ In windows memory forensics using volatility3, most of the times we do not requi .. code-block:: shell-session $ python3 vol.py -f MemDump.DMP windows.pstree | head -n 20 - Volatility 3 Framework 2.0.2 PDB scanning finished + Volatility 3 Framework 2.0.1 PDB scanning finished PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime From 1ccd31b506768ef5e9201d5722aeff59be309919 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:47:47 +0530 Subject: [PATCH 300/404] Added note regarding pipe in windows doc and moved winPEM to tip --- doc/source/Windows.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index e722be0a3..2f8d58a04 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -6,7 +6,9 @@ This guide gives you a brief introduction to how volatility3 works and some demo Acquiring memory ---------------- -Volatility does not provide the ability to acquire memory. In this tutorial we will see how we can use `WinPmem `_ for this purpose. +Volatility does not provide the ability to acquire memory. + +.. tip:: You could use `WinPmem `_ for collecting windows memory dump. Listing Plugins --------------- @@ -20,6 +22,8 @@ Listing Plugins windows.dlllist.DllList Lists the loaded modules in a particular windows +.. note:: Here the the command is piped to grep and head in-order to give you sample list of plugins. + Using plugins ------------- From 73eec3386dc843761e37bfff6420e173cff891e8 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:48:49 +0530 Subject: [PATCH 301/404] Reference to memory dump removed --- doc/source/Windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index 2f8d58a04..a9e712fc9 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -40,7 +40,7 @@ Example Example 1 ~~~~~~~~~ -In this example we will be using memory dump from PragyanCTF'22. The dump is available `here `_. +In this example we will be using memory dump from PragyanCTF'22. We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. In windows memory forensics using volatility3, most of the times we do not require creating a ISF file. From bcc923b1b92567c66f3e96c5996084fdff892895 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:50:33 +0530 Subject: [PATCH 302/404] Info regarding pipe added --- doc/source/Windows.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index a9e712fc9..d26f41aa8 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -88,6 +88,9 @@ In windows memory forensics using volatility3, most of the times we do not requi ``windows.pstree`` helps us to display the parent child relation of processes. +.. note:: Here the the command is piped to head in-order to give you smaller output of process here top 20. + + .. code-block:: shell-session $ python3 vol.py -f MemDump.DMP windows.hashdump From 596047c251a6363e49656e79284a56e974b0c8a3 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:51:32 +0530 Subject: [PATCH 303/404] small adjustment made in note regarding pipe --- doc/source/Linux.rst | 2 +- doc/source/Windows.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 8132453ce..b5a3db09c 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -47,7 +47,7 @@ For plugin requests, Please create an issue with description of the plugin. linux.check_creds.Check_creds linux.check_idt.Check_idt -.. note:: Here the the command is piped to grep and head in-order to give you sample list of plugins. +.. note:: Here the the command is piped to grep and head in-order to give you sample list of linux plugins. Using plugins diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index d26f41aa8..55677a67c 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -22,7 +22,7 @@ Listing Plugins windows.dlllist.DllList Lists the loaded modules in a particular windows -.. note:: Here the the command is piped to grep and head in-order to give you sample list of plugins. +.. note:: Here the the command is piped to grep and head in-order to give you sample list of windows plugins. Using plugins ------------- From e7b33f6c841b250db1a1014d2acd412f456705df Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:54:37 +0530 Subject: [PATCH 304/404] Description on listing plugins in windows added --- doc/source/Windows.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index 55677a67c..a6c67780e 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -13,6 +13,9 @@ Volatility does not provide the ability to acquire memory. Listing Plugins --------------- +Following are the sample of linux plugins available for volatility3. More plugins will be available on future releases. +For plugin requests, Please create an issue with description of the plugin. + .. code-block:: shell-session $ python3 vol.py --help | grep windows | head -n 5 From 3714fa8c9254c6a29bc2657788521de285baf6f3 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:56:45 +0530 Subject: [PATCH 305/404] Note regarding using sudo added --- doc/source/Linux.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index b5a3db09c..180e7c697 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -23,6 +23,8 @@ It also supports capture from Android devices. See below for example commands bu $ ls -alh /tmp/ubuntu.lime -r--r--r-- 1 root root 2.0G Aug 17 19:37 /tmp/ubuntu.lime +.. note:: The above command required sudo inorder to access the files which are root only. + Procedure to create symbol tables for linux -------------------------------------------- From df277b9e802899368186aa04c4d106ba06de1e9b Mon Sep 17 00:00:00 2001 From: Frank Gomulka Date: Wed, 13 Jul 2022 13:49:30 -0500 Subject: [PATCH 306/404] Add testing framework --- .github/workflows/test.yaml | 54 +++++ test/README.md | 34 +++ test/conftest.py | 40 ++++ test/known_files.json | 19 ++ test/requirements-testing.txt | 8 + test/test_volatility.py | 381 ++++++++++++++++++++++++++++++++++ 6 files changed, 536 insertions(+) create mode 100644 .github/workflows/test.yaml create mode 100644 test/README.md create mode 100644 test/conftest.py create mode 100644 test/known_files.json create mode 100644 test/requirements-testing.txt create mode 100644 test/test_volatility.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..5a3f90565 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,54 @@ +name: Test Volatility3 +on: [push] +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install Cmake + pip install setuptools wheel + pip install -U pytest + pip install -r ./test/requirements-testing.txt + + - name: Build PyPi packages + run: | + python setup.py sdist --formats=gztar,zip + python setup.py bdist_wheel + + - name: Download images + run: | + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz" + gunzip linux-sample-1.bin.gz + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" + gunzip win-xp-laptop-2005-06-25.img.gz + + - name: Download and Extract symbols + run: | + cd ./volatility3/symbols + curl -sLO https://downloads.volatilityfoundation.org/volatility3/symbols/linux.zip + unzip linux.zip + cd - + + - name: Testing... + run: | + py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v + py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + + - name: Clean up post-test + run: | + rm -rf *.lime + rm -rf *.img + cd volatility3/symbols + rm -rf linux + rm -rf linux.zip + cd - diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..dcbe289b0 --- /dev/null +++ b/test/README.md @@ -0,0 +1,34 @@ +# Volatility 3 Testing Framework + +## Requirements + +The Volatility 3 Testing Framework requires the same version of Python as Volatility3 itself. To install the current set of dependencies that the framework requires, use a command like this: + +```shell +pip3 install -r requirements-testing.txt +``` + +NOTE: `requirements-testing.txt` can be found in this current `test/` directory. + +## Quick Start: Manual Testing + +1. To test Volatility 3 on an image, first download one with a command such as: + +```shell +curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" +gunzip win-xp-laptop-2005-06-25.img.gz +``` + +2. In many cases, more symbols are required to be downloaded to the `./volatility3/symbols` directory. + +3. To manually run the tests, run a command, such as: + +```shell +py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows +``` + +The above command runs all available tests for windows on the `win-xp-laptop-2005-06-25.img` image. To choose a more specific set of tests, change the phrase after `-k` in this command. + +## Github Actions + +This framework currently tests two images (one linux image and one windows image) after every push on any branch. For more information/context, find the actions setup in `./github/workflows/test.yaml` \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 000000000..9d3d27fc5 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,40 @@ +# This file is used to augment the test configuration + +import os +import pytest + +def pytest_addoption(parser): + parser.addoption("--volatility", action="store", default=None, + required=True, + help="path to the volatility script") + + parser.addoption("--python", action="store", default="python3", + help="The name of the interpreter to use when running the volatility script") + + parser.addoption("--image", action="append", default=[], + help="path to an image to test") + + parser.addoption("--image-dir", action="append", default=[], + help="path to a directory containing images to test") + +def pytest_generate_tests(metafunc): + """Parameterize tests based on image names""" + + images = metafunc.config.getoption('image') + for d in metafunc.config.getoption('image_dir'): + images = images + [os.path.join(d, x) for x in os.listdir(d)] + + # tests with "image" parameter are run against images + if 'image' in metafunc.fixturenames: + metafunc.parametrize("image", + images, + ids=[os.path.basename(image) for image in images]) + +# Fixtures +@pytest.fixture +def volatility(request): + return request.config.getoption("--volatility") + +@pytest.fixture +def python(request): + return request.config.getoption("--python") diff --git a/test/known_files.json b/test/known_files.json new file mode 100644 index 000000000..fbc40e48b --- /dev/null +++ b/test/known_files.json @@ -0,0 +1,19 @@ +{ + "windows_dumpfiles": { + "win-xp-laptop-2005-06-25.img": { + "0x82220e78": [ + "9bdd5532286f1660f3778e68bc36efe6", + "e3bc1e9e7370e3b5a661ebe591ecf4ec" + ], + "0x82350bf8": [ + "e5c5e8d97b6280745b41f6572c85d1f0", + "8589f1463422884dbf1411aaad278465" + ], + "0x81eaf418": [ + "f7a1ae2060a58f8470b97affdb46dccf", + "54fd611021fa784912530b8007545986" + ], + "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" + } + } + } \ No newline at end of file diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt new file mode 100644 index 000000000..d37dc93c3 --- /dev/null +++ b/test/requirements-testing.txt @@ -0,0 +1,8 @@ +# These packages are required for core functionality. +pefile>=2017.8.1 #foo + +# The following packages are optional. +# If certain packages are not necessary, place a comment (#) at the start of the line. + +# This is required for the yara plugins +yara-python>=3.8.0 diff --git a/test/test_volatility.py b/test/test_volatility.py new file mode 100644 index 000000000..527d86dd3 --- /dev/null +++ b/test/test_volatility.py @@ -0,0 +1,381 @@ +# volatility3 tests +# + +# +# IMPORTS +# + +import os +import subprocess +import sys +import shutil +import tempfile +import hashlib +import ntpath +import json + +import pytest + +# +# HELPER FUNCTIONS +# + +def runvol(args, volatility, python): + volpy = volatility + python_cmd = python + + cmd = [python_cmd, volpy] + args + print(" ".join(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + print("stdout:") + sys.stdout.write(str(stdout)) + print("") + print("stderr:") + sys.stdout.write(str(stderr)) + print("") + + return p.returncode, stdout, stderr + +def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): + args = globalargs + [ + "--single-location", + img, + "-q", + plugin, + ] + pluginargs + + return runvol(args, volatility, python) + +# +# TESTS +# + +# WINDOWS + +def test_windows_pslist(image, volatility, python): + rc, out, err = runvol_plugin("windows.pslist.PsList", image, volatility, python) + out = out.lower() + assert out.find(b"system") != -1 + assert out.find(b"csrss.exe") != -1 + assert out.find(b"svchost.exe") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + + rc, out, err = runvol_plugin( + "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"]) + out = out.lower() + assert out.find(b"system") != -1 + assert out.count(b"\n") < 10 + assert rc == 0 + assert rc == 0 + +def test_windows_psscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) + out = out.lower() + assert out.find(b"system") != -1 + assert out.find(b"csrss.exe") != -1 + assert out.find(b"svchost.exe") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_dlllist(image, volatility, python): + rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) + out = out.lower() + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_modules(image, volatility, python): + rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) + out = out.lower() + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_hivelist(image, volatility, python): + rc, out, err = runvol_plugin("windows.registry.hivelist.HiveList", image, volatility, python) + out = out.lower() + + not_xp = out.find(b"\\systemroot\\system32\\config\\software") + if not_xp == -1: + assert out.find(b"\\device\\harddiskvolume1\\windows\\system32\\config\\software") != -1 + + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_windows_dumpfiles(image, volatility, python): + + json_file = open('./test/known_files.json') + + known_files = json.load(json_file) + + failed_chksms = 0 + + if sys.platform == 'win32': + file_name = ntpath.basename(image) + else: + file_name = os.path.basename(image) + + try: + for addr in known_files["windows_dumpfiles"][file_name]: + + path = tempfile.mkdtemp() + + rc, out, err = runvol_plugin("windows.dumpfiles.DumpFiles", image, volatility, python, globalargs=["-o", path], pluginargs=["--virtaddr", addr]) + + for file in os.listdir(path): + fp = open(os.path.join(path, file), "rb") + if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: + failed_chksms += 1 + fp.close() + + shutil.rmtree(path) + + json_file.close() + + assert failed_chksms == 0 + assert rc == 0 + except Exception as e: + json_file.close() + print("Key Error raised on " + str(e)) + assert False + +def test_windows_handles(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"System Pid 4") != -1 + assert out.find(b"MACHINE\\SYSTEM\\CONTROLSET001\\CONTROL\\SESSION MANAGER\\MEMORY MANAGEMENT\\PREFETCHPARAMETERS") != -1 + assert out.find(b"MACHINE\\SYSTEM\\SETUP") != -1 + assert out.count(b"\n") > 500 + assert rc == 0 + +def test_windows_svcscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) + + assert out.find(b"Microsoft ACPI Driver") != -1 + assert out.count(b"\n") > 250 + assert rc == 0 + +def test_windows_privileges(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"SeCreateTokenPrivilege") != -1 + assert out.find(b"SeCreateGlobalPrivilege") != -1 + assert out.find(b"SeAssignPrimaryTokenPrivilege") != -1 + assert out.count(b"\n") > 20 + assert rc == 0 + +def test_windows_getsids(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"Local System") != -1 + assert out.find(b"Administrators") != -1 + assert out.find(b"Everyone") != -1 + assert out.find(b"Authenticated Users") != -1 + assert rc == 0 + +def test_windows_envars(image, volatility, python): + rc, out, err = runvol_plugin("windows.envars.Envars", image, volatility, python) + + assert out.find(b"PATH") != -1 + assert out.find(b"PROCESSOR_ARCHITECTURE") != -1 + assert out.find(b"USERNAME") != -1 + assert out.find(b"SystemRoot") != -1 + assert out.find(b"CommonProgramFiles") != -1 + assert out.count(b"\n") > 500 + assert rc == 0 + +def test_windows_callbacks(image, volatility, python): + rc, out, err = runvol_plugin("windows.callbacks.Callbacks", image, volatility, python) + + assert out.find(b"PspCreateProcessNotifyRoutine") != -1 + assert out.find(b"KeBugCheckCallbackListHead") != -1 + assert out.find(b"KeBugCheckReasonCallbackListHead") != -1 + assert out.count(b"KeBugCheckReasonCallbackListHead ") > 5 + assert rc == 0 + +# LINUX + +def test_linux_pslist(image, volatility, python): + rc, out, err = runvol_plugin("linux.pslist.PsList", image, volatility, python) + out = out.lower() + + assert ((out.find(b"init") != -1) or (out.find(b"systemd") != -1)) + assert out.find(b"watchdog") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_check_idt(image, volatility, python): + rc, out, err = runvol_plugin("linux.check_idt.Check_idt", image, volatility, python) + out = out.lower() + + assert out.count(b"__kernel__") >= 10 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_check_syscall(image, volatility, python): + rc, out, err = runvol_plugin("linux.check_syscall.Check_syscall", image, volatility, python) + out = out.lower() + + assert out.find(b"sys_close") != -1 + assert out.find(b"sys_open") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_linux_lsmod(image, volatility, python): + rc, out, err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_lsof(image, volatility, python): + rc, out, err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) + out = out.lower() + + assert out.count(b"socket:") >= 10 + assert out.count(b"\n") > 35 + assert rc == 0 + +def test_linux_proc_maps(image, volatility, python): + rc, out, err = runvol_plugin("linux.proc.Maps", image, volatility, python) + out = out.lower() + + assert out.count(b"anonymous mapping") >= 10 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_linux_tty_check(image, volatility, python): + rc, out, err = runvol_plugin("linux.tty_check.tty_check", image, volatility, python) + out = out.lower() + + assert out.find(b"__kernel__") != -1 + assert out.count(b"\n") >= 5 + assert rc == 0 + +# MAC + +def test_mac_pslist(image, volatility, python): + rc, out, err = runvol_plugin("mac.pslist.PsList", image, volatility, python) + out = out.lower() + + assert ((out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1)) + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_check_syscall(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_syscall.Check_syscall", image, volatility, python) + out = out.lower() + + assert out.find(b"chmod") != -1 + assert out.find(b"chown") != -1 + assert out.find(b"nosys") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_mac_check_sysctl(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_sysctl.Check_sysctl", image, volatility, python) + out = out.lower() + + assert out.find(b"__kernel__") != -1 + assert out.count(b"\n") > 250 + assert rc == 0 + +def test_mac_check_trap_table(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_trap_table.Check_trap_table", image, volatility, python) + out = out.lower() + + assert out.count(b"kern_invalid") >= 10 + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_ifconfig(image, volatility, python): + rc, out, err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) + out = out.lower() + + assert out.find(b"127.0.0.1") != -1 + assert out.find(b"false") != -1 + assert out.count(b"\n") > 9 + assert rc == 0 + +def test_mac_lsmod(image, volatility, python): + rc, out, err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) + out = out.lower() + + assert out.find(b"com.apple") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_lsof(image, volatility, python): + rc, out, err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_malfind(image, volatility, python): + rc, out, err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 20 + assert rc == 0 + +def test_mac_mount(image, volatility, python): + rc, out, err = runvol_plugin("mac.mount.Mount", image, volatility, python) + out = out.lower() + + assert out.find(b"/dev") != -1 + assert out.count(b"\n") > 7 + assert rc == 0 + +def test_mac_netstat(image, volatility, python): + rc, out, err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) + + assert out.find(b"TCP") != -1 + assert out.find(b"UDP") != -1 + assert out.find(b"UNIX") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_proc_maps(image, volatility, python): + rc, out, err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) + out = out.lower() + + assert out.find(b"[heap]") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_mac_psaux(image, volatility, python): + rc, out, err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) + out = out.lower() + + assert out.find(b"executable_path") != -1 + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_socket_filters(image, volatility, python): + rc, out, err = runvol_plugin("mac.socket_filters.Socket_filters", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 9 + assert rc == 0 + +def test_mac_timers(image, volatility, python): + rc, out, err = runvol_plugin("mac.timers.Timers", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 6 + assert rc == 0 + +def test_mac_trustedbsd(image, volatility, python): + rc, out, err = runvol_plugin("mac.trustedbsd.Trustedbsd", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 From f96f004b13c15b3c0334f7ca96b2c0459a8f9cf1 Mon Sep 17 00:00:00 2001 From: Frank Gomulka Date: Fri, 15 Jul 2022 18:01:48 -0500 Subject: [PATCH 307/404] @digitalisx suggested changes --- .github/workflows/test.yaml | 6 +++--- test/test_volatility.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5a3f90565..a3ecd7c7e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,5 +1,5 @@ name: Test Volatility3 -on: [push] +on: [push, pull_request] jobs: build: @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.x + - name: Set up Python 3.6 uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.6' - name: Install dependencies run: | diff --git a/test/test_volatility.py b/test/test_volatility.py index 527d86dd3..a55dffb27 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -127,10 +127,9 @@ def test_windows_dumpfiles(image, volatility, python): rc, out, err = runvol_plugin("windows.dumpfiles.DumpFiles", image, volatility, python, globalargs=["-o", path], pluginargs=["--virtaddr", addr]) for file in os.listdir(path): - fp = open(os.path.join(path, file), "rb") - if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: - failed_chksms += 1 - fp.close() + with open(os.path.join(path, file), "rb") as fp: + if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: + failed_chksms += 1 shutil.rmtree(path) From f295e5d91a6b7ecc7a1f03099d2984a98cf43eda Mon Sep 17 00:00:00 2001 From: fgomulka <60993471+fgomulka@users.noreply.github.com> Date: Sat, 16 Jul 2022 16:34:47 -0500 Subject: [PATCH 308/404] Add newline Co-authored-by: Donghyun Kim --- test/known_files.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/known_files.json b/test/known_files.json index fbc40e48b..089896714 100644 --- a/test/known_files.json +++ b/test/known_files.json @@ -16,4 +16,5 @@ "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" } } - } \ No newline at end of file + } + \ No newline at end of file From 3e748e7d488eeb96904d94039ca02d57c6368fff Mon Sep 17 00:00:00 2001 From: fgomulka <60993471+fgomulka@users.noreply.github.com> Date: Sat, 16 Jul 2022 16:35:26 -0500 Subject: [PATCH 309/404] Use more descriptive variable names Co-authored-by: Donghyun Kim --- test/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 9d3d27fc5..9057e1676 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -21,8 +21,8 @@ def pytest_generate_tests(metafunc): """Parameterize tests based on image names""" images = metafunc.config.getoption('image') - for d in metafunc.config.getoption('image_dir'): - images = images + [os.path.join(d, x) for x in os.listdir(d)] + for image_dir in metafunc.config.getoption('image_dir'): + images = images + [os.path.join(image_dir, dir) for dir in os.listdir(image_dir)] # tests with "image" parameter are run against images if 'image' in metafunc.fixturenames: From 5bc517aa42f09bb467136866d92811760a92169b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 00:15:57 +0000 Subject: [PATCH 310/404] Automagic: Use sqlite to cache identifiers --- volatility3/framework/automagic/linux.py | 35 +- volatility3/framework/automagic/mac.py | 28 +- .../framework/automagic/symbol_cache.py | 480 ++++++++++++------ .../framework/automagic/symbol_finder.py | 25 +- .../framework/configuration/requirements.py | 12 +- volatility3/framework/constants/__init__.py | 7 +- volatility3/framework/interfaces/automagic.py | 9 +- volatility3/framework/plugins/isfinfo.py | 39 +- volatility3/framework/symbols/intermed.py | 3 +- .../framework/symbols/windows/pdbutil.py | 58 +-- 10 files changed, 417 insertions(+), 279 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f1d6c91e4..2c152996d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -5,8 +5,9 @@ import logging from typing import Optional, Tuple, Type -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder +from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import linux @@ -23,6 +24,13 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify linux within this layer.""" + # Version check the SQlite cache + required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + vollog.info( + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + return None + # Bail out by default unless we can stack properly layer = context.layers[layer_name] join = interfaces.configuration.path_join @@ -32,7 +40,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - linux_banners = LinuxBannerCache.load_banners() + linux_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + operating_system = 'linux') # If we have no banners, don't bother scanning if not linux_banners: vollog.info("No Linux banners found - if this is a linux plugin, please check your symbol files location") @@ -43,15 +52,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = linux_banners.get(banner, None) - if symbol_files: - if len(symbol_files) > 1: - using = "*" - vollog.warning(f"Multiple symbol files identified (using {using}):") - for symbol_file in symbol_files: - vollog.warning(f" {using} {symbol_file}") - using = " " - isf_path = symbol_files[0] + isf_path = linux_banners.get(banner, None) + if isf_path: table_name = context.symbol_space.free_table_name('LintelStacker') table = linux.LinuxKernelIntermedSymbols(context, 'temporary.' + table_name, @@ -147,20 +149,11 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): return addr - 0xc0000000 -class LinuxBannerCache(symbol_cache.SymbolBannerCache): - """Caches the banners found in the Linux symbol files.""" - - os = "linux" - symbol_name = "linux_banner" - banner_path = constants.LINUX_BANNERS_PATH - exclusion_list = ['mac', 'windows'] - - class LinuxSymbolFinder(symbol_finder.SymbolFinder): """Linux symbol loader based on uname signature strings.""" banner_config_key = "kernel_banner" - banner_cache = LinuxBannerCache + operating_system = 'linux' symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] exclusion_list = ['mac', 'windows'] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index c37aef463..246462878 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -6,8 +6,9 @@ import logging import struct from typing import Optional -from volatility3.framework import interfaces, constants, layers, exceptions +from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.automagic import symbol_cache, symbol_finder +from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import mac @@ -24,6 +25,13 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify mac within this layer.""" + # Version check the SQlite cache + required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + vollog.info( + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + return None + # Bail out by default unless we can stack properly layer = context.layers[layer_name] new_layer = None @@ -34,7 +42,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - mac_banners = MacBannerCache.load_banners() + mac_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + operating_system = 'mac') # If we have no banners, don't bother scanning if not mac_banners: vollog.info("No Mac banners found - if this is a mac plugin, please check your symbol files location") @@ -46,9 +55,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = mac_banners.get(banner, None) - if symbol_files: - isf_path = symbol_files[0] + isf_path = mac_banners.get(banner, None) + if isf_path: table_name = context.symbol_space.free_table_name('MacintelStacker') table = mac.MacKernelIntermedSymbols(context = context, config_path = join('temporary', table_name), @@ -197,19 +205,11 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): yield offset, banner -class MacBannerCache(symbol_cache.SymbolBannerCache): - """Caches the banners found in the Mac symbol files.""" - os = "mac" - symbol_name = "version" - banner_path = constants.MAC_BANNERS_PATH - exclusion_list = ['windows', 'linux'] - - class MacSymbolFinder(symbol_finder.SymbolFinder): """Mac symbol loader based on uname signature strings.""" banner_config_key = 'kernel_banner' - banner_cache = MacBannerCache + operating_system = 'mac' find_aslr = MacIntelStacker.find_aslr symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols" exclusion_list = ['windows', 'linux'] diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 7b6adf9b4..fe717b8be 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -2,18 +2,20 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import base64 -import gc import json import logging import os -import pickle +import sqlite3 import urllib import urllib.parse import urllib.request -import zipfile -from typing import Dict, List, Optional +from abc import abstractmethod +from typing import Dict, Generator, List, Optional -from volatility3.framework import constants, exceptions, interfaces +import volatility3.framework +import volatility3.schemas +from volatility3.framework import constants, interfaces +from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources from volatility3.framework.symbols import intermed @@ -22,164 +24,324 @@ vollog = logging.getLogger(__name__) BannersType = Dict[bytes, List[str]] -class SymbolBannerCache(interfaces.automagic.AutomagicInterface): - """Runs through all symbols tables and caches their banners.""" +### Identifiers - # Since this is necessary for ConstructionMagic, we set a lower priority - # The user would run it eventually either way, but running it first means it can be used that run +class IdentifierProcessor: + operating_system = None + + def __init__(self): + pass + + @classmethod + @abstractmethod + def get_identifier(cls, json) -> Optional[bytes]: + """Method to extract the identifier from a particular operating system's JSON + + Returns: + identifier is valid or None if not found + """ + raise NotImplemented("This base class has no get_identifier method defined") + + +class WindowsIdentifier(IdentifierProcessor): + operating_system = 'windows' + separator = '|' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + """Returns the identifier for the file if one can be found""" + windows_metadata = json.get('metadata', {}).get('windows', {}).get('pdb', {}) + if windows_metadata: + guid = windows_metadata.get('GUID', None) + age = windows_metadata.get('age', None) + database = windows_metadata.get('database', None) + if guid and age and database: + return cls.generate(database, guid, age) + return None + + @classmethod + def generate(cls, pdb_name: str, guid: str, age: int) -> bytes: + return bytes(cls.separator.join([pdb_name, guid.upper(), str(age)]), 'latin-1') + + +class MacIdentifier(IdentifierProcessor): + operating_system = 'mac' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + mac_banner = json.get('symbols', {}).get('version', {}).get('constant_data', None) + if mac_banner: + return base64.b64decode(mac_banner) + return None + + +class LinuxIdentifier(IdentifierProcessor): + operating_system = 'linux' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + linux_banner = json.get('symbols', {}).get('linux_banner', {}).get('constant_data', None) + if linux_banner: + return base64.b64decode(linux_banner) + return None + + +### CacheManagers + +class CacheManagerInterface(interfaces.configuration.VersionableInterface): + def __init__(self, filename: str): + super().__init__() + self._filename = filename + self._classifiers = {} + for subclazz in volatility3.framework.class_subclasses(IdentifierProcessor): + self._classifiers[subclazz.operating_system] = subclazz + + def add_identifier(self, location: str, operating_system: str, identifier: str): + """Adds an identifier to the store""" + pass + + def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + """Returns the location of the symbol file given the identifier + + Args: + identifier: string that uniquely identifies a particular symbolt table + operating_system: optional string to restrict identifiers to just those for a particular operating system + + Returns: + The location of the symbols file that matches the identifier + """ + pass + + def get_local_locations(self) -> List[str]: + """Returns a list of all the local locations""" + pass + + def update(self): + """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. + This also updates remote locations based on a cache timeout. + + """ + pass + + def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ + Dict[bytes, str]: + """Returns a dictionary of identifiers and locations + + Args: + operating_system: If set, limits responses to a specific operating system + local_only: Returns only local locations + + Returns: + A dictionary of identifiers mapped to a location + """ + pass + + def get_identifier(self, location: str) -> Optional[bytes]: + """Returns an identifier based on a specific location or None""" + pass + + def get_identifiers(self, operating_system: Optional[str]): + """Returns all identifiers for a particular operating system""" + pass + + +class SqliteCache(CacheManagerInterface): + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + def __init__(self, filename: str): + super().__init__(filename) + try: + self._database = self._connect_storage(filename) + except sqlite3.DatabaseError: + os.unlink(filename) + self._database = self._connect_storage(filename) + + def _connect_storage(self, path: str): + database = sqlite3.connect(path, isolation_level = None) + database.row_factory = sqlite3.Row + database.cursor().execute( + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + return database + + def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + """Returns the location of the symbol file given the identifier. + If multiple locations exist for an identifier, the last found is returned + + Args: + identifier: string that uniquely identifies a particular symbolt table + operating_system: optional string to restrict identifiers to just those for a particular operating system + + Returns: + The location of the symbols file that matches the identifier or None + """ + statement = 'SELECT location FROM cache WHERE identifier = ?' + parameters = (identifier,) + if operating_system is not None: + statement = 'SELECT location FROM cache WHERE identifier = ? AND operating_system = ?' + parameters = (identifier, operating_system) + results = self._database.cursor().execute(statement, parameters).fetchall() + result = None + for row in results: + result = row['location'] + return result + + def get_local_locations(self) -> Generator[str, None, None]: + result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = True').fetchall() + for row in result: + yield row['location'] + + def is_url_local(self, url: str) -> bool: + """Determines whether an url is local or not""" + parsed = urllib.parse.urlparse(url) + if parsed.scheme in ['file', 'jar']: + return True + + def get_identifier(self, location: str) -> Optional[bytes]: + results = self._database.cursor().execute('SELECT identifier FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['identifier'] + return None + + def update(self, progress_callback = None): + """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. + This also updates remote locations based on a cache timeout. + + """ + on_disk_locations = set([filename for filename in intermed.IntermediateSymbolTable.file_symbol_url('')]) + cached_locations = set(self.get_local_locations()) + + new_locations = on_disk_locations.difference(cached_locations) + missing_locations = cached_locations.difference(on_disk_locations) + + cache_update = set() + files_to_timestamp = on_disk_locations.intersection(cached_locations) + if files_to_timestamp: + result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " + "AND cached < date('now', '-3 days');") + for row in result: + if row['location'] in files_to_timestamp: + cache_update.add(row['location']) + + idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + + counter = 0 + files_to_process = new_locations.union(cache_update) + number_files_to_process = len(files_to_process) + for location in files_to_process: + # Open location + counter += 1 + progress_callback(counter * 100 / number_files_to_process, + "Updating caches for {number_files_to_process} files...") + try: + with resources.ResourceAccessor().open(location) as fp: + json_obj = json.load(fp) + identifier = None + for idextractor in idextractors: + identifier = idextractor.get_identifier(json_obj) + operating_system = idextractor.operating_system + if identifier is not None: + break + if identifier is not None: + # We don't try to validate schemas here, we do that on first use + # Store in database + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") + else: + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + None, + None, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") + except Exception as excp: + vollog.log(constants.LOGLEVEL_VVVV, excp) + + if not constants.OFFLINE and constants.REMOTE_ISF_URL: + remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) + for operating_system in ['mac', 'linux', 'windows']: + identifiers = remote_identifiers.process({}, operating_system = operating_system) + for identifier in identifiers: + for location in identifiers[identifier]: + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now')", + (location, identifier, operating_system, False) + ) + + if missing_locations: + self._database.cursor().execute( + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + + def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ + Dict[bytes, str]: + output = {} + additions = [] + statement = 'SELECT location, identifier FROM cache' + if local_only: + additions.append('local = True') + if operating_system: + additions.append(f"operating_system = '{operating_system}'") + if additions: + statement += f" WHERE {' AND '.join(additions)}" + results = self._database.cursor().execute(statement) + for row in results: + if row['identifier'] in output and row['identifier'] and row['location']: + vollog.debug( + f"Duplicate entry for identifier {row['identifier']}: {row['location']} and {output[row['identifier']]}") + output[row['identifier']] = row['location'] + return output + + def get_identifiers(self, operating_system: Optional[str]): + if operating_system: + results = self._database.cursor().execute('SELECT identifier FROM cache WHERE operating_system = ?', + (operating_system,)).fetchall() + else: + results = self._database.cursor().execute('SELECT identifier FROM cache').fetchall() + output = [] + for row in results: + output.append(row['identifier']) + return output + + +### Automagic + +class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): + """Runs through all symbol tables and caches their identifiers""" priority = 0 - os: Optional[str] = None - symbol_name: str = "banner_name" - banner_path: Optional[str] = None - - @classmethod - def load_banners(cls) -> BannersType: - if not cls.banner_path: - raise ValueError("Banner_path not appropriately set") - banners: BannersType = {} - if os.path.exists(cls.banner_path): - with open(cls.banner_path, "rb") as f: - # We use pickle over JSON because we're dealing with bytes objects - banners.update(pickle.load(f)) - - # Remove possibilities that can't exist locally. - remove_banners = [] - for banner in banners: - for path in banners[banner]: - url = urllib.parse.urlparse(path) - if url.scheme == 'file' and not os.path.exists(urllib.request.url2pathname(url.path)): - vollog.log( - constants.LOGLEVEL_VV, "Removing cached path {} for banner {}: file does not exist".format( - path, str(banner or b'', 'latin-1'))) - banners[banner].remove(path) - # This is probably excessive, but it's here if we need it - if url.scheme == 'jar': - zip_file, zip_path = url.path.split("!") - zip_file = urllib.parse.urlparse(zip_file).path - if ((not os.path.exists(zip_file)) or (zip_path not in zipfile.ZipFile(zip_file).namelist())): - vollog.log(constants.LOGLEVEL_VV, - "Removing cached path {} for banner {}: file does not exist".format(path, banner)) - banners[banner].remove(path) - - if not banners[banner]: - remove_banners.append(banner) - for remove_banner in remove_banners: - del banners[remove_banner] - return banners - - @classmethod - def save_banners(cls, banners): - - with open(cls.banner_path, "wb") as f: - pickle.dump(banners, f) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cache = SqliteCache(constants.IDENTIFIERS_PATH) def __call__(self, context, config_path, configurable, progress_callback = None): """Runs the automagic over the configurable.""" - - # Bomb out if we're just the generic interface - if self.os is None: - return - - # We only need to be called once, so no recursion necessary - banners = self.load_banners() - - cacheables = self.find_new_banner_files(banners, self.os) - - new_banners = self.read_new_banners(context, config_path, cacheables, self.symbol_name, self.os, - progress_callback) - - # Add in any new banners to the existing list - for new_banner in new_banners: - banner_list = banners.get(new_banner, []) - banners[new_banner] = list(set(banner_list + new_banners[new_banner])) - - # Do remote banners *after* the JSON loading, so that it doesn't pull down all the remote JSON - self.remote_banners(banners, self.os) - - # Rewrite the cached banners each run, since writing is faster than the banner_cache validation portion - self.save_banners(banners) - - if progress_callback is not None: - progress_callback(100, f"Built {self.os} caches") + self._cache.update(progress_callback) @classmethod - def read_new_banners(cls, context: interfaces.context.ContextInterface, config_path: str, new_urls: List[str], - symbol_name: str, operating_system: str = None, - progress_callback = None) -> Optional[Dict[bytes, List[str]]]: - """Reads the any new banners for the OS in question""" - if operating_system is None: - return None - - banners = {} - - total = len(new_urls) - if total > 0: - vollog.info(f"Building {operating_system} caches...") - for current in range(total): - if progress_callback is not None: - progress_callback(current * 100 / total, f"Building {operating_system} caches") - isf_url = new_urls[current] - - isf = None - try: - # Loading the symbol table will be very slow until it's been validated - isf = intermed.IntermediateSymbolTable(context, config_path, "temp", isf_url, validate = False) - - # We should store the banner against the filename - # We don't bother with the hash (it'll likely take too long to validate) - # but we should check at least that the banner matches on load. - banner = isf.get_symbol(symbol_name).constant_data - vollog.log(constants.LOGLEVEL_VV, f"Caching banner {banner} for file {isf_url}") - - bannerlist = banners.get(banner, []) - bannerlist.append(isf_url) - banners[banner] = bannerlist - except exceptions.SymbolError: - pass - except json.JSONDecodeError: - vollog.log(constants.LOGLEVEL_VV, f"Caching file {isf_url} failed due to JSON error") - finally: - # Get rid of the loaded file, in case it sits in memory - if isf: - del isf - gc.collect() - return banners - - @classmethod - def find_new_banner_files(cls, banners: Dict[bytes, List[str]], operating_system: str) -> List[str]: - """Gathers all files and remove existing banners""" - cacheables = list(intermed.IntermediateSymbolTable.file_symbol_url(operating_system)) - for banner in banners: - for json_file in banners[banner]: - if json_file in cacheables: - cacheables.remove(json_file) - return cacheables - - @classmethod - def remote_banners(cls, banners: Dict[bytes, List[str]], operating_system = None, banner_location = None): - """Adds remote URLs to the banner list""" - if operating_system is None: - return None - - if banner_location is None: - banner_location = constants.REMOTE_ISF_URL - - if not constants.OFFLINE and banner_location is not None: - try: - rbf = RemoteBannerFormat(banner_location) - rbf.process(banners, operating_system) - except urllib.error.URLError: - vollog.debug(f"Unable to download remote banner list from {banner_location}") + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + """Returns a list of RequirementInterface objects required by this + object.""" + return [requirements.VersionRequirement(name = 'SQLiteCache', component = SqliteCache, version = (1, 0, 0))] -class RemoteBannerFormat: +class RemoteIdentifierFormat: def __init__(self, location: str): self._location = location with resources.ResourceAccessor().open(url = location) as fp: self._data = json.load(fp) if not self._verify(): - raise ValueError("Unsupported version for remote banner list format") + raise ValueError("Unsupported version for remote identifier list format") def _verify(self) -> bool: version = self._data.get('version', 0) @@ -188,23 +350,23 @@ class RemoteBannerFormat: return True return False - def process(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): - raise ValueError("Banner List version not verified") + def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]): + raise ValueError("Identifier List version not verified") - def process_v1(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): + def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]): if operating_system in self._data: - for banner in self._data[operating_system]: - binary_banner = base64.b64decode(banner) - file_list = banners.get(binary_banner, []) - for value in self._data[operating_system][banner]: + for identifier in self._data[operating_system]: + binary_identifier = base64.b64decode(identifier) + file_list = identifiers.get(binary_identifier, []) + for value in self._data[operating_system][identifier]: if value not in file_list: file_list = file_list + [value] - banners[binary_banner] = file_list + identifiers[binary_identifier] = file_list if 'additional' in self._data: for location in self._data['additional']: try: - subrbf = RemoteBannerFormat(location) - subrbf.process(banners, operating_system) + subrbf = RemoteIdentifierFormat(location) + subrbf.process(identifiers, operating_system) except IOError: vollog.debug(f"Remote file not found: {location}") - return banners + return identifiers diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 143abd02e..610ed0e18 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -3,9 +3,9 @@ # import logging -from typing import Any, Iterable, List, Tuple, Type, Optional, Callable +from typing import Any, Callable, Iterable, List, Optional, Tuple -from volatility3.framework import interfaces, constants, layers +from volatility3.framework import constants, interfaces, layers from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners @@ -18,7 +18,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): priority = 40 banner_config_key: str = "banner" - banner_cache: Optional[Type[symbol_cache.SymbolBannerCache]] = None + operating_system: Optional[str] = None symbol_class: Optional[str] = None find_aslr: Optional[Callable] = None @@ -27,14 +27,21 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): self._requirements: List[Tuple[str, interfaces.configuration.RequirementInterface]] = [] self._banners: symbol_cache.BannersType = {} + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.VersionRequirement(name = 'SQLiteCache', + component = symbol_cache.SqliteCache, + version = (1, 0, 0)) + ] + @property def banners(self) -> symbol_cache.BannersType: """Creates a cached copy of the results, but only it's been requested.""" if not self._banners: - if not self.banner_cache: - raise RuntimeError(f"Cache has not been properly defined for {self.__class__.__name__}") - self._banners = self.banner_cache.load_banners() + cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system) return self._banners def __call__(self, @@ -103,8 +110,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): vollog.debug(f"Identified banner: {repr(banner)}") symbol_files = self.banners.get(banner, None) if symbol_files: - isf_path = symbol_files[0] - vollog.debug(f"Using symbol library: {symbol_files[0]}") + isf_path = symbol_files + vollog.debug(f"Using symbol library: {symbol_files}") clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join @@ -117,7 +124,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): break else: if symbol_files: - vollog.debug(f"Symbol library path not found: {symbol_files[0]}") + vollog.debug(f"Symbol library path not found: {symbol_files}") # print("Kernel", banner, hex(banner_offset)) else: vollog.debug("No existing banners found") diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 4edc6d17c..b31c4767f 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -408,13 +408,19 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type config_path = interfaces.configuration.path_join(config_path, self.name) - if len(self._version) > 0 and self._component.version[0] != self._version[0]: - return {config_path: self} - if len(self._version) > 1 and self._component.version[1] < self._version[1]: + if not self.matches_required(self._version, self._component.version): return {config_path: self} context.config[interfaces.configuration.path_join(config_path, self.name)] = True return {} + @classmethod + def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]): + if len(required) > 0 and version[0] != required[0]: + return False + if len(required) > 1 and version[1] < required[1]: + return False + return True + class PluginRequirement(VersionRequirement): diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index f08819f29..322e574e1 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -68,10 +68,13 @@ if sys.platform == 'win32': os.makedirs(CACHE_PATH, exist_ok = True) LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") -""""Default location to record information about available linux banners""" +"""Default location to record information about available linux banners""" MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") -""""Default location to record information about available mac banners""" +"""Default location to record information about available mac banners""" + +IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") +"""Default location to record information about available identifiers""" BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index c96c9bdbe..713f91da0 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -9,9 +9,9 @@ that a user has not filled. """ import logging from abc import ABCMeta -from typing import Any, List, Optional, Tuple, Union, Type +from typing import Any, List, Optional, Tuple, Type, Union -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -47,9 +47,10 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla super().__init__(context, config_path) for requirement in self.get_requirements(): if not isinstance(requirement, (interfaces.configuration.SimpleTypeRequirement, - requirements.ChoiceRequirement, requirements.ListRequirement)): + requirements.ChoiceRequirement, requirements.ListRequirement, + requirements.VersionRequirement)): raise TypeError( - "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement or ListRequirement") + "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement, ListRequirement or VersionRequirement") def __call__(self, context: interfaces.context.ContextInterface, diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 575f25426..b2960733d 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -1,17 +1,16 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import base64 import json import logging import os import pathlib import zipfile -from typing import List, Type, Any, Generator +from typing import Generator, List from volatility3 import schemas, symbols -from volatility3.framework import interfaces, renderers, constants -from volatility3.framework.automagic import mac, linux, symbol_cache +from volatility3.framework import constants, interfaces, renderers +from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import resources @@ -23,7 +22,7 @@ class IsfInfo(plugins.PluginInterface): """Determines information about the currently available ISF files, or a specific one""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -39,7 +38,10 @@ class IsfInfo(plugins.PluginInterface): requirements.BooleanRequirement(name = 'validate', description = 'Validate against schema if possible', default = False, - optional = True) + optional = True), + requirements.VersionRequirement(name = 'SQLiteCache', + component = symbol_cache.SqliteCache, + version = (1, 0, 0)) ] @classmethod @@ -62,14 +64,6 @@ class IsfInfo(plugins.PluginInterface): if filename.endswith(extension): yield pathlib.Path(base_name).as_uri() - def _get_banner(self, clazz: Type[symbol_cache.SymbolBannerCache], data: Any) -> str: - """Gets a banner from an ISF file""" - banner_symbol = data.get('symbols', {}).get(clazz.symbol_name, {}).get('constant_data', - renderers.NotAvailableValue()) - if not isinstance(banner_symbol, interfaces.renderers.BaseAbsentValue): - banner_symbol = str(base64.b64decode(banner_symbol), encoding = 'latin-1') - return banner_symbol - def _generator(self): if self.config.get('isf', None) is not None: file_list = [self.config['isf']] @@ -101,7 +95,6 @@ class IsfInfo(plugins.PluginInterface): # Process the filtered list for entry in filtered_list: num_types = num_enums = num_bases = num_symbols = 0 - windows_info = linux_banner = mac_banner = renderers.NotAvailableValue() valid = "Unknown" with resources.ResourceAccessor().open(url = entry) as fp: try: @@ -111,20 +104,20 @@ class IsfInfo(plugins.PluginInterface): num_enums = len(data.get('enums', [])) num_bases = len(data.get('base_types', [])) - linux_banner = self._get_banner(linux.LinuxBannerCache, data) - mac_banner = self._get_banner(mac.MacBannerCache, data) - if not linux_banner and not mac_banner: - windows_info = os.path.splitext(os.path.basename(entry))[0] + identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifier = identifier_cache.get_identifier(location = entry) + if identifier: + identifier = identifier.decode('utf-8', errors = 'replace') + else: + identifier = renderers.NotAvailableValue() valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {entry}") - yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, windows_info, linux_banner, - mac_banner)) + yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) # Try to open the file, load it as JSON, read the data from it def run(self): return renderers.TreeGrid([("URI", str), ("Valid", str), ("Number of base_types", int), ("Number of types", int), ("Number of symbols", int), - ("Number of enums", int), ("Windows info", str), ("Linux banner", str), - ("Mac banner", str)], self._generator()) + ("Number of enums", int), ("Identifying infomration", str)], self._generator()) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index a6a7a0fae..1fceb1bcc 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -202,8 +202,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): pass # Finally try looking in zip files - zip_path = os.path.join(path, sub_path + ".zip") - if os.path.exists(zip_path): + for zip_path in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + '.zip'): # We have a zipfile, so run through it and look for sub files that match the filename with zipfile.ZipFile(zip_path) as zfile: for name in zfile.namelist(): diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 41037d464..af3741bbe 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -14,6 +14,8 @@ from urllib import parse, request from volatility3 import symbols from volatility3.framework import constants, contexts, exceptions, interfaces +from volatility3.framework.automagic import symbol_cache +from volatility3.framework.configuration import requirements from volatility3.framework.configuration.requirements import SymbolTableRequirement from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbconv @@ -74,9 +76,15 @@ class PDBUtility(interfaces.configuration.VersionableInterface): isf_path = None # Take the first result of search for the intermediate file - for value in intermed.IntermediateSymbolTable.file_symbol_url("windows", filter_string): + if not requirements.VersionRequirement.matches_required((1, 0, 0), symbol_cache.SqliteCache.version): + vollog.debug(f"Required version of SQLiteCache not found") + return None + + value = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).find_location( + symbol_cache.WindowsIdentifier.generate(pdb_name.strip('\x00'), guid.upper(), age), 'windows') + + if value: isf_path = value - break else: # If none are found, attempt to download the pdb, convert it and try again cls.download_pdb_isf(context, guid.upper(), age, pdb_name, progress_callback) @@ -336,46 +344,12 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - module_name = guid["pdb_name"].strip('.pdb') - - symbol_table_name = cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) - - new_module_name = None - if create_module: - new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], - symbol_table_name = symbol_table_name) - new_module_name = new_module.name - - return new_module_name, symbol_table_name - - @classmethod - def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, - pdb_name: str, module_offset: int = None, module_size: int = None) -> str: - """Creates a module in the specified layer_name based on a pdb name. - - Searches the memory section of the loaded module for its PDB GUID - and loads the associated symbol table into the symbol space. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - config_path: The config path where to find symbol files - layer_name: The name of the layer on which to operate - module_offset: This memory dump's module image offset - module_size: The size of the module for this dump - - Returns: - The name of the constructed and loaded symbol table - """ - - module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, - module_size, create_module = True) - - return module_name + return cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) class PdbSignatureScanner(interfaces.layers.ScannerInterface): From 2729d25d89576b3d31785c1326671eb86455495e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 00:40:01 +0000 Subject: [PATCH 311/404] Automagic: speed up caching by db commit when necessary --- .../framework/automagic/symbol_cache.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index fe717b8be..4b27b8e0f 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -10,7 +10,7 @@ import urllib import urllib.parse import urllib.request from abc import abstractmethod -from typing import Dict, Generator, List, Optional +from typing import Dict, Generator, List, Optional, Tuple import volatility3.framework import volatility3.schemas @@ -158,10 +158,11 @@ class SqliteCache(CacheManagerInterface): self._database = self._connect_storage(filename) def _connect_storage(self, path: str): - database = sqlite3.connect(path, isolation_level = None) + database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + database.commit() return database def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: @@ -229,6 +230,7 @@ class SqliteCache(CacheManagerInterface): counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) + cursor = self._database.cursor() for location in files_to_process: # Open location counter += 1 @@ -246,7 +248,7 @@ class SqliteCache(CacheManagerInterface): if identifier is not None: # We don't try to validate schemas here, we do that on first use # Store in database - self._database.cursor().execute( + cursor.execute( "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", ( location, @@ -256,7 +258,7 @@ class SqliteCache(CacheManagerInterface): )) vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") else: - self._database.cursor().execute( + cursor.execute( "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", ( location, @@ -267,21 +269,27 @@ class SqliteCache(CacheManagerInterface): vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") except Exception as excp: vollog.log(constants.LOGLEVEL_VVVV, excp) + self._database.commit() if not constants.OFFLINE and constants.REMOTE_ISF_URL: + progress_callback(0, 'Reading remote ISF list') remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) + progress_callback(50, 'Reading remote ISF list') + cursor = self._database.cursor() for operating_system in ['mac', 'linux', 'windows']: identifiers = remote_identifiers.process({}, operating_system = operating_system) - for identifier in identifiers: - for location in identifiers[identifier]: - self._database.cursor().execute( - "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now')", - (location, identifier, operating_system, False) - ) + for identifier, location in identifiers: + cursor.execute( + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + (location, identifier, operating_system, False) + ) + progress_callback(100, 'Reading remote ISF list') + self._database.commit() if missing_locations: self._database.cursor().execute( f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ Dict[bytes, str]: @@ -350,23 +358,23 @@ class RemoteIdentifierFormat: return True return False - def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]): + def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]) -> Generator[ + Tuple[bytes, str], None, None]: raise ValueError("Identifier List version not verified") - def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]): + def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]) -> Generator[ + Tuple[bytes, str], None, None]: if operating_system in self._data: for identifier in self._data[operating_system]: binary_identifier = base64.b64decode(identifier) file_list = identifiers.get(binary_identifier, []) for value in self._data[operating_system][identifier]: - if value not in file_list: - file_list = file_list + [value] - identifiers[binary_identifier] = file_list + yield binary_identifier, value if 'additional' in self._data: for location in self._data['additional']: try: subrbf = RemoteIdentifierFormat(location) - subrbf.process(identifiers, operating_system) + yield from subrbf.process(identifiers, operating_system) except IOError: vollog.debug(f"Remote file not found: {location}") return identifiers From 57a202ae1d69de5968a6a49e9bc199724d364152 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 01:04:32 +0000 Subject: [PATCH 312/404] Automagic: Use cache delay for remote locations --- volatility3/framework/automagic/symbol_cache.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 4b27b8e0f..3c7049986 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -149,6 +149,8 @@ class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) + cache_period = '-3 days' + def __init__(self, filename: str): super().__init__(filename) try: @@ -220,13 +222,15 @@ class SqliteCache(CacheManagerInterface): files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " - "AND cached < date('now', '-3 days');") + f"AND cached < date('now', {self.cache_period});") for row in result: if row['location'] in files_to_timestamp: cache_update.add(row['location']) idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + # New or not recently updated + counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) @@ -271,11 +275,15 @@ class SqliteCache(CacheManagerInterface): vollog.log(constants.LOGLEVEL_VVVV, excp) self._database.commit() + # Remote Entries + if not constants.OFFLINE and constants.REMOTE_ISF_URL: progress_callback(0, 'Reading remote ISF list') + cursor = self._database.cursor() + cursor.execute( + f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') - cursor = self._database.cursor() for operating_system in ['mac', 'linux', 'windows']: identifiers = remote_identifiers.process({}, operating_system = operating_system) for identifier, location in identifiers: @@ -286,6 +294,8 @@ class SqliteCache(CacheManagerInterface): progress_callback(100, 'Reading remote ISF list') self._database.commit() + # Missing entries + if missing_locations: self._database.cursor().execute( f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) From fe466386406556a17ba2f474558257e4bb4e8457 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 01:36:17 +0000 Subject: [PATCH 313/404] Automagic: Update to use more recent OS categories --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 3c7049986..8bbedf3e8 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -284,7 +284,7 @@ class SqliteCache(CacheManagerInterface): f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') - for operating_system in ['mac', 'linux', 'windows']: + for operating_system in constants.OS_CATEGORIES: identifiers = remote_identifiers.process({}, operating_system = operating_system) for identifier, location in identifiers: cursor.execute( From 371267f38a61f03007bde4f880b9c45a4b4c2e41 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 21:50:08 +0000 Subject: [PATCH 314/404] Automagic: Ensure partial caching survives --- .../framework/automagic/symbol_cache.py | 80 ++++++++++--------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 8bbedf3e8..54ee13ca2 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -222,7 +222,7 @@ class SqliteCache(CacheManagerInterface): files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " - f"AND cached < date('now', {self.cache_period});") + f"AND cached < date('now', '{self.cache_period}');") for row in result: if row['location'] in files_to_timestamp: cache_update.add(row['location']) @@ -235,45 +235,47 @@ class SqliteCache(CacheManagerInterface): files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) cursor = self._database.cursor() - for location in files_to_process: - # Open location - counter += 1 - progress_callback(counter * 100 / number_files_to_process, - "Updating caches for {number_files_to_process} files...") - try: - with resources.ResourceAccessor().open(location) as fp: - json_obj = json.load(fp) - identifier = None - for idextractor in idextractors: - identifier = idextractor.get_identifier(json_obj) - operating_system = idextractor.operating_system + try: + for location in files_to_process: + # Open location + counter += 1 + progress_callback(counter * 100 / number_files_to_process, + f"Updating caches for {number_files_to_process} files...") + try: + with resources.ResourceAccessor().open(location) as fp: + json_obj = json.load(fp) + identifier = None + for idextractor in idextractors: + identifier = idextractor.get_identifier(json_obj) + operating_system = idextractor.operating_system + if identifier is not None: + break if identifier is not None: - break - if identifier is not None: - # We don't try to validate schemas here, we do that on first use - # Store in database - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - identifier, - operating_system, - self.is_url_local(location) - )) - vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") - else: - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - None, - None, - self.is_url_local(location) - )) - vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") - except Exception as excp: - vollog.log(constants.LOGLEVEL_VVVV, excp) - self._database.commit() + # We don't try to validate schemas here, we do that on first use + # Store in database + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") + else: + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + None, + None, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") + except Exception as excp: + vollog.log(constants.LOGLEVEL_VVVV, excp) + finally: + self._database.commit() # Remote Entries From d16861b5925a473c0bf36a0949bc052321197399 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 22:15:35 +0000 Subject: [PATCH 315/404] Documentation: Update documentation for isf caching feature --- doc/source/symbol-tables.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 4dea6077d..d41e8797a 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -12,20 +12,20 @@ Volatility will automatically decompress them on use. It will also cache their under the user's home directory, in :file:`.cache/volatility3`, along with other useful data. The cache directory currently cannot be altered. -Symbol table JSON files live, by default, under the :file:`volatility3/symbols`, underneath an operating system directory -(currently one of :file:`windows`, :file:`mac` or :file:`linux`). The symbols directory is configurable within the framework and can -usually be set within the user interface. +Symbol table JSON files live, by default, under the :file:`volatility3/symbols` directory. The symbols directory is +configurable within the framework and can usually be set within the user interface. These files can also be compressed into ZIP files, which Volatility will process in order to locate symbol files. -The ZIP file must be named after the appropriate operating system (such as `linux.zip`, `mac.zip` or `windows.zip`). -Inside the ZIP file, the directory structure should match the uncompressed operating system directory. + +Volatility maintains a cache mapping the appropriate identifier for each symbol file against its filename. This cache +is update by automagic called as part of the standard automagic that's run each time a plugin is run. Windows symbol tables --------------------- For Windows systems, Volatility accepts a string made up of the GUID and Age of the required PDB file. It then -searches all files under the configured symbol directories under the windows subdirectory. Any that match the filename -pattern of :file:`/-.json` (or any compressed variant) will be used. If such a symbol table cannot be found, then +searches all files under the configured symbol directories under the windows subdirectory. Any that contain metadata +which matches the pdb name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then the associated PDB file will be downloaded from Microsoft's Symbol Server and converted into the appropriate JSON format, and will be saved in the correct location. @@ -41,11 +41,10 @@ or a virtual environment. Mac/Linux symbol tables ----------------------- -For Mac/Linux systems, both use the same mechanism for identification. JSON files live under the symbol directories, -under either the :file:`linux` or :file:`mac` directories. The generated files contain an identifying string (the operating system +For Mac/Linux systems, both use the same mechanism for identification. The generated files contain an identifying string (the operating system banner), which Volatility's automagic can detect. Volatility caches the mapping between the strings and the symbol tables they come from, meaning the precise file names don't matter and can be organized under any necessary hierarchy -under the operating system directory. +under the symbols directory. Linux and Mac symbol tables can be generated from a DWARF file using a tool called `dwarf2json `_. Currently a kernel with debugging symbols is the only suitable means for recovering all the information required by From 2d64deb18ec0b341a40f416429da3e8b0d1ddb44 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 22:52:04 +0000 Subject: [PATCH 316/404] Plugins: Update isfinfo to use the cache unless --live --- .../framework/automagic/symbol_cache.py | 96 +++++++++++++++---- volatility3/framework/constants/__init__.py | 3 + volatility3/framework/plugins/isfinfo.py | 56 ++++++----- 3 files changed, 112 insertions(+), 43 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 54ee13ca2..c09904713 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -104,7 +104,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns the location of the symbol file given the identifier Args: - identifier: string that uniquely identifies a particular symbolt table + identifier: string that uniquely identifies a particular symbol table operating_system: optional string to restrict identifiers to just those for a particular operating system Returns: @@ -144,6 +144,18 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns all identifiers for a particular operating system""" pass + def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + """Returns ISF statistics based on the location + + Returns: + A tuple of base_types, types, enums, symbols, or None is location not found""" + + def get_verified(self, location: str) -> bool: + """Returns whether a location ISF has been verified against its schema""" + + def set_verified(self, location: str, state: bool = True) -> None: + """Sets the verified state of a location based on whether it has been successfully verified against its schema""" + class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) @@ -163,7 +175,23 @@ class SqliteCache(CacheManagerInterface): database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( - 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCEMA_VERSION})') + schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() + if not schema_version: + database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCEMA_VERSION})') + elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCEMA_VERSION: + # All good, so pass and move on + pass + else: + vollog.info(f"Previous cache schema version found: {schema_version['schema_version']}") + # TODO: Implement code if the schema changes + # Current this should never happen so we start over again + database.close() + os.unlink(path) + return self._connect_storage(path) + database.cursor().execute( + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, verified BOOL DEFAULT False,' + 'stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)') database.commit() return database @@ -207,6 +235,25 @@ class SqliteCache(CacheManagerInterface): return row['identifier'] return None + def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + results = self._database.cursor().execute( + 'SELECT stats_base_types, stats_types, stats_enums, stats_symbols FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['stats_base_types'], row['stats_types'], row['stats_enums'], row['stats_symbols'] + return None + + def get_verified(self, location: str) -> bool: + results = self._database.cursor().execute('SELECT verified FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['verified'] + return False + + def set_verified(self, location: str, state: bool = True) -> None: + self._database.cursor().execute('UPDATE cache (verified) VALUES (?) WHERE location = ?', + (state, location,)) + def update(self, progress_callback = None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. @@ -245,32 +292,39 @@ class SqliteCache(CacheManagerInterface): with resources.ResourceAccessor().open(location) as fp: json_obj = json.load(fp) identifier = None + + # Get stats + stats_base_types = len(json_obj.get('base_types', {})) + stats_types = len(json_obj.get('types', {})) + stats_enums = len(json_obj.get('enums', {})) + stats_symbols = len(json_obj.get('symbols', {})) + + operating_system = None for idextractor in idextractors: identifier = idextractor.get_identifier(json_obj) - operating_system = idextractor.operating_system if identifier is not None: + operating_system = idextractor.operating_system break + + # We don't try to validate schemas here, we do that on first use + # Store in database + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, " + "stats_base_types, stats_types, stats_enums, stats_symbols, " + "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + stats_base_types, + stats_types, + stats_enums, + stats_symbols, + self.is_url_local(location) + )) if identifier is not None: - # We don't try to validate schemas here, we do that on first use - # Store in database - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - identifier, - operating_system, - self.is_url_local(location) - )) vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") else: - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - None, - None, - self.is_url_local(location) - )) vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") except Exception as excp: vollog.log(constants.LOGLEVEL_VVVV, excp) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 322e574e1..3b499adea 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -76,6 +76,9 @@ MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") """Default location to record information about available identifiers""" +CACHE_SQLITE_SCEMA_VERSION = 1 +"""Version for the sqlite3 cache schema""" + BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" ProgressCallback = Optional[Callable[[float, str], None]] diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index b2960733d..b94cfd69a 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -41,7 +41,11 @@ class IsfInfo(plugins.PluginInterface): optional = True), requirements.VersionRequirement(name = 'SQLiteCache', component = symbol_cache.SqliteCache, - version = (1, 0, 0)) + version = (1, 0, 0)), + requirements.BooleanRequirement(name = 'live', + description = 'Traverse all files, rather than use the cache', + default = False, + optional = True) ] @classmethod @@ -92,28 +96,36 @@ class IsfInfo(plugins.PluginInterface): def check_valid(data): return "Unknown" - # Process the filtered list - for entry in filtered_list: - num_types = num_enums = num_bases = num_symbols = 0 - valid = "Unknown" - with resources.ResourceAccessor().open(url = entry) as fp: - try: - data = json.load(fp) - num_symbols = len(data.get('symbols', [])) - num_types = len(data.get('user_types', [])) - num_enums = len(data.get('enums', [])) - num_bases = len(data.get('base_types', [])) + if self.config['live']: + # Process the filtered list + for entry in filtered_list: + num_types = num_enums = num_bases = num_symbols = 0 + valid = "Unknown" + with resources.ResourceAccessor().open(url = entry) as fp: + try: + data = json.load(fp) + num_symbols = len(data.get('symbols', [])) + num_types = len(data.get('user_types', [])) + num_enums = len(data.get('enums', [])) + num_bases = len(data.get('base_types', [])) - identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) - identifier = identifier_cache.get_identifier(location = entry) - if identifier: - identifier = identifier.decode('utf-8', errors = 'replace') - else: - identifier = renderers.NotAvailableValue() - valid = check_valid(data) - except (UnicodeDecodeError, json.decoder.JSONDecodeError): - vollog.warning(f"Invalid ISF: {entry}") - yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) + identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifier = identifier_cache.get_identifier(location = entry) + if identifier: + identifier = identifier.decode('utf-8', errors = 'replace') + else: + identifier = renderers.NotAvailableValue() + valid = check_valid(data) + except (UnicodeDecodeError, json.decoder.JSONDecodeError): + vollog.warning(f"Invalid ISF: {entry}") + yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) + else: + cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + valid = 'Unknown' + for identifier, location in cache.get_identifier_dictionary().items(): + num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) + if identifier: + yield (0, (location, valid, num_bases, num_types, num_symbols, num_enums, str(identifier))) # Try to open the file, load it as JSON, read the data from it From 1f02fea5d10be5c193f2b100bb18c973335504be Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 23:40:47 +0000 Subject: [PATCH 317/404] Automagic: Change database to store ISF hash instead of verified state --- .../framework/automagic/symbol_cache.py | 27 ++++++++----------- volatility3/framework/plugins/isfinfo.py | 12 +++++++++ volatility3/schemas/__init__.py | 16 +++++++++-- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index c09904713..77bc46265 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -14,6 +14,7 @@ from typing import Dict, Generator, List, Optional, Tuple import volatility3.framework import volatility3.schemas +from volatility3 import schemas from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -150,11 +151,8 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A tuple of base_types, types, enums, symbols, or None is location not found""" - def get_verified(self, location: str) -> bool: - """Returns whether a location ISF has been verified against its schema""" - - def set_verified(self, location: str, state: bool = True) -> None: - """Sets the verified state of a location based on whether it has been successfully verified against its schema""" + def get_hash(self, location: str) -> bool: + """Returns the hash of the JSON from within a location ISF""" class SqliteCache(CacheManagerInterface): @@ -190,7 +188,7 @@ class SqliteCache(CacheManagerInterface): os.unlink(path) return self._connect_storage(path) database.cursor().execute( - 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, verified BOOL DEFAULT False,' + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, hash TEXT,' 'stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)') database.commit() return database @@ -243,16 +241,11 @@ class SqliteCache(CacheManagerInterface): return row['stats_base_types'], row['stats_types'], row['stats_enums'], row['stats_symbols'] return None - def get_verified(self, location: str) -> bool: - results = self._database.cursor().execute('SELECT verified FROM cache WHERE location = ?', + def get_hash(self, location: str) -> Optional[str]: + results = self._database.cursor().execute('SELECT hash FROM cache WHERE location = ?', (location,)).fetchall() for row in results: - return row['verified'] - return False - - def set_verified(self, location: str, state: bool = True) -> None: - self._database.cursor().execute('UPDATE cache (verified) VALUES (?) WHERE location = ?', - (state, location,)) + return row['hash'] def update(self, progress_callback = None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. @@ -291,6 +284,7 @@ class SqliteCache(CacheManagerInterface): try: with resources.ResourceAccessor().open(location) as fp: json_obj = json.load(fp) + hash = schemas.create_json_hash(json_obj) identifier = None # Get stats @@ -309,13 +303,14 @@ class SqliteCache(CacheManagerInterface): # We don't try to validate schemas here, we do that on first use # Store in database cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, " + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, hash," "stats_base_types, stats_types, stats_enums, stats_symbols, " - "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", + "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", ( location, identifier, operating_system, + hash, stats_base_types, stats_types, stats_enums, diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index b94cfd69a..af095b69d 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -125,6 +125,18 @@ class IsfInfo(plugins.PluginInterface): for identifier, location in cache.get_identifier_dictionary().items(): num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) if identifier: + json_hash = cache.get_hash(location) + if json_hash and json_hash in schemas.cached_validations: + valid = 'True (cached)' + if self.config['validate']: + # Even if we're not live, if we've been explicitly asked to validate, then do-so + with resources.ResourceAccessor().open(url = location) as fp: + try: + data = json.load(fp) + valid = check_valid(data) + except (UnicodeDecodeError, json.decoder.JSONDecodeError): + vollog.warning(f"Invalid ISF: {location}") + yield (0, (location, valid, num_bases, num_types, num_symbols, num_enums, str(identifier))) # Try to open the file, load it as JSON, read the data from it diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 65329a4f5..8666680b3 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -6,7 +6,7 @@ import hashlib import json import logging import os -from typing import Set, Any, Dict +from typing import Any, Dict, Optional, Set from volatility3.framework import constants @@ -51,9 +51,21 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: return valid(input, schema, use_cache) -def create_json_hash(input: Dict[str, Any], schema: Dict[str, Any]) -> str: +def create_json_hash(input: Dict[str, Any], schema: Optional[Dict[str, Any]] = None) -> Optional[str]: """Constructs the hash of the input and schema to create a unique identifier for a particular JSON file.""" + if schema is None: + format = input.get('metadata', {}).get('format', None) + if not format: + vollog.debug("No schema format defined") + return None + basepath = os.path.abspath(os.path.dirname(__file__)) + schema_path = os.path.join(basepath, 'schema-' + format + '.json') + if not os.path.exists(schema_path): + vollog.debug(f"Schema for format not found: {schema_path}") + return None + with open(schema_path, 'r') as s: + schema = json.load(s) return hashlib.sha1(bytes(json.dumps((input, schema), sort_keys = True), 'utf-8')).hexdigest() From bda200168a378f79c76acacf93edb6500f766e55 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sat, 28 May 2022 23:49:15 +0100 Subject: [PATCH 318/404] Update volatility3/framework/plugins/isfinfo.py Yep, good spot as ever, thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/plugins/isfinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index af095b69d..6b13f10b6 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -144,4 +144,4 @@ class IsfInfo(plugins.PluginInterface): def run(self): return renderers.TreeGrid([("URI", str), ("Valid", str), ("Number of base_types", int), ("Number of types", int), ("Number of symbols", int), - ("Number of enums", int), ("Identifying infomration", str)], self._generator()) + ("Number of enums", int), ("Identifying information", str)], self._generator()) From ebab09e53a0c56632edc45260e013b42f8097af2 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sat, 28 May 2022 23:50:23 +0100 Subject: [PATCH 319/404] Update volatility3/framework/automagic/symbol_cache.py Cool, I always forget about that, I think it's just what I'm used to, thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 77bc46265..2908774c0 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -276,7 +276,7 @@ class SqliteCache(CacheManagerInterface): number_files_to_process = len(files_to_process) cursor = self._database.cursor() try: - for location in files_to_process: + for counter, location in enumerate(files_to_process): # Open location counter += 1 progress_callback(counter * 100 / number_files_to_process, From a4aa93f05945ab3c3778a25a0c0d3e4709e62c01 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 28 May 2022 23:51:54 +0100 Subject: [PATCH 320/404] Core: Clean up unneeded counter variable, now we're using enumerate --- volatility3/framework/automagic/symbol_cache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 2908774c0..156a1e8c2 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -271,14 +271,12 @@ class SqliteCache(CacheManagerInterface): # New or not recently updated - counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) cursor = self._database.cursor() try: for counter, location in enumerate(files_to_process): # Open location - counter += 1 progress_callback(counter * 100 / number_files_to_process, f"Updating caches for {number_files_to_process} files...") try: From 1e80bb54deb5c8a7cf82057f996bf197058828e7 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:34:44 +0100 Subject: [PATCH 321/404] Update volatility3/framework/configuration/requirements.py Yep, not sure why I forgot, thanks 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index b31c4767f..cc4f05ae6 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -414,7 +414,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): return {} @classmethod - def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]): + def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]) -> bool: if len(required) > 0 and version[0] != required[0]: return False if len(required) > 1 and version[1] < required[1]: From 6f34e1350e67ca893d0f1c5984c45813d9892b5a Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:35:42 +0100 Subject: [PATCH 322/404] Update volatility3/framework/automagic/symbol_cache.py Hehehe, I guess I'm just a little shy about handing out complex objects, but you're right and it is a private method. 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 156a1e8c2..efe1ce601 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -169,7 +169,7 @@ class SqliteCache(CacheManagerInterface): os.unlink(filename) self._database = self._connect_storage(filename) - def _connect_storage(self, path: str): + def _connect_storage(self, path: str) -> sqlite3.Connection: database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( From 98f7fe17433b11a2ef3950e87fabaab8e757e67d Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:49:20 +0100 Subject: [PATCH 323/404] Update volatility3/framework/automagic/symbol_cache.py Quite right, thanks for the catch! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index efe1ce601..47ff66121 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -151,7 +151,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A tuple of base_types, types, enums, symbols, or None is location not found""" - def get_hash(self, location: str) -> bool: + def get_hash(self, location: str) -> Optional[str]: """Returns the hash of the JSON from within a location ISF""" From 504229e46886d9f6d8d3c6a6b8782d67e3656600 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 May 2022 10:52:29 +0100 Subject: [PATCH 324/404] Automagic: include fixes from @digitalisx on review --- volatility3/framework/automagic/symbol_cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 47ff66121..378424ef5 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -10,7 +10,7 @@ import urllib import urllib.parse import urllib.request from abc import abstractmethod -from typing import Dict, Generator, List, Optional, Tuple +from typing import Dict, Generator, Iterable, List, Optional, Tuple import volatility3.framework import volatility3.schemas @@ -113,7 +113,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """ pass - def get_local_locations(self) -> List[str]: + def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" pass @@ -141,7 +141,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns an identifier based on a specific location or None""" pass - def get_identifiers(self, operating_system: Optional[str]): + def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" pass @@ -369,7 +369,7 @@ class SqliteCache(CacheManagerInterface): output[row['identifier']] = row['location'] return output - def get_identifiers(self, operating_system: Optional[str]): + def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: if operating_system: results = self._database.cursor().execute('SELECT identifier FROM cache WHERE operating_system = ?', (operating_system,)).fetchall() From 47accf520bb322040e0cbf1facfa74e32dc944bb Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:55:17 +0100 Subject: [PATCH 325/404] Update volatility3/framework/automagic/symbol_cache.py Yep, you're quite right, not sure how that got left behind. Thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 378424ef5..ed64746fd 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -426,7 +426,6 @@ class RemoteIdentifierFormat: if operating_system in self._data: for identifier in self._data[operating_system]: binary_identifier = base64.b64decode(identifier) - file_list = identifiers.get(binary_identifier, []) for value in self._data[operating_system][identifier]: yield binary_identifier, value if 'additional' in self._data: From c475b792a53305fe7769134f46d1cf502e29e9b6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jun 2022 17:34:22 +0100 Subject: [PATCH 326/404] Automgic: Fix removing stale entries --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index ed64746fd..46431b773 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -347,7 +347,7 @@ class SqliteCache(CacheManagerInterface): if missing_locations: self._database.cursor().execute( - f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", [x for x in missing_locations]) self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ From 225c36631403fe3fa58208befc9d36ad93424b83 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jun 2022 18:54:00 +0100 Subject: [PATCH 327/404] Windows: Update PDB to store correct age value --- volatility3/framework/symbols/windows/pdbconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index da8254ffd..15b5c733a 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -521,7 +521,7 @@ class PdbReader: self.metadata['windows']['pdb'] = { "GUID": self.convert_bytes_to_guid(pdb_info.GUID), - "age": pdb_info.age, + "age": self._dbiheader.age, "database": self._database_name or 'unknown.pdb', "machine_type": self._dbiheader.machine } From ae48a8ab479cc1f60550eef6fe9502b0d49f2e74 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 20 Jul 2022 20:40:23 +0100 Subject: [PATCH 328/404] Documentation: Update text about long cache updates --- README.md | 3 +++ doc/source/symbol-tables.rst | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9f9c1bbb7..348121e44 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,9 @@ Symbol tables zip files must be placed, as named, into the `volatility3/symbols` Windows symbols that cannot be found will be queried, downloaded, generated and cached. Mac and Linux symbol tables must be manually produced by a tool such as [dwarf2json](https://github.com/volatilityfoundation/dwarf2json). +Important: The first run of volatility with new symbol files will require the cache to be updated. The symbol packs contain a large number of symbol files and so may take some time to update! +However, this process only needs to be run once on each new symbol file, so assuming the pack stays in the same location will not need to be done again. Please also note it can be interrupted and next run will restart itself. + Please note: These are representative and are complete up to the point of creation for Windows and Mac. Due to the ease of compiling Linux kernels and the inability to uniquely distinguish them, an exhaustive set of Linux symbol tables cannot easily be supplied. ## Documentation diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index d41e8797a..fd8b8933e 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -18,7 +18,9 @@ configurable within the framework and can usually be set within the user interfa These files can also be compressed into ZIP files, which Volatility will process in order to locate symbol files. Volatility maintains a cache mapping the appropriate identifier for each symbol file against its filename. This cache -is update by automagic called as part of the standard automagic that's run each time a plugin is run. +is updated by automagic called as part of the standard automagic that's run each time a plugin is run. If a large number of new +symbols file are detected, this may take some time, but can be safely interrupted and restarted and will not need to run again +as long as the symbol files stay in the same location. Windows symbol tables --------------------- @@ -92,4 +94,4 @@ file, the banners must match exactly (down to the compilation date). * Copy the `.json` file to the symbols directory into `[symbols directory]/linux` - * For Mac change `linux` to `mac` \ No newline at end of file + * For Mac change `linux` to `mac` From d5c7ef1a9e61fca00b40db1dab7ed52343637278 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:16 +0900 Subject: [PATCH 329/404] Remove: pytest install command --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a3ecd7c7e..cf70b66cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -17,7 +17,6 @@ jobs: python -m pip install --upgrade pip pip install Cmake pip install setuptools wheel - pip install -U pytest pip install -r ./test/requirements-testing.txt - name: Build PyPi packages From 5db182d4303db48dbc4ba401f22b9ad5ff354f7c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:39 +0900 Subject: [PATCH 330/404] Add: .gitignore for test --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index d26e17d91..b3c86d49b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ ENV/ # Memory dump files *.dmp *.vmem +*.img + +# PyTest cache files +.pytest_cache/ \ No newline at end of file From 723fd9b4293b5e5231a2dacd05aa973567c0a9ca Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:51 +0900 Subject: [PATCH 331/404] Fix: json prettier --- test/known_files.json | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/test/known_files.json b/test/known_files.json index 089896714..a579c8053 100644 --- a/test/known_files.json +++ b/test/known_files.json @@ -1,20 +1,19 @@ { - "windows_dumpfiles": { - "win-xp-laptop-2005-06-25.img": { - "0x82220e78": [ - "9bdd5532286f1660f3778e68bc36efe6", - "e3bc1e9e7370e3b5a661ebe591ecf4ec" - ], - "0x82350bf8": [ - "e5c5e8d97b6280745b41f6572c85d1f0", - "8589f1463422884dbf1411aaad278465" - ], - "0x81eaf418": [ - "f7a1ae2060a58f8470b97affdb46dccf", - "54fd611021fa784912530b8007545986" - ], - "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" - } + "windows_dumpfiles": { + "win-xp-laptop-2005-06-25.img": { + "0x82220e78": [ + "9bdd5532286f1660f3778e68bc36efe6", + "e3bc1e9e7370e3b5a661ebe591ecf4ec" + ], + "0x82350bf8": [ + "e5c5e8d97b6280745b41f6572c85d1f0", + "8589f1463422884dbf1411aaad278465" + ], + "0x81eaf418": [ + "f7a1ae2060a58f8470b97affdb46dccf", + "54fd611021fa784912530b8007545986" + ], + "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" } } - \ No newline at end of file +} From 8e9bf4f27cf26cb4578a053808d5c305c15d171c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:50:46 +0900 Subject: [PATCH 332/404] Add: pytest in requirements-test.txt --- test/requirements-testing.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index d37dc93c3..e47f72fa2 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -6,3 +6,5 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 + +pytest>=7.1.2 \ No newline at end of file From 929d19aa50b8b7eef492bcc4215f39b29055fc16 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:51:54 +0900 Subject: [PATCH 333/404] Add: EOF in requirements-test.txt --- test/requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index e47f72fa2..e5966906c 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -7,4 +7,4 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 -pytest>=7.1.2 \ No newline at end of file +pytest>=7.1.2 From 3587828820a21d02fdb901a6d2c61dd1733ae935 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:54:36 +0900 Subject: [PATCH 334/404] Add: EOF in requirements-test.txt --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b3c86d49b..328ba5f83 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,4 @@ ENV/ *.img # PyTest cache files -.pytest_cache/ \ No newline at end of file +.pytest_cache/ From 986088b1a1b0084c8739200b7ff0792f025cc79b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:56:17 +0900 Subject: [PATCH 335/404] Fix: pytest version --- test/requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index e5966906c..7afe19b94 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -7,4 +7,4 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 -pytest>=7.1.2 +pytest>=7.0.0 From 7755328226af96ba811a63d06c8d222877cf28a8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 22 Jul 2022 01:11:35 +0900 Subject: [PATCH 336/404] Fix: psscan required framework version bump --- volatility3/framework/plugins/windows/psscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index cc030b4bf..335624672 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -22,7 +22,7 @@ vollog = logging.getLogger(__name__) class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for processes present in a particular windows memory image.""" - _required_framework_version = (2, 2, 1) + _required_framework_version = (2, 3, 1) _version = (1, 1, 0) @classmethod From 0fe1f47c9c4978c993de4b9fdb6af3919f72370f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 22 Jul 2022 21:10:43 +0900 Subject: [PATCH 337/404] Fix: support swapped exceptions --- volatility3/framework/plugins/windows/devicetree.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index 8e92de0cc..1b6b55cb7 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -78,7 +78,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): """Listing tree based on drivers and attached devices in a particular windows memory image.""" _required_framework_version = (2, 0, 3) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -96,7 +96,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): try: try: driver_name = driver.get_driver_name() - except (ValueError, exceptions.PagedInvalidAddressException): + except (ValueError, exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Failed to get Driver name : {driver.vol.offset:x}") driver_name = renderers.UnparsableValue() @@ -114,7 +114,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): for device in driver.get_devices(): try: device_name = device.get_device_name() - except (ValueError, exceptions.PagedInvalidAddressException): + except (ValueError, exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Failed to get Device name : {device.vol.offset:x}") device_name = renderers.UnparsableValue() @@ -134,7 +134,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): for level, attached_device in enumerate(device.get_attached_devices(), start=2): try: device_name = attached_device.get_device_name() - except (ValueError, exceptions.PagedInvalidAddressException): + except (ValueError, exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Failed to get Attached Device Name: {attached_device.vol.offset:x}") device_name = renderers.UnparsableValue() @@ -151,7 +151,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): attached_device_type )) - except(exceptions.PagedInvalidAddressException): + except(exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in drivers and devices: {driver.vol.offset:x}") continue From 65d825626e6d847d1a007e9672aa686138bf447b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 22 Jul 2022 21:11:01 +0900 Subject: [PATCH 338/404] Add: test for windows.devicetree --- test/test_volatility.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index a55dffb27..eb713783b 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -199,6 +199,17 @@ def test_windows_callbacks(image, volatility, python): assert out.count(b"KeBugCheckReasonCallbackListHead ") > 5 assert rc == 0 +def test_windows_devicetree(image, volatility, python): + rc, out, err = runvol_plugin("windows.devicetree.DeviceTree", image, volatility, python) + + assert out.find(b"DEV") != -1 + assert out.find(b"DRV") != -1 + assert out.find(b"ATT") != -1 + assert out.find(b"FILE_DEVICE_CONTROLLER") != -1 + assert out.find(b"FILE_DEVICE_DISK") != -1 + assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 + assert rc == 0 + # LINUX def test_linux_pslist(image, volatility, python): From 5a33acd8f955df513f8660d2a5d449a025818262 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 28 Jul 2022 17:44:03 +0300 Subject: [PATCH 339/404] bugfix --- volatility3/framework/automagic/symbol_cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 46431b773..c7cb6a5b8 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -216,7 +216,7 @@ class SqliteCache(CacheManagerInterface): return result def get_local_locations(self) -> Generator[str, None, None]: - result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = True').fetchall() + result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = 1').fetchall() for row in result: yield row['location'] @@ -261,7 +261,7 @@ class SqliteCache(CacheManagerInterface): cache_update = set() files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: - result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " + result = self._database.cursor().execute("SELECT location FROM cache WHERE local = 1 " f"AND cached < date('now', '{self.cache_period}');") for row in result: if row['location'] in files_to_timestamp: @@ -330,7 +330,7 @@ class SqliteCache(CacheManagerInterface): progress_callback(0, 'Reading remote ISF list') cursor = self._database.cursor() cursor.execute( - f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") + f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') for operating_system in constants.OS_CATEGORIES: @@ -356,7 +356,7 @@ class SqliteCache(CacheManagerInterface): additions = [] statement = 'SELECT location, identifier FROM cache' if local_only: - additions.append('local = True') + additions.append('local = 1') if operating_system: additions.append(f"operating_system = '{operating_system}'") if additions: From 95c0ca4ffa0756854a8e16d657175aa67bd7077a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 01:47:18 +0900 Subject: [PATCH 340/404] Remove: pytest module --- test/test_volatility.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index eb713783b..515bef1cc 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -14,8 +14,6 @@ import hashlib import ntpath import json -import pytest - # # HELPER FUNCTIONS # @@ -61,7 +59,6 @@ def test_windows_pslist(image, volatility, python): assert out.find(b"svchost.exe") != -1 assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 rc, out, err = runvol_plugin( "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"]) @@ -69,7 +66,6 @@ def test_windows_pslist(image, volatility, python): assert out.find(b"system") != -1 assert out.count(b"\n") < 10 assert rc == 0 - assert rc == 0 def test_windows_psscan(image, volatility, python): rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) @@ -79,21 +75,18 @@ def test_windows_psscan(image, volatility, python): assert out.find(b"svchost.exe") != -1 assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 def test_windows_dlllist(image, volatility, python): rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 def test_windows_modules(image, volatility, python): rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 - assert rc == 0 def test_windows_hivelist(image, volatility, python): rc, out, err = runvol_plugin("windows.registry.hivelist.HiveList", image, volatility, python) From 64621d90e97cbb2300689559142f009f1af83f9e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:20:01 +0900 Subject: [PATCH 341/404] Add: VSL for frameworkinfo plugin --- volatility3/framework/plugins/frameworkinfo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/frameworkinfo.py b/volatility3/framework/plugins/frameworkinfo.py index b7c887d5c..63ba24d09 100644 --- a/volatility3/framework/plugins/frameworkinfo.py +++ b/volatility3/framework/plugins/frameworkinfo.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from typing import List from volatility3 import framework From 9bfa80e59cb1907163b5adfc054a6a192332630f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:26:21 +0900 Subject: [PATCH 342/404] Add: VSL for initialize file --- volatility3/framework/layers/codecs/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index 550161e6d..e019bcbcd 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + """Codecs used for encoding or decoding data should live here From 54f11d7e18b12c7a8fd384fa333406e5c3dded25 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:26:47 +0900 Subject: [PATCH 343/404] Add: VSL for automagic/module --- volatility3/framework/automagic/module.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index 3d2bb584a..6810a58e2 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from volatility3.framework import interfaces, constants, configuration From a78bf32bd8df8fc075f52fed211e0bf4a9bb7840 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:26:58 +0900 Subject: [PATCH 344/404] Add: VSL for layers/avml --- volatility3/framework/layers/avml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index acc4493f4..f31737232 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + """Functions that read AVML files. The user of the file doesn't have to worry about the compression, From 85a94efd67c41993b15d066343c545da05c2c898 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:27:08 +0900 Subject: [PATCH 345/404] Add: VSL for layers/leechcore --- volatility3/framework/layers/leechcore.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index 8c492ca85..fb0442cfe 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import io import logging import urllib.parse From 5c76dc88e9bf4ec809f1e914a35ace5f14b446c2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 Aug 2022 14:27:15 +0900 Subject: [PATCH 346/404] Add: VSL for layers/linear --- volatility3/framework/layers/linear.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/linear.py b/volatility3/framework/layers/linear.py index c5cb47bdc..383f3d558 100644 --- a/volatility3/framework/layers/linear.py +++ b/volatility3/framework/layers/linear.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import functools from typing import List, Optional, Tuple, Iterable From 6f991f8d4f6d663bd69d33b8c83f61ce37ea39f2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Aug 2022 22:01:21 +0100 Subject: [PATCH 347/404] Core: Fix up LGTM issues across the codebase --- volatility3/framework/automagic/symbol_cache.py | 13 ++++++------- volatility3/framework/automagic/symbol_finder.py | 5 ++--- volatility3/framework/plugins/linux/psaux.py | 16 ++++++++-------- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index c7cb6a5b8..558bfb2f1 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -12,9 +12,7 @@ import urllib.request from abc import abstractmethod from typing import Dict, Generator, Iterable, List, Optional, Tuple -import volatility3.framework -import volatility3.schemas -from volatility3 import schemas +from volatility3 import framework, schemas from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -41,7 +39,7 @@ class IdentifierProcessor: Returns: identifier is valid or None if not found """ - raise NotImplemented("This base class has no get_identifier method defined") + raise NotImplementedError("This base class has no get_identifier method defined") class WindowsIdentifier(IdentifierProcessor): @@ -94,7 +92,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): super().__init__() self._filename = filename self._classifiers = {} - for subclazz in volatility3.framework.class_subclasses(IdentifierProcessor): + for subclazz in framework.class_subclasses(IdentifierProcessor): self._classifiers[subclazz.operating_system] = subclazz def add_identifier(self, location: str, operating_system: str, identifier: str): @@ -267,7 +265,7 @@ class SqliteCache(CacheManagerInterface): if row['location'] in files_to_timestamp: cache_update.add(row['location']) - idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + idextractors = list(framework.class_subclasses(IdentifierProcessor)) # New or not recently updated @@ -347,7 +345,8 @@ class SqliteCache(CacheManagerInterface): if missing_locations: self._database.cursor().execute( - f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", [x for x in missing_locations]) + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", + [x for x in missing_locations]) self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 610ed0e18..a9221a7cc 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -123,9 +123,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): requirement.construct(context, config_path) break else: - if symbol_files: - vollog.debug(f"Symbol library path not found: {symbol_files}") - # print("Kernel", banner, hex(banner_offset)) + vollog.debug(f"Symbol library path not found for: {banner}") + # print("Kernel", banner, hex(banner_offset)) else: vollog.debug("No existing banners found") # TODO: Fallback to generic regex search? diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index ed91c66f2..d8b844ca4 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -4,11 +4,12 @@ from typing import Optional +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework import symbols, exceptions, renderers, interfaces +from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.plugins.linux import pslist -from volatility3.framework.interfaces import plugins + class PsAux(plugins.PluginInterface): """ Lists processes with their command line arguments """ @@ -29,7 +30,7 @@ class PsAux(plugins.PluginInterface): ] def _get_command_line_args(self, task: interfaces.objects.ObjectInterface, - name: str) -> Optional[str]: + name: str) -> Optional[str]: """ Reads the command line arguments of a process These are stored on the userland stack @@ -104,8 +105,7 @@ class PsAux(plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], - self._generator( - pslist.PsList.list_tasks(self.context, - self.config['kernel'], - filter_func = filter_func))) - + self._generator( + pslist.PsList.list_tasks(self.context, + self.config['kernel'], + filter_func = filter_func))) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index af3741bbe..430ad6a30 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -13,7 +13,7 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Union from urllib import parse, request from volatility3 import symbols -from volatility3.framework import constants, contexts, exceptions, interfaces +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.configuration.requirements import SymbolTableRequirement From 4c4ccbf4e0e1893b8eefacdb4264ff14585a8802 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Aug 2022 22:03:26 +0100 Subject: [PATCH 348/404] Core: Fix remaining LGTM error --- volatility3/framework/layers/physical.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 0633637ca..b09055c90 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -5,7 +5,7 @@ import logging import threading from typing import Any, Dict, IO, List, Optional, Union -from volatility3.framework import exceptions, interfaces, constants +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -191,7 +191,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): """Closes the file handle.""" self._file.close() - def __exit__(self) -> None: + def __exit__(self, type, value, traceback) -> None: self.destroy() @classmethod From f8506862c4d92422a5e8927f70778f7faf69faf9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Aug 2022 22:59:45 +0100 Subject: [PATCH 349/404] Core: Move jsonschema to dev requirements --- requirements-dev.txt | 26 ++++++++++++++++++++++++++ requirements.txt | 3 --- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..3ff7c50b8 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,26 @@ +# The following packages are required for core functionality. +pefile>=2017.8.1 + +# The following packages are optional. +# If certain packages are not necessary, place a comment (#) at the start of the line. + +# This is required for the yara plugins +yara-python>=3.8.0 + +# This is required for several plugins that perform malware analysis and disassemble code. +# It can also improve accuracy of Windows 8 and later memory samples. +capstone>=3.0.5 + +# This is required by plugins that decrypt passwords, password hashes, etc. +pycryptodome + +# This can improve error messages regarding improperly configured ISF files, +# but is only recommended for development +# jsonschema>=2.3.0 + +# This is required for memory acquisition via leechcore/pcileech. +leechcorepyc>=2.4.0 + +# This is required for analyzing Linux samples compressed using AVMLs native +# compression format. It is not required for AVML's standard LiME compression. +python-snappy==0.6.0 diff --git a/requirements.txt b/requirements.txt index 290d9ca97..1793012f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,9 +14,6 @@ capstone>=3.0.5 # This is required by plugins that decrypt passwords, password hashes, etc. pycryptodome -# This can improve error messages regarding improperly configured ISF files. -jsonschema>=2.3.0 - # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 From 989b4c73273b1acfd767f64a63599fbc511d77dc Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 5 Aug 2022 03:08:18 +0900 Subject: [PATCH 350/404] Fix: cache path for python of Windows Store version --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3b499adea..e83d27108 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -64,7 +64,7 @@ CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" if sys.platform == 'win32': - CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") + CACHE_PATH = os.path.realpath(os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3")) os.makedirs(CACHE_PATH, exist_ok = True) LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") From 837e1ef39df3f5db6422d541b163b47d8226bb83 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 15:58:07 +0900 Subject: [PATCH 351/404] Fix: error handling for netstat plugin --- volatility3/framework/plugins/windows/netstat.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 486957565..3051b950e 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -433,7 +433,8 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self.context, interfaces.configuration.path_join(self.config_path, 'tcpip'), kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) except exceptions.VolatilityException: - vollog.warning("Unable to locate symbols for the memory image's tcpip module") + vollog.error("Unable to locate symbols for the memory image's tcpip module") + raise for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): From a04cb4e031f0a0092aec57ff72c371485893dd66 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 16:13:29 +0900 Subject: [PATCH 352/404] Fix: return syntax --- volatility3/framework/plugins/windows/netstat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 3051b950e..4d6ec5f62 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -434,7 +434,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) except exceptions.VolatilityException: vollog.error("Unable to locate symbols for the memory image's tcpip module") - raise + return for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): From 4f77be32a541563b35279dc7bfda7ee6a52ca853 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 16:30:14 +0900 Subject: [PATCH 353/404] Remove: dump file namespace --- volatility3/plugins/windows/registry/certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 429db96a6..8ef5abcdd 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -47,7 +47,7 @@ class Certificates(interfaces.plugins.PluginInterface): Optional[interfaces.plugins.FileHandlerInterface]: try: if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - dump_name = "{} - {} - {}.crt".format(hive_offset, reg_section, key_hash) + dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) file_handle = open_method(dump_name) file_handle.write(certificate_data) return file_handle From 0c8d4f75ae63a2396deceef229e1c4d2e26135f3 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 7 Aug 2022 16:57:43 +0900 Subject: [PATCH 354/404] Fix: wide exceptions --- volatility3/plugins/windows/registry/certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 8ef5abcdd..6029c0a5c 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -68,7 +68,7 @@ class Certificates(interfaces.plugins.PluginInterface): "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - with contextlib.suppress(KeyError, exceptions.SwappedInvalidAddressException): + with contextlib.suppress(KeyError, exceptions.InvalidAddressException): # Walk it node_path = hive.get_key(top_key, return_list = True) for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): From 471551fda0bc5871f8738f05b06fef1f35c5f826 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 03:13:40 +0900 Subject: [PATCH 355/404] Add: initialize for windows.joblinks plugin --- test/test_volatility.py | 5 ++ .../framework/plugins/windows/joblinks.py | 72 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 volatility3/framework/plugins/windows/joblinks.py diff --git a/test/test_volatility.py b/test/test_volatility.py index 515bef1cc..1126aa9d7 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -203,6 +203,11 @@ def test_windows_devicetree(image, volatility, python): assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 assert rc == 0 +def test_windows_joblinks(image, volatility, python): + rc, out, err = runvol_plugin("windows.joblinks.JobLinks", image, volatility, python) + + assert rc == 0 + # LINUX def test_linux_pslist(image, volatility, python): diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py new file mode 100644 index 000000000..c440cad29 --- /dev/null +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -0,0 +1,72 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterable, Iterator, List, Tuple + +from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.constants import LOGLEVEL_VVVV +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + +class JobLinks(interfaces.plugins.PluginInterface): + """Print process job link information""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls)-> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.BooleanRequirement(name = 'physical', + description = "Display physical offset instead of virtual", + default = False, + optional = True), + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) + ] + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config['kernel']] + memory = self.context.layers[kernel.layer_name] + + for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, + kernel.symbol_table_name): + try: + if not self.config['physical']: + offset = proc.vol.offset + else: + (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + + job = proc.Job.dereference() + + yield (0, ( + format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), + job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, + renderers.NotApplicableValue(), + "(Original Process)" + )) + + vollog.log(LOGLEVEL_VVVV, proc.JobLinks) + vollog.log(LOGLEVEL_VVVV, job.JobLinks) + vollog.log(LOGLEVEL_VVVV, job.ProcessListHead) + + except (exceptions.InvalidAddressException): + continue + + def run(self)-> renderers.TreeGrid: + offsettype = "(V)" if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT) else "(P)" + + return renderers.TreeGrid([ + (f"Offset{offsettype}", format_hints.Hex), ("Name", str), ("PID", int), + ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), + ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str) + ], self._generator()) \ No newline at end of file From e0edb87d7f15b883aea2fbec5539ec1850307037 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 03:25:21 +0900 Subject: [PATCH 356/404] Add: EOF --- volatility3/framework/plugins/windows/joblinks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index c440cad29..2ca32e09d 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -69,4 +69,4 @@ class JobLinks(interfaces.plugins.PluginInterface): (f"Offset{offsettype}", format_hints.Hex), ("Name", str), ("PID", int), ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str) - ], self._generator()) \ No newline at end of file + ], self._generator()) From 7bec33b07d9ab74e798bee315ae9bb8ed8eab26d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 14:40:35 +0900 Subject: [PATCH 357/404] Add: debug log code --- .../framework/plugins/windows/joblinks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 2ca32e09d..e5a206af2 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -56,8 +56,24 @@ class JobLinks(interfaces.plugins.PluginInterface): )) vollog.log(LOGLEVEL_VVVV, proc.JobLinks) + vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Flink)) + vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Blink)) vollog.log(LOGLEVEL_VVVV, job.JobLinks) + vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Flink)) + vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Blink)) vollog.log(LOGLEVEL_VVVV, job.ProcessListHead) + vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Flink)) + vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Blink)) + vollog.log(LOGLEVEL_VVVV, "") + + for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): + yield (1, ( + format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, + entry.InheritedFromUniqueProcessId, entry.get_session_id(), renderers.NotApplicableValue(), entry.get_is_wow64(), + renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + "(Original Process)" + )) except (exceptions.InvalidAddressException): continue From fa686a9fa69c23361df9410b860823b34e31fe38 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:04:24 +0900 Subject: [PATCH 358/404] Add: Peb.ProcessParameters.ImagePathName --- volatility3/framework/plugins/windows/joblinks.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index e5a206af2..1ee64544f 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -55,24 +55,13 @@ class JobLinks(interfaces.plugins.PluginInterface): "(Original Process)" )) - vollog.log(LOGLEVEL_VVVV, proc.JobLinks) - vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Flink)) - vollog.log(LOGLEVEL_VVVV, hex(proc.JobLinks.Blink)) - vollog.log(LOGLEVEL_VVVV, job.JobLinks) - vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Flink)) - vollog.log(LOGLEVEL_VVVV, hex(job.JobLinks.Blink)) - vollog.log(LOGLEVEL_VVVV, job.ProcessListHead) - vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Flink)) - vollog.log(LOGLEVEL_VVVV, hex(job.ProcessListHead.Blink)) - vollog.log(LOGLEVEL_VVVV, "") - for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): yield (1, ( format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), renderers.NotApplicableValue(), entry.get_is_wow64(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), - renderers.NotApplicableValue(), - "(Original Process)" + "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string() )) except (exceptions.InvalidAddressException): From ea65649548d708aabe2b4568aa712ef3e8e58ff2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:08:13 +0900 Subject: [PATCH 359/404] Remove: test_windows_joblinks function for test --- test/test_volatility.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 1126aa9d7..515bef1cc 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -203,11 +203,6 @@ def test_windows_devicetree(image, volatility, python): assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 assert rc == 0 -def test_windows_joblinks(image, volatility, python): - rc, out, err = runvol_plugin("windows.joblinks.JobLinks", image, volatility, python) - - assert rc == 0 - # LINUX def test_linux_pslist(image, volatility, python): From 65e7b5302c12068cb78b712d8358f4870eab49dd Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:18:11 +0900 Subject: [PATCH 360/404] Fix: job detail info to zero --- volatility3/framework/plugins/windows/joblinks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 1ee64544f..321e6f791 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -58,8 +58,8 @@ class JobLinks(interfaces.plugins.PluginInterface): for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): yield (1, ( format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, - entry.InheritedFromUniqueProcessId, entry.get_session_id(), renderers.NotApplicableValue(), entry.get_is_wow64(), - renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), + entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), + 0, 0, 0, "Yes", entry.get_peb().ProcessParameters.ImagePathName.get_string() )) From 3556b2374a3f9593d58564431d057ead5859cea7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:22:07 +0900 Subject: [PATCH 361/404] Fix: indent for prettier code --- volatility3/framework/plugins/windows/joblinks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 321e6f791..e88b7a3d3 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -57,12 +57,12 @@ class JobLinks(interfaces.plugins.PluginInterface): for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): yield (1, ( - format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, - entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), - 0, 0, 0, - "Yes", - entry.get_peb().ProcessParameters.ImagePathName.get_string() - )) + format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, + entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), + 0, 0, 0, + "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string() + )) except (exceptions.InvalidAddressException): continue From 2918c13046b91ecdae1fba6599d291fc7ba95ce7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:24:40 +0900 Subject: [PATCH 362/404] Fix: offset for job entry --- volatility3/framework/plugins/windows/joblinks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index e88b7a3d3..39089a741 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -56,6 +56,12 @@ class JobLinks(interfaces.plugins.PluginInterface): )) for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): + + if not self.config['physical']: + offset = entry.vol.offset + else: + (_, _, offset, _, _) = list(memory.mapping(offset = entry.vol.offset, length = 0))[0] + yield (1, ( format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), From cd6a73939ed19426e47209c532481c962b223204 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:27:39 +0900 Subject: [PATCH 363/404] Refactor: apply code style by yapf --- .../framework/plugins/windows/joblinks.py | 49 ++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index 39089a741..e30044538 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -15,6 +15,7 @@ from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) + class JobLinks(interfaces.plugins.PluginInterface): """Print process job link information""" @@ -22,62 +23,54 @@ class JobLinks(interfaces.plugins.PluginInterface): _version = (1, 0, 0) @classmethod - def get_requirements(cls)-> List[interfaces.configuration.RequirementInterface]: + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), requirements.BooleanRequirement(name = 'physical', description = "Display physical offset instead of virtual", default = False, optional = True), - requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) ] def _generator(self) -> Iterator[Tuple]: kernel = self.context.modules[self.config['kernel']] memory = self.context.layers[kernel.layer_name] - for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, - kernel.symbol_table_name): + for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, kernel.symbol_table_name): try: if not self.config['physical']: offset = proc.vol.offset else: (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] - + job = proc.Job.dereference() - - yield (0, ( - format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, - proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), - job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, - renderers.NotApplicableValue(), - "(Original Process)" - )) + + yield (0, (format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), + job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, + renderers.NotApplicableValue(), "(Original Process)")) for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): - if not self.config['physical']: offset = entry.vol.offset else: (_, _, offset, _, _) = list(memory.mapping(offset = entry.vol.offset, length = 0))[0] - yield (1, ( - format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), entry.UniqueProcessId, - entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, entry.get_is_wow64(), - 0, 0, 0, - "Yes", - entry.get_peb().ProcessParameters.ImagePathName.get_string() - )) + yield (1, (format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), + entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, + entry.get_is_wow64(), 0, 0, 0, "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string())) except (exceptions.InvalidAddressException): continue - def run(self)-> renderers.TreeGrid: + def run(self) -> renderers.TreeGrid: offsettype = "(V)" if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT) else "(P)" - return renderers.TreeGrid([ - (f"Offset{offsettype}", format_hints.Hex), ("Name", str), ("PID", int), - ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), - ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str) - ], self._generator()) + return renderers.TreeGrid([(f"Offset{offsettype}", format_hints.Hex), ("Name", str), + ("PID", int), ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), + ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str)], + self._generator()) From 0aedc6a071c9bc0a2b88a7ef978a80be3a9d2e03 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 9 Aug 2022 15:40:17 +0900 Subject: [PATCH 364/404] Remove: unused module --- volatility3/framework/plugins/windows/joblinks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py index e30044538..40d09b9ea 100644 --- a/volatility3/framework/plugins/windows/joblinks.py +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -4,11 +4,10 @@ import logging -from typing import Iterable, Iterator, List, Tuple +from typing import Iterator, List, Tuple from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.constants import LOGLEVEL_VVVV from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist From 154659cd0d0049ba7be1911af9a7add6ba3e5fa8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 21 Aug 2022 04:58:54 +0900 Subject: [PATCH 365/404] Fix: typo for cache sqlite schema version --- volatility3/framework/automagic/symbol_cache.py | 6 +++--- volatility3/framework/constants/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 558bfb2f1..ab19965c7 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -171,11 +171,11 @@ class SqliteCache(CacheManagerInterface): database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( - f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCEMA_VERSION})') + f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})') schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() if not schema_version: - database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCEMA_VERSION})') - elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCEMA_VERSION: + database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCHEMA_VERSION})') + elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCHEMA_VERSION: # All good, so pass and move on pass else: diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3b499adea..af3f7c0a0 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -76,7 +76,7 @@ MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") """Default location to record information about available identifiers""" -CACHE_SQLITE_SCEMA_VERSION = 1 +CACHE_SQLITE_SCHEMA_VERSION = 1 """Version for the sqlite3 cache schema""" BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" From 3e071b563d03d69cc06042eb05dfd2136cc49b2e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 21 Aug 2022 05:02:34 +0900 Subject: [PATCH 366/404] Fix: typo for symbol table --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index ab19965c7..a24dc3fd0 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -196,7 +196,7 @@ class SqliteCache(CacheManagerInterface): If multiple locations exist for an identifier, the last found is returned Args: - identifier: string that uniquely identifies a particular symbolt table + identifier: string that uniquely identifies a particular symbol table operating_system: optional string to restrict identifiers to just those for a particular operating system Returns: From ed8d240a7cf1b7d3b39bc467af8ec67d4c8ac190 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 24 Aug 2022 10:24:14 +0300 Subject: [PATCH 367/404] return given layer by base --- volatility3/framework/automagic/windows.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index f5dd720d6..08a5027d1 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -214,6 +214,9 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): context.config[interfaces.configuration.path_join( config_path, "page_map_offset")] = base_layer.metadata['page_map_offset'] layer = layer_type(context, config_path = config_path, name = new_layer_name, metadata = {'os': 'Windows'}) + page_map_offset = context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] + vollog.debug(f"DTB was given to as by base layer: {hex(page_map_offset)}") + return layer # Self Referential finder for description, tests, sections in cls.test_sets: From 253c4b5bb1cc7411255277639a866f8b0c9f87ac Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 24 Aug 2022 10:26:04 +0300 Subject: [PATCH 368/404] typo --- volatility3/framework/automagic/windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 08a5027d1..aaef3e820 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -215,7 +215,7 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): config_path, "page_map_offset")] = base_layer.metadata['page_map_offset'] layer = layer_type(context, config_path = config_path, name = new_layer_name, metadata = {'os': 'Windows'}) page_map_offset = context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] - vollog.debug(f"DTB was given to as by base layer: {hex(page_map_offset)}") + vollog.debug(f"DTB was given to us by base layer: {hex(page_map_offset)}") return layer # Self Referential finder From 9ca83763ba7e1b1012c09af4fb0f416a0b6de7cf Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Tue, 10 May 2022 09:17:03 -0500 Subject: [PATCH 369/404] refs #713 add a vad.get_size() method and fix several off-by-one issues with calculating vad size --- volatility3/framework/plugins/windows/malfind.py | 2 +- .../framework/plugins/windows/skeleton_key_check.py | 2 +- volatility3/framework/plugins/windows/vadinfo.py | 7 ++++--- volatility3/framework/plugins/windows/vadyarascan.py | 4 +--- .../framework/symbols/windows/extensions/__init__.py | 12 ++++++++---- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 700ced8ee..e63b81fb2 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -56,7 +56,7 @@ class Malfind(interfaces.plugins.PluginInterface): all_zero_page = b"\x00" * CHUNK_SIZE offset = 0 - vad_length = vad.get_end() - vad.get_start() + vad_length = vad.get_size() while offset < vad_length: next_addr = vad.get_start() + offset diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index 4a1b48c9a..cb1dd06c6 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -262,7 +262,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): if isinstance(filename, str) and filename.lower().endswith("cryptdll.dll"): base = vad.get_start() - return base, vad.get_end() - base + return base, vad.get_size() return None, None diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index e357b150a..50a69f8fb 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -132,7 +132,7 @@ class VadInfo(interfaces.plugins.PluginInterface): vollog.debug("Unable to find the starting/ending VPN member") return None - if 0 < maxsize < (vad_end - vad_start): + if 0 < maxsize < vad.get_size(): vollog.debug(f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit") return None @@ -151,8 +151,9 @@ class VadInfo(interfaces.plugins.PluginInterface): file_handle = open_method(file_name) chunk_size = 1024 * 1024 * 10 offset = vad_start - while offset < vad_end: - to_read = min(chunk_size, vad_end - offset) + vad_size = vad.get_size() + while offset < vad_start + vad_size: + to_read = min(chunk_size, vad_start + vad_size - offset) data = proc_layer.read(offset, to_read, pad = True) if not data: break diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 06a87d003..3954288eb 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -82,9 +82,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """ vad_root = task.get_vad_root() for vad in vad_root.traverse(): - end = vad.get_end() - start = vad.get_start() - yield (start, end - start) + yield (vad.get_start(), vad.get_size()) def run(self): return renderers.TreeGrid([('Offset', format_hints.Hex), ('PID', int), ('Rule', str), ('Component', str), diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index fe32a0322..bf44d1368 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -197,8 +197,8 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the parent member") - def get_start(self): - """Get the VAD's starting virtual address.""" + def get_start(self) -> int: + """Get the VAD's starting virtual address. This is the first accessible byte in the range.""" if self.has_member("StartingVpn"): @@ -216,8 +216,8 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the starting VPN member") - def get_end(self): - """Get the VAD's ending virtual address.""" + def get_end(self) -> int: + """Get the VAD's ending virtual address. This is the last accessible byte in the range.""" if self.has_member("EndingVpn"): @@ -234,6 +234,10 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the ending VPN member") + def get_size(self) -> int: + """Get the size of the VAD region. The OS ensures page granularity.""" + return (self.get_end() - self.get_start()) + 1 + def get_commit_charge(self): """Get the VAD's commit charge (number of committed pages)""" From 8bbcb51bcb3c27c7871dc6629d50570dc866e6bd Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 25 Aug 2022 01:47:19 +0900 Subject: [PATCH 370/404] Remove: return syntax --- volatility3/framework/plugins/windows/netstat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 4d6ec5f62..93ac3af93 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -434,7 +434,6 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) except exceptions.VolatilityException: vollog.error("Unable to locate symbols for the memory image's tcpip module") - return for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): From a5fe38339a038852cdff47acb0d4942e98fdaefd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Aug 2022 21:15:50 +0100 Subject: [PATCH 371/404] Core: Allow for deprecation of constants gracefully --- volatility3/framework/__init__.py | 2 +- volatility3/framework/automagic/linux.py | 4 ++- volatility3/framework/automagic/mac.py | 4 ++- .../framework/automagic/symbol_cache.py | 3 +- .../framework/automagic/symbol_finder.py | 4 ++- volatility3/framework/constants/__init__.py | 29 ++++++++++++++----- volatility3/framework/plugins/isfinfo.py | 6 ++-- .../framework/symbols/windows/pdbutil.py | 3 +- 8 files changed, 40 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 176eb2242..9b11143b2 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 6, 0) +required_python_version = (3, 7, 0) if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): raise RuntimeError( diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2c152996d..9bb2dae9b 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,6 +3,7 @@ # import logging +import os from typing import Optional, Tuple, Type from volatility3.framework import constants, interfaces @@ -40,7 +41,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - linux_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + linux_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( operating_system = 'linux') # If we have no banners, don't bother scanning if not linux_banners: diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 246462878..9bb3ad5f0 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -3,6 +3,7 @@ # import logging +import os import struct from typing import Optional @@ -42,7 +43,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - mac_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + mac_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( operating_system = 'mac') # If we have no banners, don't bother scanning if not mac_banners: diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 558bfb2f1..d69009721 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -388,7 +388,8 @@ class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._cache = SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + self._cache = SqliteCache(identifiers_path) def __call__(self, context, config_path, configurable, progress_callback = None): """Runs the automagic over the configurable.""" diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index a9221a7cc..7a197dffc 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -3,6 +3,7 @@ # import logging +import os from typing import Any, Callable, Iterable, List, Optional, Tuple from volatility3.framework import constants, interfaces, layers @@ -40,7 +41,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): """Creates a cached copy of the results, but only it's been requested.""" if not self._banners: - cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + cache = symbol_cache.SqliteCache(identifiers_path) self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system) return self._banners diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3b499adea..1f646416b 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,6 +9,7 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys +import warnings from typing import Callable, Optional import volatility3.framework.constants.linux @@ -67,13 +68,7 @@ if sys.platform == 'win32': CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") os.makedirs(CACHE_PATH, exist_ok = True) -LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") -"""Default location to record information about available linux banners""" - -MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") -"""Default location to record information about available mac banners""" - -IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") +IDENTIFIERS_FILENAME = "identifier.cache" """Default location to record information about available identifiers""" CACHE_SQLITE_SCEMA_VERSION = 1 @@ -107,3 +102,23 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" + +### +# DEPRECATED VALUES +### + +_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, 'linux_banners.cache') +"""This value is deprecated and is no longer used within volatility""" + +_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, 'mac_banners.cache') +"""This value is deprecated and is no longer used within volatility""" + +_deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) +"""This value is deprecated in favour of CACHE_PATH joined to IDENTIFIER_FILENAME""" + + +def __getattr__(name): + deprecated_tag = '_deprecated_' + if name in [x[len(deprecated_tag):] for x in globals() if x.startswith(deprecated_tag)]: + warnings.warn(f"{name} is deprecated", FutureWarning) + return globals()[f"{deprecated_tag}{name}"] diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 6b13f10b6..efffa9b87 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -109,7 +109,8 @@ class IsfInfo(plugins.PluginInterface): num_enums = len(data.get('enums', [])) num_bases = len(data.get('base_types', [])) - identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + identifier_cache = symbol_cache.SqliteCache(identifiers_path) identifier = identifier_cache.get_identifier(location = entry) if identifier: identifier = identifier.decode('utf-8', errors = 'replace') @@ -120,7 +121,8 @@ class IsfInfo(plugins.PluginInterface): vollog.warning(f"Invalid ISF: {entry}") yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) else: - cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + cache = symbol_cache.SqliteCache(identifiers_path) valid = 'Unknown' for identifier, location in cache.get_identifier_dictionary().items(): num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 430ad6a30..079b0e826 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -80,7 +80,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Required version of SQLiteCache not found") return None - value = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).find_location( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + value = symbol_cache.SqliteCache(identifiers_path).find_location( symbol_cache.WindowsIdentifier.generate(pdb_name.strip('\x00'), guid.upper(), age), 'windows') if value: From a337ec732a6feaf70032c405a48f1f3ceae39ae5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Aug 2022 21:20:56 +0100 Subject: [PATCH 372/404] Test: Update build tests to new minimum python version --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cf70b66cd..2d3729981 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.7' - name: Install dependencies run: | From d7301d653fca9c1195f83642c7133514c1f6a9a7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 25 Aug 2022 10:55:58 +0100 Subject: [PATCH 373/404] Core: Additional updates with the bump to python 3.7.0 Kindly pointed out by @digitalisx --- README.md | 2 +- setup.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 348121e44..502e26f10 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/setup.py b/setup.py index f6bb687f2..bce21ca66 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,10 @@ from volatility3.framework import constants with open("README.md", "r", encoding = "utf-8") as fh: long_description = fh.read() + def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -19,6 +20,7 @@ def get_install_requires(): requirements.append(stripped_line) return requirements + setuptools.setup(name = "volatility3", description = "Memory forensics framework", version = constants.PACKAGE_VERSION, @@ -34,7 +36,7 @@ setuptools.setup(name = "volatility3", "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires = '>=3.6.0', + python_requires = '>=3.7.0', include_package_data = True, exclude_package_data = { '': ['development', 'development.*'], From e8b4944f9a61e0c833354d8765174576069f48c4 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 27 Aug 2022 01:06:06 +0900 Subject: [PATCH 374/404] Fix: typo for simple-plugin.rst --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index e2143f1b7..c4908caf3 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -9,7 +9,7 @@ of a normal plugin, and reuses other plugins appropriately. .. note:: This document will not include the complete code necessary for a - working plugin (such as imports, etc) since it's designed to focus on the necessary componets for writing a plugin. + working plugin (such as imports, etc) since it's designed to focus on the necessary components for writing a plugin. For complete and functioning plugins, the ``framework/plugins`` directory should be consulted. Inherit from PluginInterface From 1f1355711d08e5e62b27156e5d186e3ed59366b2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 30 Aug 2022 10:54:26 +0100 Subject: [PATCH 375/404] Windows: Fix faulty pdbutil API Commit 5bc517aa appears to have been a broken merge that removed some of the changes made to the pdbutil API unintentionally. This was kindly pointed out in PR #822 by @digitalisx. --- .../framework/symbols/windows/pdbutil.py | 48 ++++++++++++++++--- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 430ad6a30..137d5f4a2 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -13,7 +13,7 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Union from urllib import parse, request from volatility3 import symbols -from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework import constants, contexts, exceptions, interfaces from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.configuration.requirements import SymbolTableRequirement @@ -344,12 +344,46 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - return cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) + module_name = guid["pdb_name"].strip('.pdb') + + symbol_table_name = cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) + + new_module_name = None + if create_module: + new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], + symbol_table_name = symbol_table_name) + new_module_name = new_module.name + + return new_module_name, symbol_table_name + + @classmethod + def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: + """Creates a module in the specified layer_name based on a pdb name. + + Searches the memory section of the loaded module for its PDB GUID + and loads the associated symbol table into the symbol space. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + config_path: The config path where to find symbol files + layer_name: The name of the layer on which to operate + module_offset: This memory dump's module image offset + module_size: The size of the module for this dump + + Returns: + The name of the constructed and loaded symbol table + """ + + module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size, create_module = True) + + return module_name class PdbSignatureScanner(interfaces.layers.ScannerInterface): From 4ed534bc8411408194399dc9698cd688a8d6cf44 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 2 Sep 2022 15:48:46 +0900 Subject: [PATCH 376/404] Fix: typo for yapf style file --- .style.yapf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.style.yapf b/.style.yapf index 8159be910..3f154e07b 100644 --- a/.style.yapf +++ b/.style.yapf @@ -107,7 +107,7 @@ each_dict_entry_on_separate_line=True i18n_comment= # The i18n function call names. The presence of this function stops -# reformattting on that line, because the string it has cannot be moved +# reformatting on that line, because the string it has cannot be moved # away from the i18n comment. i18n_function_call= From a49e7cfeca434e622d68f14d3d9fd567c7d450e6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 02:43:05 +0900 Subject: [PATCH 377/404] Fix: duplicate comments --- volatility3/framework/plugins/windows/cachedump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index ddfa856b9..f77c6257b 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -46,7 +46,7 @@ class Cachedump(interfaces.plugins.PluginInterface): rc4 = ARC4.new(rc4key) data = rc4.encrypt(edata) # lgtm [py/weak-cryptographic-algorithm] else: - # based on Based on code from http://lab.mediaservice.net/code/cachedump.rb + # Based on code from http://lab.mediaservice.net/code/cachedump.rb aes = AES.new(nlkm[16:32], AES.MODE_CBC, ch) data = b"" for i in range(0, len(edata), 16): From 3da028c7346d34cea11198dd897cf817d1e8f621 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 02:54:26 +0900 Subject: [PATCH 378/404] Remove: unused module --- volatility3/framework/plugins/windows/ldrmodules.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 284d1afc2..ba8d049a6 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -1,5 +1,4 @@ -from volatility3.framework import interfaces, constants -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed From 9e578e66da923121c44b8940aa1c0c691352f616 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 02:59:26 +0900 Subject: [PATCH 379/404] Remove: duplicate paragraph --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 96f222187..2a37fd0ed 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -31,7 +31,7 @@ If you make any Additions available to others, such as by providing copies of th - You are responsible to ensure you have rights in Additions necessary to comply with this section. Contributing -If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution." +If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution." Trademarks This license grants you no rights to any trademarks or service marks. From 626e352b18c9288b70dcf1cebb615a8b03379989 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 4 Sep 2022 03:15:28 +0900 Subject: [PATCH 380/404] Add: api changes description for 2.3.1 version --- API_CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/API_CHANGES.md b/API_CHANGES.md index 4d8733286..f74f754f9 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,10 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.3.1 +===== +Update in the windows `_EPROCESS.owning_process` method for support Windows Vista and later versions. + 2.3.0 ===== Add in `child_template` to template class From 97638ffc0dd05c587d031303f431f646ca3752f8 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 12 Sep 2022 16:35:31 +0300 Subject: [PATCH 381/404] fix lineterminator --- volatility3/cli/text_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ecb5179e0..623153fae 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -224,7 +224,7 @@ class CSVRenderer(CLIRenderer): # Ignore the type because namedtuples don't realize they have accessible attributes header_list.append(f"{column.name}") - writer = csv.DictWriter(outfd, header_list) + writer = csv.DictWriter(outfd, header_list, lineterminator='\n') writer.writeheader() def visitor(node: interfaces.renderers.TreeNode, accumulator): From ee3895867f3c124f3aa80c5e2f4add5e02ada33b Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Wed, 21 Sep 2022 13:40:28 -0500 Subject: [PATCH 382/404] refs #713 bump VERSION_MINOR to 4 --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/malfind.py | 2 +- volatility3/framework/plugins/windows/skeleton_key_check.py | 2 +- volatility3/framework/plugins/windows/vadinfo.py | 2 +- volatility3/framework/plugins/windows/vadyarascan.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 6eec88d26..0e661a474 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,7 +39,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 3 # Number of changes that only add to the interface +VERSION_MINOR = 4 # Number of changes that only add to the interface VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index e63b81fb2..9b5fab3f5 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -17,7 +17,7 @@ vollog = logging.getLogger(__name__) class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) @classmethod def get_requirements(cls): diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index cb1dd06c6..f6f41864a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -41,7 +41,7 @@ vollog = logging.getLogger(__name__) class Skeleton_Key_Check(interfaces.plugins.PluginInterface): """ Looks for signs of Skeleton Key malware """ - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) @classmethod def get_requirements(cls): diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 50a69f8fb..d3997c8c8 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -33,7 +33,7 @@ winnt_protections = { class VadInfo(interfaces.plugins.PluginInterface): """Lists process memory ranges.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) _version = (2, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 3954288eb..b71e2f605 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -17,7 +17,7 @@ vollog = logging.getLogger(__name__) class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) _version = (1, 0, 0) @classmethod From 941b5ff9c1aef35d1c06e665e5a119a2a82ba79e Mon Sep 17 00:00:00 2001 From: ikelos Date: Wed, 21 Sep 2022 20:19:42 +0100 Subject: [PATCH 383/404] Update volatility3/framework/constants/__init__.py Yep, quite right Co-authored-by: Donghyun Kim --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 0e661a474..00ae15f4e 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 4 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 4985dcd9a3ddff808004da71d636611f5956385b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Sep 2022 20:53:41 +0100 Subject: [PATCH 384/404] Windows: When constructing a buffer, manually dereference onto the native layer --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index fe32a0322..e290ef52d 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -488,7 +488,7 @@ class UNICODE_STRING(objects.StructType): # We manually construct an object rather than casting a dereferenced pointer in case # the buffer length is 0 and the pointer is a NULL pointer return self._context.object(self.vol.type_name.split(constants.BANG)[0] + constants.BANG + 'string', - layer_name = self.Buffer.vol.layer_name, + layer_name = self.Buffer.vol.native_layer_name, offset = self.Buffer, max_length = self.Length, errors = 'replace', encoding = 'utf16') From e5d4e599d3ea1b71853c530f82662e4d8d6c88bf Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Wed, 21 Sep 2022 14:55:17 -0500 Subject: [PATCH 385/404] refs #713 update API_CHANGES.md --- API_CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/API_CHANGES.md b/API_CHANGES.md index 4d8733286..a541e9619 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,10 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.4.0 +===== +Add a `get_size()` method to Windows VAD structures and fix several off-by-one issues when calculating VAD sizes. + 2.3.0 ===== Add in `child_template` to template class From 7529c7b246734ae02b51bb80dc22bbeddb819078 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Sep 2022 21:15:40 +0100 Subject: [PATCH 386/404] Core: Revert volatility 3.7 bump and associated features --- .github/workflows/test.yaml | 4 ++-- README.md | 2 +- setup.py | 6 ++---- volatility3/framework/__init__.py | 2 +- volatility3/framework/constants/__init__.py | 21 --------------------- 5 files changed, 6 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2d3729981..cf70b66cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.7 + - name: Set up Python 3.6 uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.6' - name: Install dependencies run: | diff --git a/README.md b/README.md index 502e26f10..348121e44 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/setup.py b/setup.py index bce21ca66..f6bb687f2 100644 --- a/setup.py +++ b/setup.py @@ -9,10 +9,9 @@ from volatility3.framework import constants with open("README.md", "r", encoding = "utf-8") as fh: long_description = fh.read() - def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -20,7 +19,6 @@ def get_install_requires(): requirements.append(stripped_line) return requirements - setuptools.setup(name = "volatility3", description = "Memory forensics framework", version = constants.PACKAGE_VERSION, @@ -36,7 +34,7 @@ setuptools.setup(name = "volatility3", "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires = '>=3.7.0', + python_requires = '>=3.6.0', include_package_data = True, exclude_package_data = { '': ['development', 'development.*'], diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 9b11143b2..176eb2242 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 7, 0) +required_python_version = (3, 6, 0) if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): raise RuntimeError( diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 1f646416b..d6fb96e1c 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,7 +9,6 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys -import warnings from typing import Callable, Optional import volatility3.framework.constants.linux @@ -102,23 +101,3 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" - -### -# DEPRECATED VALUES -### - -_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, 'linux_banners.cache') -"""This value is deprecated and is no longer used within volatility""" - -_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, 'mac_banners.cache') -"""This value is deprecated and is no longer used within volatility""" - -_deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) -"""This value is deprecated in favour of CACHE_PATH joined to IDENTIFIER_FILENAME""" - - -def __getattr__(name): - deprecated_tag = '_deprecated_' - if name in [x[len(deprecated_tag):] for x in globals() if x.startswith(deprecated_tag)]: - warnings.warn(f"{name} is deprecated", FutureWarning) - return globals()[f"{deprecated_tag}{name}"] From 3523985d0a7123f2cf4648568a7e41865c9edd57 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 23 Sep 2022 07:47:03 +0900 Subject: [PATCH 387/404] Fix: to find_namepsace_packages method --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f6bb687f2..a4bd3fffe 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setuptools.setup(name = "volatility3", '': ['development', 'development.*'], 'development': ['*'] }, - packages = setuptools.find_packages(exclude = ["development", "development.*"]), + packages = setuptools.find_namespace_packages(exclude = ["development", "development.*"]), entry_points = { 'console_scripts': [ 'vol = volatility3.cli:main', From 949b15a36812d3f4ad33cf55e3e8fb387a0d7cee Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 23 Sep 2022 08:01:11 +0900 Subject: [PATCH 388/404] Fix: unsused module for objects initialize code --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 62e6de553..eedd22bb2 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -9,7 +9,7 @@ import struct from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload from volatility3.framework import constants, interfaces -from volatility3.framework.objects import templates, utility +from volatility3.framework.objects import templates vollog = logging.getLogger(__name__) From 1ffe9f222f984512a2441ee65e7317b7b4953531 Mon Sep 17 00:00:00 2001 From: a5hlynx Date: Fri, 7 Oct 2022 00:57:20 +0900 Subject: [PATCH 389/404] correct ImageFileName --- volatility3/framework/plugins/windows/handles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index ab11d30d6..bdff88075 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -323,7 +323,7 @@ class Handles(interfaces.plugins.PluginInterface): obj_name = item.file_name_with_device() elif obj_type == "Process": item = entry.Body.cast("_EPROCESS") - obj_name = f"{utility.array_to_string(proc.ImageFileName)} Pid {item.UniqueProcessId}" + obj_name = f"{utility.array_to_string(item.ImageFileName)} Pid {item.UniqueProcessId}" elif obj_type == "Thread": item = entry.Body.cast("_ETHREAD") obj_name = f"Tid {item.Cid.UniqueThread} Pid {item.Cid.UniqueProcess}" From 146afc0f0786a9e849e480b54e85562f9a2a19a1 Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 16 Oct 2022 08:07:43 +0530 Subject: [PATCH 390/404] Incomplete sentence - fixedf --- doc/source/Linux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 180e7c697..223c9fc07 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -1,7 +1,7 @@ Linux Tutorial ============== -This guide gives you a brief introduction to how volatility3 works and some demonstration of several of the plugins available from +This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite of plugins. Acquiring memory ---------------- From 95e4078b77fd802147b5b3d4662ca8734ef2d6df Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 16 Oct 2022 08:25:35 +0530 Subject: [PATCH 391/404] Added FTK as another example to avoid favouritism --- doc/source/Windows.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index a6c67780e..d732e0cdc 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -8,7 +8,8 @@ Acquiring memory Volatility does not provide the ability to acquire memory. -.. tip:: You could use `WinPmem `_ for collecting windows memory dump. +.. tip:: - You could use `WinPmem `_ for collecting windows memory dump. + - You could also use `FTK Imager `_ Listing Plugins --------------- From fba734b284e0bed5f5cb7b4ac91c3194ea8bab2b Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 16 Oct 2022 19:10:55 +0530 Subject: [PATCH 392/404] AVML added. Restructured Acquiring Memory. --- doc/source/Linux.rst | 7 ++++--- doc/source/Windows.rst | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 223c9fc07..9fb5a686e 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -6,9 +6,10 @@ This guide will give you a brief overview of how volatility3 works as well as a Acquiring memory ---------------- -Volatility3 does not provide the ability to acquire memory. In this tutorial we will see how we can use `LiME `_ for this purpose. -It supports 32 and 64 bit captures from native Intel hardware as well as virtual machine guests. -It also supports capture from Android devices. See below for example commands building and running LiME: +Volatility3 does not provide the ability to acquire memory. +You can use any of the following tools to Acquire memory or the ones you are convenient with: + - `AVML - Acquire Volatile Memory for Linux `_ + - `LIME - Linux Memory Extract `_ .. code-block:: shell-session diff --git a/doc/source/Windows.rst b/doc/source/Windows.rst index d732e0cdc..80bc6ddc2 100644 --- a/doc/source/Windows.rst +++ b/doc/source/Windows.rst @@ -7,9 +7,9 @@ Acquiring memory ---------------- Volatility does not provide the ability to acquire memory. - -.. tip:: - You could use `WinPmem `_ for collecting windows memory dump. - - You could also use `FTK Imager `_ +You can use any of the following tools to Acquire memory or the ones you are convenient with: + - `WinPmem `_ + - `FTK Imager `_ Listing Plugins --------------- From 9d9eb226ab64aeefaec82df217ddd478859b340d Mon Sep 17 00:00:00 2001 From: Tejas <47889755+tejas15802@users.noreply.github.com> Date: Sun, 16 Oct 2022 19:21:42 +0530 Subject: [PATCH 393/404] Removed the commands which were present for LIME --- doc/source/Linux.rst | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/doc/source/Linux.rst b/doc/source/Linux.rst index 9fb5a686e..ea6c2223c 100644 --- a/doc/source/Linux.rst +++ b/doc/source/Linux.rst @@ -11,20 +11,6 @@ You can use any of the following tools to Acquire memory or the ones you are con - `AVML - Acquire Volatile Memory for Linux `_ - `LIME - Linux Memory Extract `_ -.. code-block:: shell-session - - $ tar -xvzf lime-forensics-1.1-r14.tar.gz - $ cd lime-forensics-1.1-r14/src - $ make - .... - CC [M] lime-forensics-1.1-r14/src/tcp.o - CC [M] lime-forensics-1.1-r14/src/disk.o - .... - $ sudo insmod lime-3.2.0-23-generic.ko "path=/tmp/ubuntu.lime format=lime" - $ ls -alh /tmp/ubuntu.lime - -r--r--r-- 1 root root 2.0G Aug 17 19:37 /tmp/ubuntu.lime - -.. note:: The above command required sudo inorder to access the files which are root only. Procedure to create symbol tables for linux -------------------------------------------- From b71e367d387ed13a083bde17cb7a586a1c28cf67 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Oct 2022 15:50:22 +0100 Subject: [PATCH 394/404] Documentation: Rename, fix grammar and avoid using personal pronouns --- ...rst => getting-started-linux-tutorial.rst} | 51 +++++++++++-------- ...t => getting-started-windows-tutorial.rst} | 43 +++++++++------- doc/source/index.rst | 18 +++---- doc/source/symbol-tables.rst | 2 +- 4 files changed, 66 insertions(+), 48 deletions(-) rename doc/source/{Linux.rst => getting-started-linux-tutorial.rst} (71%) rename doc/source/{Windows.rst => getting-started-windows-tutorial.rst} (71%) diff --git a/doc/source/Linux.rst b/doc/source/getting-started-linux-tutorial.rst similarity index 71% rename from doc/source/Linux.rst rename to doc/source/getting-started-linux-tutorial.rst index ea6c2223c..15a1f0d1b 100644 --- a/doc/source/Linux.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -1,31 +1,34 @@ Linux Tutorial ============== -This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite of plugins. +This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite. Acquiring memory ---------------- -Volatility3 does not provide the ability to acquire memory. -You can use any of the following tools to Acquire memory or the ones you are convenient with: - - `AVML - Acquire Volatile Memory for Linux `_ - - `LIME - Linux Memory Extract `_ +Volatility3 does not provide the ability to acquire memory. Below are some examples of tools that can be used to acquire memory, but more are available: + +* `AVML - Acquire Volatile Memory for Linux `_ +* `LIME - Linux Memory Extract `_ Procedure to create symbol tables for linux -------------------------------------------- -To create a symbol table please refer this :ref:`symbol-tables:Mac or Linux symbol tables`. +To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. -.. tip:: We can also find some ISF files from `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. - After creating the file or downloading the file from the ISF server, please place the file under the directory ``volatility3/symbols/linux``. Make a directory linux under symbols. +.. tip:: It may be possible to locate pre-made ISF files from the `Linux ISF Server `_ , + which is built and maintained by `kevthehermit `_. + After creating the file or downloading it from the ISF server, place the file under the directory ``volatility3/symbols/linux``. + If necessary create a linux directory under the symbols directory (this will become unncessary in future versions). Listing plugins --------------- -Following are the sample of linux plugins available for volatility3. More plugins will be available on future releases. -For plugin requests, Please create an issue with description of the plugin. +The following is a sample of the linux plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. .. code-block:: shell-session @@ -36,13 +39,13 @@ For plugin requests, Please create an issue with description of the plugin. linux.check_creds.Check_creds linux.check_idt.Check_idt -.. note:: Here the the command is piped to grep and head in-order to give you sample list of linux plugins. +.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of linux plugins. Using plugins ------------- -The following is the syntax to run volatility tool. +The following is the syntax to run the volatility CLI. .. code-block:: shell-session @@ -52,11 +55,11 @@ The following is the syntax to run volatility tool. Example ------- -Example 1 -~~~~~~~~~ +banners +~~~~~~~ -In this example we will be using memory dump from Insomni'hack teaser 2020 CTF. Challenge name Getdents. We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. -I'd like to say thanks to `stuxnet `_ for providing this memory dump and `writeup `_. +In this example we will be using a memory dump from the Insomni'hack teaser 2020 CTF Challenge called Getdents. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge. +Thanks go to `stuxnet `_ for providing this memory dump and `writeup `_. .. code-block:: shell-session @@ -75,11 +78,13 @@ I'd like to say thanks to `stuxnet `_ for provid 0x7fde0010 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) -This above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for ISF file from the ISF server. -If you do not find the ISF file then, please follow the instructions on :ref:`Linux:Procedure to create symbol tables for linux`. After that place the ISF file under ``volatility3/symbols/linux`` directory. +The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server. +If ISF file cannt be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. .. tip:: Use the banner text which is most repeated to search from ISF Server. +linux.pslist +~~~~~~~~~~~~ .. code-block:: shell-session @@ -109,6 +114,9 @@ If you do not find the ISF file then, please follow the instructions on :ref:`Li ``linux.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. +linux.pstree +~~~~~~~~~~~~ + .. code-block:: shell-session $ python3 vol.py -f memory.vmem linux.pstree @@ -148,9 +156,12 @@ If you do not find the ISF file then, please follow the instructions on :ref:`Li ***** 1548 1266 gsd-keyboard ***** 1550 1266 gsd-media-keys -``linux.pstree`` helps us to display the parent child relation of processes. +``linux.pstree`` helps us to display the parent child relationships between processes. -Now to find the commands ran in bash shell. Lets use ``linux.bash``. +linux.bash +~~~~~~~~~~ + +Now to find the commands that were run in the bash shell by using ``linux.bash``. .. code-block:: shell-session diff --git a/doc/source/Windows.rst b/doc/source/getting-started-windows-tutorial.rst similarity index 71% rename from doc/source/Windows.rst rename to doc/source/getting-started-windows-tutorial.rst index 80bc6ddc2..c89b065f5 100644 --- a/doc/source/Windows.rst +++ b/doc/source/getting-started-windows-tutorial.rst @@ -1,21 +1,23 @@ Windows Tutorial ================ -This guide gives you a brief introduction to how volatility3 works and some demonstration on suite of plugins available from +This guide provides a brief introduction to how volatility3 works as a demonstration of several of the plugins available in the suite. Acquiring memory ---------------- Volatility does not provide the ability to acquire memory. -You can use any of the following tools to Acquire memory or the ones you are convenient with: - - `WinPmem `_ - - `FTK Imager `_ +Memory can be acquired using a number of tools, below are some examples but others exist: + +* `WinPmem `_ +* `FTK Imager `_ Listing Plugins --------------- -Following are the sample of linux plugins available for volatility3. More plugins will be available on future releases. -For plugin requests, Please create an issue with description of the plugin. +The following is a sample of the windows plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. .. code-block:: shell-session @@ -24,14 +26,13 @@ For plugin requests, Please create an issue with description of the plugin. windows.cmdline.CmdLine windows.crashinfo.Crashinfo windows.dlllist.DllList - Lists the loaded modules in a particular windows -.. note:: Here the the command is piped to grep and head in-order to give you sample list of windows plugins. +.. note:: Here the the command is piped to grep and head in-order to provide the start of a list of the available windows plugins. Using plugins ------------- -The following is the syntax to run volatility tool. +The following is the syntax to run the volatility CLI. .. code-block:: shell-session @@ -41,13 +42,14 @@ The following is the syntax to run volatility tool. Example ------- -Example 1 -~~~~~~~~~ +windows.pslist +~~~~~~~~~~~~~~ -In this example we will be using memory dump from PragyanCTF'22. -We will limit the discussion to memory forensics with volatility3 and not extend to other parts of the challenges. +In this example we will be using a memory dump from the PragyanCTF'22. +We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenges. -In windows memory forensics using volatility3, most of the times we do not require creating a ISF file. +When using windows plugins in volatility 3, the required ISF file can often be generated from PDB files automatically +downloaded from Microsoft servers, and therefore does not require locating or adding specific ISF files to the volatility 3 symbols directory. .. code-block:: shell-session @@ -64,7 +66,10 @@ In windows memory forensics using volatility3, most of the times we do not requi 412 396 csrss.exe 0xfa80021c5b30 9 224 1 False 2022-02-07 16:30:13.000000 N/A Disabled 468 396 winlogon.exe 0xfa8002284060 5 113 1 False 2022-02-07 16:30:14.000000 N/A Disabled -``windows.pslist`` helps us list the processes running while the memory dump was taken. +``windows.pslist`` helps list the processes running while the memory dump was taken. + +windows.pstree +~~~~~~~~~~~~~~ .. code-block:: shell-session @@ -90,10 +95,12 @@ In windows memory forensics using volatility3, most of the times we do not requi ** 616 504 svchost.exe 0xfa8002b86ab0 13 314 0 False 2022-02-07 16:32:16.000000 N/A ** 624 504 svchost.exe 0xfa8002410630 10 350 0 False 2022-02-07 16:30:14.000000 N/A -``windows.pstree`` helps us to display the parent child relation of processes. +``windows.pstree`` helps to display the parent child relationships between processes. -.. note:: Here the the command is piped to head in-order to give you smaller output of process here top 20. +.. note:: Here the the command is piped to head in-order to provide smaller output, here listing only the first 20. +windows.hashdump +~~~~~~~~~~~~~~~~ .. code-block:: shell-session @@ -108,7 +115,7 @@ In windows memory forensics using volatility3, most of the times we do not requi HomeGroupUser$ 1002 aad3b435b51404eeaad3b435b51404ee af10ecac6ea817d2bb56e3e5c33ce1cd Dennis 1003 aad3b435b51404eeaad3b435b51404ee cf96684bbc7877920adaa9663698bf54 -``windows.hashdump`` helps us to list the hashes of the users in the system. +``windows.hashdump`` helps to list the hashes of the users in the system. diff --git a/doc/source/index.rst b/doc/source/index.rst index 0d35b02ba..9b1d05858 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -7,7 +7,7 @@ Volatility 3 is Open Source. :doc:`List of plugins ` -Here are some guidelines for using Volatility 3 effectively: +Below is the main documentation regarding volatility 3: .. toctree:: :caption: Documentation @@ -19,6 +19,14 @@ Here are some guidelines for using Volatility 3 effectively: volshell glossary +There is also some information to get you started quickly: + +.. toctree:: + :caption: Getting Started + + getting-started-linux-tutorial + getting-started-windows-tutorial + .. toctree:: :caption: Python Packages @@ -26,14 +34,6 @@ Here are some guidelines for using Volatility 3 effectively: volatility3 -.. toctree:: - :caption: Getting Started - - FAQ - Installation - Linux - Windows - Indices and tables ================== diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index d912d4906..b7c26e046 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -41,7 +41,7 @@ The :envvar:`PYTHONPATH` environment variable is not required if the Volatility or a virtual environment. Mac or Linux symbol tables ------------------------ +-------------------------- For Mac/Linux systems, both use the same mechanism for identification. The generated files contain an identifying string (the operating system banner), which Volatility's automagic can detect. Volatility caches the mapping between the strings and the symbol From 537f6a6a55b830534af5715fd8bd659111188b54 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Oct 2022 15:55:45 +0100 Subject: [PATCH 395/404] Documentation: Fix minor typo --- doc/source/getting-started-linux-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst index 15a1f0d1b..e1c671c36 100644 --- a/doc/source/getting-started-linux-tutorial.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -9,7 +9,7 @@ Acquiring memory Volatility3 does not provide the ability to acquire memory. Below are some examples of tools that can be used to acquire memory, but more are available: * `AVML - Acquire Volatile Memory for Linux `_ -* `LIME - Linux Memory Extract `_ +* `LiME - Linux Memory Extract `_ Procedure to create symbol tables for linux From 439835a61d4ba3abaec3b94048350ce85585872f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 17 Oct 2022 04:50:58 +0900 Subject: [PATCH 396/404] Fix: typo for linux tutorial --- doc/source/getting-started-linux-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst index e1c671c36..6fd06bcf9 100644 --- a/doc/source/getting-started-linux-tutorial.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -20,7 +20,7 @@ To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol .. tip:: It may be possible to locate pre-made ISF files from the `Linux ISF Server `_ , which is built and maintained by `kevthehermit `_. After creating the file or downloading it from the ISF server, place the file under the directory ``volatility3/symbols/linux``. - If necessary create a linux directory under the symbols directory (this will become unncessary in future versions). + If necessary create a linux directory under the symbols directory (this will become unnecessary in future versions). Listing plugins From 88e944192093281c833b1404c4592b51ac364c9f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 22 Oct 2022 18:20:22 +0900 Subject: [PATCH 397/404] Fix: typo for linux tutorial --- doc/source/getting-started-linux-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst index 6fd06bcf9..26ad2c2e4 100644 --- a/doc/source/getting-started-linux-tutorial.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -79,7 +79,7 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server. -If ISF file cannt be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. .. tip:: Use the banner text which is most repeated to search from ISF Server. From 94bb22d4bcc35cd355b31d873c8d54f42457f2ae Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 23 Oct 2022 22:49:06 +0100 Subject: [PATCH 398/404] Automagic: Make cache period longer and configurable --- volatility3/framework/automagic/symbol_cache.py | 2 +- volatility3/framework/constants/__init__.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 1e0bba86e..30a4068b6 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -157,10 +157,10 @@ class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) - cache_period = '-3 days' def __init__(self, filename: str): super().__init__(filename) + self.cache_period = constants.SQLITE_CACHE_PERIOD try: self._database = self._connect_storage(filename) except sqlite3.DatabaseError: diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index e0083a539..4fd53a3eb 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -63,6 +63,9 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" +SQLITE_CACHE_PERIOD = '-1 month' +"""SQLite time modifier for how long each item is valid in the cache for""" + if sys.platform == 'win32': CACHE_PATH = os.path.realpath(os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3")) os.makedirs(CACHE_PATH, exist_ok = True) From aa0c2b6c744486bbb7135e754b47bf1dc60e7360 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 2 Nov 2022 20:53:00 +0000 Subject: [PATCH 399/404] Mac: Fix bug found by buildbot/npetroni due refactoring --- volatility3/framework/symbols/mac/extensions/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index a66bfb534..92410704f 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -28,8 +28,11 @@ class proc(generic.GenericIntelProcess): if not isinstance(parent_layer, interfaces.layers.TranslationLayerInterface): raise TypeError("Parent layer is not a translation layer, unable to construct process layer") - with contextlib.suppress(exceptions.InvalidAddressException): + try: dtb = self.get_task().map.pmap.pm_cr3 + except exceptions.InvalidAddressException: + # Bail out because we couldn't find the DTB + return None if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.p_pid}" From d09f23a7d7a791c6e846f401de7f1168326e34ee Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 2 Nov 2022 20:55:24 +0000 Subject: [PATCH 400/404] Mac: Fix additional possibility of failure from refactoring --- volatility3/framework/symbols/mac/extensions/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 92410704f..45dc1db70 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -41,10 +41,8 @@ class proc(generic.GenericIntelProcess): return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - with contextlib.suppress(exceptions.InvalidAddressException): - task = self.get_task() - try: + task = self.get_task() current_map = task.map.hdr.links.next except exceptions.InvalidAddressException: return From 0c80ae4f816281541e177017f9e2e1e518a78b3e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 2 Nov 2022 21:39:27 +0000 Subject: [PATCH 401/404] Automagic: Check file datetime to determine whether to recache --- .../framework/automagic/symbol_cache.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 30a4068b6..fe5dfac52 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import base64 +import datetime import json import logging import os @@ -170,6 +171,7 @@ class SqliteCache(CacheManagerInterface): def _connect_storage(self, path: str) -> sqlite3.Connection: database = sqlite3.connect(path) database.row_factory = sqlite3.Row + database.cursor().execute( f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})') schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() @@ -259,10 +261,31 @@ class SqliteCache(CacheManagerInterface): cache_update = set() files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: - result = self._database.cursor().execute("SELECT location FROM cache WHERE local = 1 " + result = self._database.cursor().execute("SELECT location, cached FROM cache WHERE local = 1 " f"AND cached < date('now', '{self.cache_period}');") for row in result: - if row['location'] in files_to_timestamp: + location = row['location'] + stored_timestamp = datetime.datetime.fromisoformat(row['cached']) + timestamp = stored_timestamp # Default to requiring update + + # See if the file is a local URL type we can handle: + parsed = urllib.parse.urlparse(location) + pathname = None + if parsed.scheme == 'file': + pathname = urllib.request.url2pathname(parsed.path) + if parsed.scheme == 'jar': + inner_url = urllib.parse.urlparse(parsed.path) + if inner_url.scheme == 'file': + pathname = inner_url.path.split('!')[0] + + if pathname: + timestamp = datetime.datetime.fromtimestamp(os.stat(pathname).st_mtime) + else: + vollog.log(constants.LOGLEVEL_VVVV, + "File location in database classed as local but not file/jar URL") + + # If we're supposed to include it, and our last check is older than (or equal to) the file timestamp + if row['location'] in files_to_timestamp and stored_timestamp < timestamp: cache_update.add(row['location']) idextractors = list(framework.class_subclasses(IdentifierProcessor)) From 5ac191b31008a1e678a77839cb2aed489310691a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 2 Nov 2022 21:43:18 +0000 Subject: [PATCH 402/404] Automagic: Set the cache period back to 3 days --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 4fd53a3eb..b19e80472 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -63,7 +63,7 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" -SQLITE_CACHE_PERIOD = '-1 month' +SQLITE_CACHE_PERIOD = '-3 days' """SQLite time modifier for how long each item is valid in the cache for""" if sys.platform == 'win32': From 364a6a75f94c6c175ee03ea03bc40559d7ae4c5c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Nov 2022 14:53:39 +0000 Subject: [PATCH 403/404] Windows: Fix bad use of strip Close #867 --- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 2fa9bd591..569af6276 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -345,7 +345,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - module_name = guid["pdb_name"].strip('.pdb') + module_name = guid["pdb_name"].replace('.pdb', '') symbol_table_name = cls.load_windows_symbol_table(context, guid["GUID"], From e694713b20f23d38564a13b38a9898614945fa8b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 30 Nov 2022 01:05:25 +0000 Subject: [PATCH 404/404] Github: Backport action changes to 2.4.0 release --- .github/workflows/build-pypi.yml | 14 ++++++++------ .github/workflows/test.yaml | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 77fe26931..061c8359a 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -15,14 +15,16 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.6"] steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.x - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cf70b66cd..8f044a365 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,14 +3,16 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.6"] steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.6 - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: '3.6' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: |