From 832bdc2ab19d795b7c4abe519ef4eea9b909e581 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 8 Dec 2021 23:34:23 +0000 Subject: [PATCH 01/63] Linux: Fix long standing typo (thanks to @gcmoreira) --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fbc02399f..0edd60608 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -108,7 +108,7 @@ class module(generic.GenericIntelProcess): "linux", "elf", native_types = None, - class_types = extensions.elf.class_types) + class_types = elf.class_types) syms = self._context.object( self.get_symbol_table().name + constants.BANG + "array", From 17e0b04cb35f44612ecfc07ef7d154eec013c5be Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 12:20:51 +1100 Subject: [PATCH 02/63] undefined glob module --- volatility3/cli/volshell/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index f2375c774..94f735ba0 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -7,6 +7,7 @@ import json import logging import os import sys +import glob import volatility3.plugins import volatility3.symbols From 36ac92c11c868490891a53b285b4a9f9093a18d6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 12:27:53 +1100 Subject: [PATCH 03/63] undefined `layers` module --- volatility3/framework/automagic/symbol_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 72dc071a5..143abd02e 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -5,7 +5,7 @@ import logging from typing import Any, Iterable, List, Tuple, Type, Optional, Callable -from volatility3.framework import interfaces, constants +from volatility3.framework import interfaces, constants, layers from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners From 8e7aff4ad8e14b77d6b792fbc5581499ee89640b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 13:09:41 +1100 Subject: [PATCH 04/63] `not in` please --- development/stock-linux-json.py | 2 +- volatility3/cli/__init__.py | 2 +- volatility3/framework/layers/resources.py | 2 +- volatility3/framework/plugins/linux/check_creds.py | 2 +- volatility3/framework/plugins/mac/list_files.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/development/stock-linux-json.py b/development/stock-linux-json.py index c863d41e4..877f78e1c 100644 --- a/development/stock-linux-json.py +++ b/development/stock-linux-json.py @@ -87,7 +87,7 @@ class Downloader: output_filename = 'unknown-kernel.json' for named_file in named_files: prefix = '--system-map' - if not 'System' in named_files[named_file]: + if 'System' not in named_files[named_file]: prefix = '--elf' output_filename = './' + '-'.join((named_file.split('/')[-1]).split('-')[2:])[:-4] + '.json.xz' args += [prefix, named_files[named_file]] diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 608fdf79c..2c5e13211 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -543,7 +543,7 @@ class CommandLine: self._file = io.open(fd, mode = 'w+b') CLIFileHandler.__init__(self, filename) for item in dir(self._file): - if not item.startswith('_') and not item in ['closed', 'close', 'mode', 'name']: + if not item.startswith('_') and item not in ('closed', 'close', 'mode', 'name'): setattr(self, item, getattr(self._file, item)) def __getattr__(self, item): diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index b6ef1b6ba..7ace25290 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -82,7 +82,7 @@ class ResourceAccessor(object): """Determines whether a URLs contents should be cached""" parsed_url = urllib.parse.urlparse(url) - return self._enable_cache and not parsed_url.scheme in self._non_cached_schemes() + return self._enable_cache and parsed_url.scheme not in self._non_cached_schemes() @staticmethod def _non_cached_schemes() -> List[str]: diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 613469eed..9bc1a067d 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -44,7 +44,7 @@ class Check_creds(interfaces.plugins.PluginInterface): cred_addr = task.cred.dereference().vol.offset - if not cred_addr in creds: + if cred_addr not in creds: creds[cred_addr] = [] creds[cred_addr].append(task.pid) diff --git a/volatility3/framework/plugins/mac/list_files.py b/volatility3/framework/plugins/mac/list_files.py index 19f28b18f..8bae986b7 100644 --- a/volatility3/framework/plugins/mac/list_files.py +++ b/volatility3/framework/plugins/mac/list_files.py @@ -72,7 +72,7 @@ class List_Files(plugins.PluginInterface): key = vnode.vol.offset added = False - if not key in loop_vnodes: + if key not in loop_vnodes: # We can't do anything with a no-name vnode v_name = cls._vnode_name(vnode) if v_name is None: @@ -108,7 +108,7 @@ class List_Files(plugins.PluginInterface): added = True parent = cls._get_parent(context, vnode) - while parent and not parent in loop_vnodes: + while parent and parent not in loop_vnodes: if not cls._walk_vnode(context, parent, loop_vnodes): break From e1942976bf95d0952d020dccd504c43751dee8a9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 12:24:22 +1100 Subject: [PATCH 05/63] wrong comparison with None --- volatility3/framework/plugins/linux/check_syscall.py | 2 +- volatility3/framework/symbols/windows/extensions/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 729a0bec6..87d252cd5 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -152,7 +152,7 @@ class Check_syscall(plugins.PluginInterface): except exceptions.SymbolError: ia32_symbol = None - if ia32_symbol != None: + if ia32_symbol is not None: ia32_info = self._get_table_info(vmlinux, "ia32_sys_call_table", ptr_sz) tables.append(("32bit", ia32_info)) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ae7c45d04..55f237581 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -84,7 +84,7 @@ class MMVAD_SHORT(objects.StructType): if tag in ["VadS", "VadF"]: target = "_MMVAD_SHORT" - elif tag != None and tag.startswith("Vad"): + elif tag is not None and tag.startswith("Vad"): target = "_MMVAD" elif depth == 0: # the root node at depth 0 is allowed to not have a tag @@ -651,7 +651,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): except AttributeError: return False - return value != 0 and value != None + return not value def get_vad_root(self): From d2ad867d1e422579e3ef024d342bfeb0658a054f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 10 Dec 2021 13:20:43 +1100 Subject: [PATCH 06/63] my mistake, it should be negated twice to get True when is valid --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 55f237581..7c931f148 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -651,7 +651,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): except AttributeError: return False - return not value + return not not value def get_vad_root(self): From c5c1f355ef7dd20a024a5358db7f7ae758187af7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Dec 2021 09:51:25 +1100 Subject: [PATCH 07/63] Changing `not not` for a more explicit if statement --- volatility3/framework/symbols/windows/extensions/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 7c931f148..84c47e733 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -651,7 +651,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): except AttributeError: return False - return not not value + if value: + return True + + return False def get_vad_root(self): From 997b8572465ef811677c3e2775655e731cdb27fd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 19 Dec 2021 22:16:53 +0000 Subject: [PATCH 08/63] CLI: Add in None renderer to avoid text output --- volatility3/cli/text_renderer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 35b2468e8..e62f705ce 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -182,6 +182,17 @@ class QuickTextRenderer(CLIRenderer): outfd.write("\n") +class NoneRenderer(CLIRenderer): + """Outputs no results""" + name = "none" + + def get_render_options(self): + pass + + def render(self, grid: interfaces.renderers.TreeGrid) -> None: + if not grid.populated: + grid.populate(lambda x, y: True, True) + class CSVRenderer(CLIRenderer): _type_renderers = { format_hints.Bin: quoted_optional(lambda x: f"0b{x:b}"), From 3bc90b82009c8cb4774c635714a5c5d7009d2a5e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 20 Dec 2021 20:38:03 +0000 Subject: [PATCH 09/63] Plugins: Add more information to layerwriter --list --- volatility3/framework/plugins/layerwriter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index b2a02116e..0068ec224 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -110,9 +110,9 @@ class LayerWriter(plugins.PluginInterface): def _generate_layers(self): """List layer names from this run""" for name in self.context.layers: - yield (0, (name, )) + yield (0, (name, self.context.layers[name].__class__.__name__)) def run(self): if self.config['list']: - return renderers.TreeGrid([("Layer name", str)], self._generate_layers()) + return renderers.TreeGrid([("Layer name", str), ('Layer type', str)], self._generate_layers()) return renderers.TreeGrid([("Status", str)], self._generator()) From a96f5de6a0d635e381717b84f5bf643d7bbc6a3b Mon Sep 17 00:00:00 2001 From: cstation Date: Tue, 28 Dec 2021 18:45:08 +0100 Subject: [PATCH 10/63] QEMU: Add 'dirty-bitmap' and 'pbs-state', handle page_size more consistently --- volatility3/framework/layers/qemu.py | 31 +++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 1c5319dfd..df383a04a 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -54,7 +54,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, name: str) -> Any: """Reads the JSON configuration from the end of the file""" - chunk_size = 0x4096 + chunk_size = 4096 data = b'' for i in range(base_layer.maximum_address, base_layer.minimum_address, -chunk_size): if i != base_layer.maximum_address: @@ -65,6 +65,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if start_of_json >= 0: data = data[start_of_json:] return json.loads(data) + # No JSON configuration found at the end of the file, return empty dict return dict() raise exceptions.LayerException(name, "Invalid JSON configuration at the end of the file") @@ -79,9 +80,11 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): addr = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', offset = index, layer_name = self._base_layer) + # Flags are stored in the n least significant bits, where n equals the bit-length of pagesize flags = addr & (page_size - 1) - page_size_bits = int(math.log(page_size, 2)) - addr = (addr >> page_size_bits) << page_size_bits + # addr equals the highest multiple of pagesize <= offset + # (We assume that page_size is a power of 2) + addr = addr ^ (addr & (page_size - 1)) index += 8 if flags & self.SEGMENT_FLAG_MEM_SIZE: @@ -126,6 +129,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._configuration = self._read_configuration(base_layer, self.name) section_byte = -1 index = 8 + section_info = dict() current_section_id = -1 version_id = -1 name = None @@ -162,6 +166,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): offset = index, layer_name = self._base_layer) index += 4 + # Store section info for handling QEVM_SECTION_PARTs later on + section_info[current_section_id] = {'name': name, 'version_id': version_id} # Read additional data index = self.extract_data(index, name, version_id) elif section_byte == self.QEVM_SECTION_PART or section_byte == self.QEVM_SECTION_END: @@ -171,7 +177,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): current_section_id = section_id index += 4 # Read additional data - index = self.extract_data(index, name, version_id) + index = self.extract_data(index, section_info[current_section_id]['name'], + section_info[current_section_id]['version_id']) elif section_byte == self.QEVM_SECTION_FOOTER: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', offset = index, @@ -189,7 +196,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if name == 'ram': if version_id != 4: raise exceptions.LayerException(f"QEMU unknown RAM version_id {version_id}") - new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', None) or 4096) + new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', 4096)) self._segments += new_segments elif name == 'spapr/htab': if version_id != 1: @@ -208,6 +215,13 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): layer_name = self._base_layer) htab_index, htab_n_valid, htab_n_invalid = htab index += 8 + (htab_n_valid * self.HASH_PTE_SIZE_64) + elif name == 'dirty-bitmap': + index += 1 + elif name == 'pbs-state': + section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', + offset = index, + layer_name = self._base_layer) + index += 8 + section_len return index def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes: @@ -217,9 +231,12 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): of the starting data. It is the responsibility of the layer to turn the provided data chunk into the right portion of data necessary. """ - start_offset = offset ^ (offset & 0xfff) + page_size = self._configuration.get('page_size', 4096) + # start_offset equals the highest multiple of pagesize <= offset + # (We assume that page_size is a power of 2) + start_offset = offset ^ (offset & (page_size - 1)) if start_offset in self._compressed: - data = (data * 0x1000) + data = (data * page_size) result = data[offset - start_offset:output_length + offset - start_offset] return result From c8dd8d08bda450d29cec91b797eb4094fbfed0e0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Dec 2021 22:26:28 +0000 Subject: [PATCH 11/63] Volshell: Synchronize with the standard CLI cache clearing --- volatility3/cli/volshell/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 94f735ba0..409123457 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -138,8 +138,7 @@ class VolShell(cli.CommandLine): console.setLevel(10 - (partial_args.verbosity - 2)) if partial_args.clear_cache: - for cache_filename in glob.glob(os.path.join(constants.CACHE_PATH, '*.cache')): - os.unlink(cache_filename) + framework.clear_cache() # Do the initialization ctx = contexts.Context() # Construct a blank context From f38fc22002714eb87734b1049cb3858e222d2fc0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Dec 2021 23:51:28 +0000 Subject: [PATCH 12/63] CLI: Support multi-line fields and tabstops in pretty renderer --- volatility3/cli/text_renderer.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index e62f705ce..8663bb995 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -283,9 +283,10 @@ class PrettyTextRenderer(CLIRenderer): column = grid.columns[column_index] renderer = self._type_renderers.get(column.type, self._type_renderers['default']) data = renderer(node.values[column_index]) + field_width = max([len(self.tab_stop(x)) for x in f"{data}".split("\n")]) max_column_widths[column.name] = max(max_column_widths.get(column.name, len(column.name)), - len(f"{data}")) - line[column] = data + field_width) + line[column] = data.split("\n") accumulator.append((node.path_depth, line)) return accumulator @@ -307,7 +308,25 @@ class PrettyTextRenderer(CLIRenderer): column_titles = [""] + [column.name for column in grid.columns] outfd.write(format_string.format(*column_titles)) for (depth, line) in final_output: - outfd.write(format_string.format("*" * depth, *[line[column] for column in grid.columns])) + nums_line = max([len(line[column]) for column in line]) + for column in line: + line[column] = line[column] + ([""] * (nums_line - len(line[column]))) + for index in range(nums_line): + if index == 0: + outfd.write(format_string.format("*" * depth, *[self.tab_stop(line[column][index]) for column in grid.columns])) + else: + outfd.write(format_string.format(" " * depth, *[self.tab_stop(line[column][index]) for column in grid.columns])) + + def tab_stop(self, line: str) -> str: + tab_width = 8 + while line.find('\t') >= 0: + i = line.find('\t') + if (tab_width > 0): + pad = " " * (tab_width - (i % tab_width)) + else: + pad = "" + line = line.replace("\t", pad, 1) + return line class JsonRenderer(CLIRenderer): From c13b49262d5ee5049d34d2781d319fbc9ac578d8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 30 Dec 2021 01:47:45 +0000 Subject: [PATCH 13/63] Renderers: Make appending rows massively more efficient Previously we were generating the list of children (for most likely the root node) for every single append statement, during which we were recalculating the length of the list, twice. In plugins that output a lot of rows this would add an enourmous overhead (that likely grew as the length of the output grew). Without this overhead, the time taken for the TreeGrid._append method went from 1230.0s to 2.4s. --- volatility3/framework/renderers/__init__.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index de1b14c91..5773861d9 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -272,20 +272,26 @@ class TreeGrid(interfaces.renderers.TreeGrid): def _append(self, parent: Optional[interfaces.renderers.TreeNode], values: Any) -> TreeNode: """Adds a new node at the top level if parent is None, or under the parent node otherwise, after all other children.""" - children = self.children(parent) - return self._insert(parent, len(children), values) + return self._insert(parent, None, values) - def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: int, values: Any) -> TreeNode: + def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: Optional[int], values: Any) -> TreeNode: """Inserts an element into the tree at a specific position.""" parent_path = "" children = self._find_children(parent) if parent is not None: parent_path = parent.path + self.path_sep - newpath = parent_path + str(position) + if position is None: + newpath = parent_path + str(len(children)) + else: + newpath = parent_path + str(position) + for node, _ in children[position:]: + self.visit(node, lambda child, _: child.path_changed(newpath, True), None) + tree_item = TreeNode(newpath, self, parent, values) - for node, _ in children[position:]: - self.visit(node, lambda child, _: child.path_changed(newpath, True), None) - children.insert(position, (tree_item, [])) + if position is None: + children.append((tree_item, [])) + else: + children.insert(position, (tree_item, [])) return tree_item def is_ancestor(self, node, descendant): From 571ab8f6590ec5306af8c6e391e3a481c2580560 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 31 Dec 2021 00:51:21 +0000 Subject: [PATCH 14/63] Volshell: Update the linux pslist requirement --- volatility3/cli/volshell/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 850e3111c..4338ae06f 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -17,7 +17,7 @@ class Volshell(generic.Volshell): def get_requirements(cls): return (super().get_requirements() + [ requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) From ac7127d05a5dc9a81870492d851b0c74c8eb8a43 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 31 Dec 2021 00:55:23 +0000 Subject: [PATCH 15/63] Volshell: Sync errors with the CLI concerning unsatisfied requirements Fixes #607 --- volatility3/cli/volshell/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 409123457..7b8a759a6 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -238,9 +238,14 @@ class VolShell(cli.CommandLine): vollog.debug("Writing out configuration data to config.json") with open("config.json", "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + except exceptions.UnsatisfiedException as excp: + self.process_unsatisfied_exceptions(excp) + parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") + try: # Construct and run the plugin - constructed.run() + if constructed: + constructed.run() except exceptions.VolatilityException as excp: self.process_exceptions(excp) parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") From dd13b427e011a84f94ebc571608a5375f5f9ccea Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 31 Dec 2021 00:51:21 +0000 Subject: [PATCH 16/63] Volshell: Update the linux pslist requirement --- volatility3/cli/volshell/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 850e3111c..4338ae06f 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -17,7 +17,7 @@ class Volshell(generic.Volshell): def get_requirements(cls): return (super().get_requirements() + [ requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) From e91e65d64b82968d466c879b1ac29c7f1d701e13 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 30 Dec 2021 01:47:45 +0000 Subject: [PATCH 17/63] Renderers: Make appending rows massively more efficient Previously we were generating the list of children (for most likely the root node) for every single append statement, during which we were recalculating the length of the list, twice. In plugins that output a lot of rows this would add an enourmous overhead (that likely grew as the length of the output grew). Without this overhead, the time taken for the TreeGrid._append method went from 1230.0s to 2.4s. --- volatility3/framework/renderers/__init__.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index de1b14c91..5773861d9 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -272,20 +272,26 @@ class TreeGrid(interfaces.renderers.TreeGrid): def _append(self, parent: Optional[interfaces.renderers.TreeNode], values: Any) -> TreeNode: """Adds a new node at the top level if parent is None, or under the parent node otherwise, after all other children.""" - children = self.children(parent) - return self._insert(parent, len(children), values) + return self._insert(parent, None, values) - def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: int, values: Any) -> TreeNode: + def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: Optional[int], values: Any) -> TreeNode: """Inserts an element into the tree at a specific position.""" parent_path = "" children = self._find_children(parent) if parent is not None: parent_path = parent.path + self.path_sep - newpath = parent_path + str(position) + if position is None: + newpath = parent_path + str(len(children)) + else: + newpath = parent_path + str(position) + for node, _ in children[position:]: + self.visit(node, lambda child, _: child.path_changed(newpath, True), None) + tree_item = TreeNode(newpath, self, parent, values) - for node, _ in children[position:]: - self.visit(node, lambda child, _: child.path_changed(newpath, True), None) - children.insert(position, (tree_item, [])) + if position is None: + children.append((tree_item, [])) + else: + children.insert(position, (tree_item, [])) return tree_item def is_ancestor(self, node, descendant): From 7aed488721153c7e786de7775a83f1056f3d8d16 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jan 2022 01:21:37 +0000 Subject: [PATCH 18/63] Automagic: Allow automagic to exclude unsupported OSes --- doc/source/using-as-a-library.rst | 3 ++- doc/source/vol2to3.rst | 4 ++++ volatility3/framework/automagic/__init__.py | 21 ++++++------------- volatility3/framework/automagic/linux.py | 2 ++ volatility3/framework/automagic/mac.py | 2 ++ volatility3/framework/constants/__init__.py | 1 + volatility3/framework/interfaces/automagic.py | 3 +++ 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index 95d2b1080..c63adcfc3 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -131,7 +131,8 @@ A suitable list of automagics for a particular plugin (based on operating system automagics = automagic.choose_automagic(available_automagics, plugin) This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just -the automagics which apply to the operating system. +the automagics which apply to the operating system. Each automagic can exclude itself from being used for specific +operating systems, so that an automagic designed for linux is not used for windows or mac plugins. These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that the plugin will be constructed on ('plugins' by default) and a progress_callback. This is a callable which takes diff --git a/doc/source/vol2to3.rst b/doc/source/vol2to3.rst index eb33b6618..e768df0c2 100644 --- a/doc/source/vol2to3.rst +++ b/doc/source/vol2to3.rst @@ -62,6 +62,10 @@ automagic processes are clearly defined and can be enabled or disabled as necess included a stacker automagic to emulate the most common feature of Volatility 2, automatically stacking address spaces (now translation layers) on top of each other. +By default the automagic chosen to be run are determined based on the plugin requested, so that linux plugins get linux +specific automagic and windows plugins get windows specific automagic. This should reduce unnecessarily searching for +linux kernels in a windows image, for example. At the moment this is not user configurableS. + Searching and Scanning ---------------------- Scanning is very similar to scanning in Volatility 2, a scanner object (such as a diff --git a/volatility3/framework/automagic/__init__.py b/volatility3/framework/automagic/__init__.py index e4d422c99..7567f206d 100644 --- a/volatility3/framework/automagic/__init__.py +++ b/volatility3/framework/automagic/__init__.py @@ -21,14 +21,6 @@ from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) -windows_automagic = [ - 'ConstructionMagic', 'LayerStacker', 'KernelPDBScanner', 'WinSwapLayers', 'KernelModule' -] - -linux_automagic = ['ConstructionMagic', 'LayerStacker', 'LinuxBannerCache', 'LinuxSymbolFinder', 'KernelModule'] - -mac_automagic = ['ConstructionMagic', 'LayerStacker', 'MacBannerCache', 'MacSymbolFinder', 'KernelModule'] - def available(context: interfaces.context.ContextInterface) -> List[interfaces.automagic.AutomagicInterface]: """Returns an ordered list of all subclasses of @@ -58,10 +50,7 @@ def choose_automagic( plugin_category = "None" plugin_categories = plugin.__module__.split('.') lowest_index = len(plugin_categories) - - automagic_categories = {'windows': windows_automagic, 'linux': linux_automagic, 'mac': mac_automagic} - - for os in automagic_categories: + for os in constants.OS_CATEGORIES: try: if plugin_categories.index(os) < lowest_index: lowest_index = plugin_categories.index(os) @@ -70,14 +59,16 @@ def choose_automagic( # The value wasn't found, try the next one pass - if plugin_category not in automagic_categories: + if plugin_category not in constants.OS_CATEGORIES: vollog.info("No plugin category detected") return automagics - vollog.info(f"Detected a {plugin_category} category plugin") + output = [] for amagic in automagics: - if amagic.__class__.__name__ in automagic_categories[plugin_category]: + if plugin_category not in amagic.exclusion_list: + # Only include uncategorized automagic, or platform specific automagic + # (This allows user defined/uncategorized automagic to be included) output += [amagic] return output diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f9fa22c07..a6577e322 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -147,6 +147,7 @@ class LinuxBannerCache(symbol_cache.SymbolBannerCache): os = "linux" symbol_name = "linux_banner" banner_path = constants.LINUX_BANNERS_PATH + exclusion_list = ['mac', 'windows'] class LinuxSymbolFinder(symbol_finder.SymbolFinder): @@ -156,3 +157,4 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder): banner_cache = LinuxBannerCache symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] + exclusion_list = ['mac', 'windows'] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index fb725a234..c37aef463 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -202,6 +202,7 @@ class MacBannerCache(symbol_cache.SymbolBannerCache): os = "mac" symbol_name = "version" banner_path = constants.MAC_BANNERS_PATH + exclusion_list = ['windows', 'linux'] class MacSymbolFinder(symbol_finder.SymbolFinder): @@ -211,3 +212,4 @@ class MacSymbolFinder(symbol_finder.SymbolFinder): banner_cache = MacBannerCache find_aslr = MacIntelStacker.find_aslr symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols" + exclusion_list = ['windows', 'linux'] diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 23598837b..82ebd4936 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -78,6 +78,7 @@ BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" ProgressCallback = Optional[Callable[[float, str], None]] """Type information for ProgressCallback objects""" +OS_CATEGORIES = ['windows', 'mac', 'linux'] class Parallelism(enum.IntEnum): """An enumeration listing the different types of parallelism applied to diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index c310f5b4a..c96c9bdbe 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -40,6 +40,9 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla priority = 10 """An ordering to indicate how soon this automagic should be run""" + exclusion_list = [] + """A list of plugin categories (typically operating systems) which the plugin will not operate on""" + def __init__(self, context: interfaces.context.ContextInterface, config_path: str, *args, **kwargs) -> None: super().__init__(context, config_path) for requirement in self.get_requirements(): From 2c949dfa50e2b530a3df9cc480933e583a1ca4e7 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Thu, 6 Jan 2022 22:53:24 +0000 Subject: [PATCH 19/63] Create MFTScanner plugin --- .../framework/plugins/windows/mftscan.py | 305 ++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 volatility3/framework/plugins/windows/mftscan.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py new file mode 100644 index 000000000..1d4ca954e --- /dev/null +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -0,0 +1,305 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from struct import unpack +from typing import Iterable + +from volatility3.framework import constants, renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework.renderers import conversion, format_hints +from volatility3.framework.symbols import intermed +from volatility3.plugins import yarascan + +vollog = logging.getLogger(__name__) + +try: + import yara +except ImportError: + vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") + raise + +signatures = { + 'mft_objects': """rule mft_headers + { + strings: + $header1 = "FILE0" + $header2 = "FILE*" + $header3 = "BAAD" + condition: + any of them + }""" +} + +# https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee9338b881c0fa25937/volatility/plugins/mftparser.py#L60 +ATTRIBUTE_TYPE_ID = { + 0x10:"STANDARD_INFORMATION", + 0x20:"ATTRIBUTE_LIST", + 0x30:"FILE_NAME", + 0x40:"OBJECT_ID", + 0x50:"SECURITY_DESCRIPTOR", + 0x60:"VOLUME_NAME", + 0x70:"VOLUME_INFORMATION", + 0x80:"DATA", + 0x90:"INDEX_ROOT", + 0xa0:"INDEX_ALLOCATION", + 0xb0:"BITMAP", + 0xc0:"REPARSE_POINT", + 0xd0:"EA_INFORMATION", #Extended Attribute + 0xe0:"EA", + 0xf0:"PROPERTY_SET", + 0x100:"LOGGED_UTILITY_STREAM", +} + +VERBOSE_STANDARD_INFO_FLAGS = { + 0x1:"Read Only", + 0x2:"Hidden", + 0x4:"System", + 0x20:"Archive", + 0x40:"Device", + 0x80:"Normal", + 0x100:"Temporary", + 0x200:"Sparse File", + 0x400:"Reparse Point", + 0x800:"Compressed", + 0x1000:"Offline", + 0x2000:"Content not indexed", + 0x4000:"Encrypted", + 0x10000000:"Directory", + 0x20000000:"Index view", +} + +FILE_NAME_NAMESPACE = { + 0x0:"POSIX", # Case sensitive, allows all Unicode chars except '/' and NULL + 0x1:"Win32", # Case insensitive, allows most Unicide except specials ('/', '\', ';', '>', '<', '?') + 0x2:"DOS", # Case insensitive, upper case, no special chars, name is 8 or fewer chars in name and 3 or less extension + 0x3:"Win32 & DOS", # Used when original name fits in DOS namespace and 2 names are not needed +} + +MFT_FLAGS = { + 0x0: "Removed", + 0x1: "File", # "In Use", + 0x2: "Directory", # if flag & 0x0002 == 0 this is a regular file + 0x3: "Directory" +} + +INDEX_ENTRY_FLAGS = { + 0x1:"Child Node Exists", + 0x2:"Last entry in list", +} + + +class MFTScan(interfaces.plugins.PluginInterface): + """Scans for MFT FILE objects present in a particular windows memory image.""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, + version = (2, 0, 0)), + ] + + # https://docs.python.org/3/library/struct.html + @classmethod + def unpack_data(self, mft_record, offset, data_type): + """Helper to unpack values from the raw mft_record""" + + if data_type == 'unsigned long': + return unpack(' 1000: + continue + + # attr_header + attr_type = self.unpack_data(mft_record, attr_offset, 'int') + attr_len = self.unpack_data(mft_record, attr_offset+4, 'int') + + # As we look for strucutres of header + 1K we can not unpack non resident structures + nr_flag = self.unpack_data(mft_record, attr_offset+8, 'unsigned char') + + # Skip headers + attr_data = attr_offset+24 # Len of Common and Resident Headers + + if attr_type in ATTRIBUTE_TYPE_ID: + vollog.debug(f'Found Attribute {ATTRIBUTE_TYPE_ID[attr_type]}') + + if ATTRIBUTE_TYPE_ID[attr_type] == 'STANDARD_INFORMATION': + creation_time_win = self.unpack_data(mft_record, attr_data, 'unsigned long long') + modified_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') + altered_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') + access_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') + flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') + permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') + + + mft_entry['attributes']['SI'] = { + "creation_time": self.human_date(creation_time_win), + "modified_time": self.human_date(modified_time_win), + "updated_time": self.human_date(altered_time_win), + "accessed_time": self.human_date(access_time_win), + "flags": permissions + } + + if ATTRIBUTE_TYPE_ID[attr_type] == 'FILE_NAME': + parent_record = self.unpack_data(mft_record, attr_data, 'unsigned long long') + creation_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') + modified_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') + altered_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') + access_time_win = self.unpack_data(mft_record, attr_data+32, 'unsigned long long') + + name_len = self.unpack_data(mft_record, attr_data+64, 'unsigned char') + name_space = self.unpack_data(mft_record, attr_data+65, 'unsigned char') + + # Unicode and partially corruprted records can break us here. + file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] + try: + file_name = file_name.replace(b'\x00', b'').decode() + except: + file_name = str(file_name.replace(b'\x00', b'')) + + flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') + permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') + + mft_entry['attributes']['FN'].append( + { + "creation_time": self.human_date(creation_time_win), + "modified_time": self.human_date(modified_time_win), + "updated_time": self.human_date(altered_time_win), + "accessed_time": self.human_date(access_time_win), + "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), + "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), + "flags": permissions, + "file_name": file_name, + "name_space": name_space + }) + + # Update Offset for next Attribute + attr_offset += attr_len + + return mft_entry + + def _generator(self): + rules = yara.compile(sources = signatures) + + layer = self.context.layers[self.config['primary']] + for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + + try: + mft_record = layer.read(offset, 1024, False) + mft_entry = self.parse_mft_record(mft_record) + except PagedInvalidAddressException: + mft_entry = None + except Exception as err: + vollog.error(err) + mft_entry = None + + if mft_entry: + vollog.debug(mft_entry) + + # Tree Grid is large and variable + si = mft_entry['attributes']['SI'] + fn = mft_entry['attributes']['FN'] + + signature = mft_entry.get('signature', 0) + record_number = mft_entry.get('record_number', 0) + link_count = mft_entry.get('link_count', 0) + permissions = mft_entry.get('flags', '') + + si_creation_time = si.get('creation_time', '') + si_modified_time = si.get('modified_time', '') + si_updated_time = si.get('updated_time', '') + si_accessed_time = si.get('accessed_time', '') + + yield 0, ( + format_hints.Hex(offset), + signature, + record_number, + link_count, + permissions, + 'Standard Information', + 'N/A', + si_creation_time, + si_modified_time, + si_updated_time, + si_accessed_time) + + for entry in fn: + # As this is variable and may or may not exist + # And could have 0-6 entries lets do it per row. + yield 0, ( + format_hints.Hex(offset), + signature, + record_number, + link_count, + permissions, + 'FileName', + entry.get('file_name', ''), + entry.get('creation_time', ''), + entry.get('modified_time', ''), + entry.get('updated_time', ''), + entry.get('accessed_time', '')) + + def run(self): + return renderers.TreeGrid([ + ('Offset', format_hints.Hex), + ('Record Type', str), + ('Record Number', int), + ('Link Count', int), + ('Permissions', str), + ('Attribute Type', str), + ('Filename', str), + ('Created', str), + ('Modified', str), + ('Updated', str), + ('Accessed', str) + ],self._generator()) From 899ec09ce3b620f75c9ecb74e117d8a3a918be3b Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Thu, 6 Jan 2022 23:09:50 +0000 Subject: [PATCH 20/63] Doc Strings --- .../framework/plugins/windows/mftscan.py | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 1d4ca954e..b75756a1d 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,13 @@ import logging from struct import unpack -from typing import Iterable +from typing import Dict -from volatility3.framework import constants, renderers, interfaces, exceptions +from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols import intermed from volatility3.plugins import yarascan vollog = logging.getLogger(__name__) @@ -109,8 +109,17 @@ class MFTScan(interfaces.plugins.PluginInterface): # https://docs.python.org/3/library/struct.html @classmethod - def unpack_data(self, mft_record, offset, data_type): - """Helper to unpack values from the raw mft_record""" + def unpack_data(self, mft_record: bytes, offset: int, data_type: str) -> bytes: + """Helper to unpack values from the raw mft_record + + Args: + mft_record: 1024 bytes starting from header value as returned by layer read + offset: how far in to the record to read + data_type: what is the data type to unpack + + Returns: + bytes: the unpacked data + """ if data_type == 'unsigned long': return unpack(' str: + """Converts a windows epoch to a date time string with a fixed format + + Args: + datetime_object: windows epoch time + + Returns: + str: strftime of the windows epoch in UTC + + """ dtg = conversion.wintime_to_datetime(datetime_object) return dtg.strftime('%Y-%m-%d %H:%M:%S %z') @classmethod - def parse_mft_record(self, mft_record): - """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes""" + def parse_mft_record(self, mft_record: bytes) -> Dict: + """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes + + Args: + mft_record: 1024 bytes starting from header value as returned by layer read + + Returns: + Dict: a Dictionary that contains the Parse MFT Record + """ # https://github.com/Invoke-IR/ForensicPosters flags = self.unpack_data(mft_record, 22, 'unsigned short') @@ -202,10 +226,11 @@ class MFTScan(interfaces.plugins.PluginInterface): # Unicode and partially corruprted records can break us here. file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - try: - file_name = file_name.replace(b'\x00', b'').decode() - except: - file_name = str(file_name.replace(b'\x00', b'')) + file_name = utility.array_to_string(file_name) + #try: + # # file_name = file_name.replace(b'\x00', b'').decode() + #except: + # file_name = str(file_name.replace(b'\x00', b'')) flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') @@ -234,6 +259,7 @@ class MFTScan(interfaces.plugins.PluginInterface): layer = self.context.layers[self.config['primary']] for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + # For each matching rule try to read 1024 bytes (size of an MFT record) at the offset. try: mft_record = layer.read(offset, 1024, False) mft_entry = self.parse_mft_record(mft_record) From 20a5868ff3df16710695b497069aba9ff0387842 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 00:23:31 +0000 Subject: [PATCH 21/63] Change return types for MFT Records DTGs --- .../framework/plugins/windows/mftscan.py | 101 ++++++++---------- 1 file changed, 44 insertions(+), 57 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index b75756a1d..ddc179fe1 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -2,14 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime import logging +import struct -from struct import unpack from typing import Dict from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework import exceptions from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints from volatility3.plugins import yarascan @@ -122,29 +123,15 @@ class MFTScan(interfaces.plugins.PluginInterface): """ if data_type == 'unsigned long': - return unpack(' str: - """Converts a windows epoch to a date time string with a fixed format - - Args: - datetime_object: windows epoch time - - Returns: - str: strftime of the windows epoch in UTC - - """ - dtg = conversion.wintime_to_datetime(datetime_object) - return dtg.strftime('%Y-%m-%d %H:%M:%S %z') + return struct.unpack(' Dict: @@ -207,10 +194,10 @@ class MFTScan(interfaces.plugins.PluginInterface): mft_entry['attributes']['SI'] = { - "creation_time": self.human_date(creation_time_win), - "modified_time": self.human_date(modified_time_win), - "updated_time": self.human_date(altered_time_win), - "accessed_time": self.human_date(access_time_win), + "creation_time": conversion.wintime_to_datetime(creation_time_win), + "modified_time": conversion.wintime_to_datetime(modified_time_win), + "updated_time": conversion.wintime_to_datetime(altered_time_win), + "accessed_time": conversion.wintime_to_datetime(access_time_win), "flags": permissions } @@ -226,21 +213,21 @@ class MFTScan(interfaces.plugins.PluginInterface): # Unicode and partially corruprted records can break us here. file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - file_name = utility.array_to_string(file_name) - #try: - # # file_name = file_name.replace(b'\x00', b'').decode() - #except: - # file_name = str(file_name.replace(b'\x00', b'')) + #file_name = utility.array_to_string(file_name) + try: + file_name = file_name.replace(b'\x00', b'').decode() + except: + file_name = str(file_name.replace(b'\x00', b'')) flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') mft_entry['attributes']['FN'].append( { - "creation_time": self.human_date(creation_time_win), - "modified_time": self.human_date(modified_time_win), - "updated_time": self.human_date(altered_time_win), - "accessed_time": self.human_date(access_time_win), + "creation_time": conversion.wintime_to_datetime(creation_time_win), + "modified_time": conversion.wintime_to_datetime(modified_time_win), + "updated_time": conversion.wintime_to_datetime(altered_time_win), + "accessed_time": conversion.wintime_to_datetime(access_time_win), "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), "flags": permissions, @@ -263,11 +250,11 @@ class MFTScan(interfaces.plugins.PluginInterface): try: mft_record = layer.read(offset, 1024, False) mft_entry = self.parse_mft_record(mft_record) - except PagedInvalidAddressException: - mft_entry = None - except Exception as err: - vollog.error(err) + except exceptions.PagedInvalidAddressException: mft_entry = None + #except Exception as err: + # vollog.error(err) + # mft_entry = None if mft_entry: vollog.debug(mft_entry) @@ -276,15 +263,15 @@ class MFTScan(interfaces.plugins.PluginInterface): si = mft_entry['attributes']['SI'] fn = mft_entry['attributes']['FN'] - signature = mft_entry.get('signature', 0) - record_number = mft_entry.get('record_number', 0) - link_count = mft_entry.get('link_count', 0) - permissions = mft_entry.get('flags', '') + signature = mft_entry.get('signature', renderers.NotAvailableValue()) + record_number = mft_entry.get('record_number', renderers.NotAvailableValue()) + link_count = mft_entry.get('link_count', renderers.NotAvailableValue()) + permissions = mft_entry.get('flags', renderers.NotAvailableValue()) - si_creation_time = si.get('creation_time', '') - si_modified_time = si.get('modified_time', '') - si_updated_time = si.get('updated_time', '') - si_accessed_time = si.get('accessed_time', '') + si_creation_time = si.get('creation_time', renderers.NotAvailableValue()) + si_modified_time = si.get('modified_time', renderers.NotAvailableValue()) + si_updated_time = si.get('updated_time', renderers.NotAvailableValue()) + si_accessed_time = si.get('accessed_time', renderers.NotAvailableValue()) yield 0, ( format_hints.Hex(offset), @@ -293,7 +280,7 @@ class MFTScan(interfaces.plugins.PluginInterface): link_count, permissions, 'Standard Information', - 'N/A', + renderers.NotApplicableValue(), si_creation_time, si_modified_time, si_updated_time, @@ -302,18 +289,18 @@ class MFTScan(interfaces.plugins.PluginInterface): for entry in fn: # As this is variable and may or may not exist # And could have 0-6 entries lets do it per row. - yield 0, ( + yield 1, ( format_hints.Hex(offset), signature, record_number, link_count, permissions, 'FileName', - entry.get('file_name', ''), - entry.get('creation_time', ''), - entry.get('modified_time', ''), - entry.get('updated_time', ''), - entry.get('accessed_time', '')) + entry.get('file_name',''), + entry.get('creation_time', renderers.NotAvailableValue()), + entry.get('modified_time', renderers.NotAvailableValue()), + entry.get('updated_time', renderers.NotAvailableValue()), + entry.get('accessed_time', renderers.NotAvailableValue())) def run(self): return renderers.TreeGrid([ @@ -324,8 +311,8 @@ class MFTScan(interfaces.plugins.PluginInterface): ('Permissions', str), ('Attribute Type', str), ('Filename', str), - ('Created', str), - ('Modified', str), - ('Updated', str), - ('Accessed', str) + ('Created', datetime.datetime), + ('Modified', datetime.datetime), + ('Updated', datetime.datetime), + ('Accessed', datetime.datetime) ],self._generator()) From 295fb453f5e73f65d39912336ef6517268e26d17 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 00:55:34 +0000 Subject: [PATCH 22/63] Add MFT Filename N/A type --- volatility3/framework/plugins/windows/mftscan.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index ddc179fe1..10db6f112 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -192,7 +192,6 @@ class MFTScan(interfaces.plugins.PluginInterface): flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - mft_entry['attributes']['SI'] = { "creation_time": conversion.wintime_to_datetime(creation_time_win), "modified_time": conversion.wintime_to_datetime(modified_time_win), @@ -296,7 +295,7 @@ class MFTScan(interfaces.plugins.PluginInterface): link_count, permissions, 'FileName', - entry.get('file_name',''), + entry.get('file_name',renderers.NotAvailableValue()), entry.get('creation_time', renderers.NotAvailableValue()), entry.get('modified_time', renderers.NotAvailableValue()), entry.get('updated_time', renderers.NotAvailableValue()), From 114a8d7c8d0195a83bfc0c0654fb24bd63202302 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 9 Jan 2022 20:32:34 +0000 Subject: [PATCH 23/63] Plugins: Update yarascan options Add in a yara_source option in the process method. Unfortunately yara_rules is still poorly named, but would require a major version bump, so to avoid major disruption, we're just adding the yara_source option instead. --- volatility3/framework/plugins/yarascan.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 94b3cba45..4d2ba88ee 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -3,7 +3,7 @@ # import logging -from typing import Iterable, Tuple, List, Dict, Any +from typing import Any, Dict, Iterable, List, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -40,7 +40,7 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -83,6 +83,8 @@ class YaraScan(plugins.PluginInterface): if config.get('wide', False): rule += " wide ascii" rules = yara.compile(sources = {'n': f'rule r1 {{strings: $a = {rule} condition: $a}}'}) + elif config.get('yara_source', None) is not None: + rules = yara.compile(source = config['yara_source']) elif config.get('yara_file', None) is not None: rules = yara.compile(file = resources.ResourceAccessor().open(config['yara_file'], "rb")) elif config.get('yara_compiled_file', None) is not None: From d34030e9ea44e81287c4836d9851280b3190d096 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 9 Jan 2022 20:52:49 +0000 Subject: [PATCH 24/63] Plugins: Add note to improve yarascan in the future --- volatility3/framework/plugins/yarascan.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 4d2ba88ee..e51669b2c 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -42,6 +42,9 @@ class YaraScan(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 1, 0) + # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string + # or something that makes more sense + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ From 9d86599e7f391c41f2fb7e06f6f0f811bd97a7b7 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 22:04:13 +0000 Subject: [PATCH 25/63] MFT Plugin use ISF instead of Struct --- .../framework/plugins/windows/mftscan.py | 352 ++++------------- .../symbols/windows/extensions/mft.py | 104 +++++ .../framework/symbols/windows/mft.json | 371 ++++++++++++++++++ volatility3/framework/symbols/windows/mft.py | 15 + 4 files changed, 577 insertions(+), 265 deletions(-) create mode 100644 volatility3/framework/symbols/windows/extensions/mft.py create mode 100644 volatility3/framework/symbols/windows/mft.json create mode 100644 volatility3/framework/symbols/windows/mft.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 10db6f112..484fcdb9a 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -4,95 +4,20 @@ import datetime import logging -import struct from typing import Dict from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions -from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints +from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags +from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols + from volatility3.plugins import yarascan vollog = logging.getLogger(__name__) -try: - import yara -except ImportError: - vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") - raise - -signatures = { - 'mft_objects': """rule mft_headers - { - strings: - $header1 = "FILE0" - $header2 = "FILE*" - $header3 = "BAAD" - condition: - any of them - }""" -} - -# https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee9338b881c0fa25937/volatility/plugins/mftparser.py#L60 -ATTRIBUTE_TYPE_ID = { - 0x10:"STANDARD_INFORMATION", - 0x20:"ATTRIBUTE_LIST", - 0x30:"FILE_NAME", - 0x40:"OBJECT_ID", - 0x50:"SECURITY_DESCRIPTOR", - 0x60:"VOLUME_NAME", - 0x70:"VOLUME_INFORMATION", - 0x80:"DATA", - 0x90:"INDEX_ROOT", - 0xa0:"INDEX_ALLOCATION", - 0xb0:"BITMAP", - 0xc0:"REPARSE_POINT", - 0xd0:"EA_INFORMATION", #Extended Attribute - 0xe0:"EA", - 0xf0:"PROPERTY_SET", - 0x100:"LOGGED_UTILITY_STREAM", -} - -VERBOSE_STANDARD_INFO_FLAGS = { - 0x1:"Read Only", - 0x2:"Hidden", - 0x4:"System", - 0x20:"Archive", - 0x40:"Device", - 0x80:"Normal", - 0x100:"Temporary", - 0x200:"Sparse File", - 0x400:"Reparse Point", - 0x800:"Compressed", - 0x1000:"Offline", - 0x2000:"Content not indexed", - 0x4000:"Encrypted", - 0x10000000:"Directory", - 0x20000000:"Index view", -} - -FILE_NAME_NAMESPACE = { - 0x0:"POSIX", # Case sensitive, allows all Unicode chars except '/' and NULL - 0x1:"Win32", # Case insensitive, allows most Unicide except specials ('/', '\', ';', '>', '<', '?') - 0x2:"DOS", # Case insensitive, upper case, no special chars, name is 8 or fewer chars in name and 3 or less extension - 0x3:"Win32 & DOS", # Used when original name fits in DOS namespace and 2 names are not needed -} - -MFT_FLAGS = { - 0x0: "Removed", - 0x1: "File", # "In Use", - 0x2: "Directory", # if flag & 0x0002 == 0 this is a regular file - 0x3: "Directory" -} - -INDEX_ENTRY_FLAGS = { - 0x1:"Child Node Exists", - 0x2:"Last entry in list", -} - - class MFTScan(interfaces.plugins.PluginInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" @@ -108,198 +33,94 @@ class MFTScan(interfaces.plugins.PluginInterface): version = (2, 0, 0)), ] - # https://docs.python.org/3/library/struct.html - @classmethod - def unpack_data(self, mft_record: bytes, offset: int, data_type: str) -> bytes: - """Helper to unpack values from the raw mft_record - - Args: - mft_record: 1024 bytes starting from header value as returned by layer read - offset: how far in to the record to read - data_type: what is the data type to unpack - - Returns: - bytes: the unpacked data - """ - - if data_type == 'unsigned long': - return struct.unpack(' Dict: - """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes - - Args: - mft_record: 1024 bytes starting from header value as returned by layer read - - Returns: - Dict: a Dictionary that contains the Parse MFT Record - """ - # https://github.com/Invoke-IR/ForensicPosters - - flags = self.unpack_data(mft_record, 22, 'unsigned short') - file_type = MFT_FLAGS.get(flags, 'Unknown') - - mft_entry = { - "signature": mft_record[:4].decode(), - "FixupArrayOffset": self.unpack_data(mft_record, 4, 'unsigned short'), - "NumFixupEntries": self.unpack_data(mft_record, 6, 'unsigned short'), - "LSN": self.unpack_data(mft_record, 8, 'unsigned long long'), - "SequenceValue": self.unpack_data(mft_record, 16, 'unsigned short'), - "link_count": self.unpack_data(mft_record, 18, 'unsigned short'), - "FirstAttrOffset": self.unpack_data(mft_record, 20, 'unsigned short'), - "flags": file_type, - "record_number": self.unpack_data(mft_record, 44, 'unsigned long'), - "attributes": { - "SI": {}, - "FN": [] - } - } - - attr_offset = mft_entry['FirstAttrOffset'] - # Check at most for 6 entries - for i in range(6): - # If we attempt to overread the entry continue out - if attr_offset > 1000: - continue - - # attr_header - attr_type = self.unpack_data(mft_record, attr_offset, 'int') - attr_len = self.unpack_data(mft_record, attr_offset+4, 'int') - - # As we look for strucutres of header + 1K we can not unpack non resident structures - nr_flag = self.unpack_data(mft_record, attr_offset+8, 'unsigned char') - - # Skip headers - attr_data = attr_offset+24 # Len of Common and Resident Headers - - if attr_type in ATTRIBUTE_TYPE_ID: - vollog.debug(f'Found Attribute {ATTRIBUTE_TYPE_ID[attr_type]}') - - if ATTRIBUTE_TYPE_ID[attr_type] == 'STANDARD_INFORMATION': - creation_time_win = self.unpack_data(mft_record, attr_data, 'unsigned long long') - modified_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') - altered_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') - access_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') - flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') - permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - - mft_entry['attributes']['SI'] = { - "creation_time": conversion.wintime_to_datetime(creation_time_win), - "modified_time": conversion.wintime_to_datetime(modified_time_win), - "updated_time": conversion.wintime_to_datetime(altered_time_win), - "accessed_time": conversion.wintime_to_datetime(access_time_win), - "flags": permissions - } - - if ATTRIBUTE_TYPE_ID[attr_type] == 'FILE_NAME': - parent_record = self.unpack_data(mft_record, attr_data, 'unsigned long long') - creation_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') - modified_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') - altered_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') - access_time_win = self.unpack_data(mft_record, attr_data+32, 'unsigned long long') - - name_len = self.unpack_data(mft_record, attr_data+64, 'unsigned char') - name_space = self.unpack_data(mft_record, attr_data+65, 'unsigned char') - - # Unicode and partially corruprted records can break us here. - file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - #file_name = utility.array_to_string(file_name) - try: - file_name = file_name.replace(b'\x00', b'').decode() - except: - file_name = str(file_name.replace(b'\x00', b'')) - - flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') - permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - - mft_entry['attributes']['FN'].append( - { - "creation_time": conversion.wintime_to_datetime(creation_time_win), - "modified_time": conversion.wintime_to_datetime(modified_time_win), - "updated_time": conversion.wintime_to_datetime(altered_time_win), - "accessed_time": conversion.wintime_to_datetime(access_time_win), - "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), - "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), - "flags": permissions, - "file_name": file_name, - "name_space": name_space - }) - - # Update Offset for next Attribute - attr_offset += attr_len - - return mft_entry def _generator(self): - rules = yara.compile(sources = signatures) - layer = self.context.layers[self.config['primary']] + + # Yara Rule to scan for MFT Header Signatures + rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) + + # Read in the Symbol File + symbol_table = MFTIntermedSymbols.create( + self.context, + self.config_path, + "windows", + "mft" + ) + + # get each of the individual Field Sets + mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + header_object = symbol_table + constants.BANG + "ATTR_HEADER" + si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Scan the layer for Raw MFT records and parse the fields for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): - - # For each matching rule try to read 1024 bytes (size of an MFT record) at the offset. try: - mft_record = layer.read(offset, 1024, False) - mft_entry = self.parse_mft_record(mft_record) + mft_record = self.context.object(mft_object, offset=offset, layer_name=layer.name) + # We will update this on each pass in the next loop and use it as the new offset. + attr_base_offset = mft_record.FirstAttrOffset + + # There is no field that has a count of Attributes + # Keep Attempting to read attributes until we get an invalid attr_header.AttrType + while True: + attr_header = self.context.object(header_object, offset=offset+attr_base_offset, layer_name=layer.name) + attr_resident_header = self.context.object(header_object, offset=offset+attr_base_offset+16, layer_name=layer.name) + + vollog.debug(f"Attr Type: {attr_header.AttrType}") + + # If this is not a valid type then exit the loop + if not AttributeTypes(attr_header.AttrType).value: + break + + # Offset past the headers to the attribute data + attr_data_offset = offset+attr_base_offset+24 + + # Standard Information Attribute + if attr_header.AttrType == 0x10: + attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) + + yield 0, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + MFTFlags(mft_record.Flags).name, + renderers.NotApplicableValue(), + AttributeTypes(attr_header.AttrType).name, + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + renderers.NotApplicableValue(), + ) + + # File Name Attribute + if attr_header.AttrType == 0x30: + attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) + file_name = attr_data.get_full_name() + + yield 1, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + MFTFlags(mft_record.Flags).name, + PermissionFlags(attr_data.Flags).name, + AttributeTypes(attr_header.AttrType).name, + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + file_name + ) + + # Update the base offset to point to the next attribute + attr_base_offset += attr_header.Length + except exceptions.PagedInvalidAddressException: - mft_entry = None - #except Exception as err: - # vollog.error(err) - # mft_entry = None + pass - if mft_entry: - vollog.debug(mft_entry) - - # Tree Grid is large and variable - si = mft_entry['attributes']['SI'] - fn = mft_entry['attributes']['FN'] - - signature = mft_entry.get('signature', renderers.NotAvailableValue()) - record_number = mft_entry.get('record_number', renderers.NotAvailableValue()) - link_count = mft_entry.get('link_count', renderers.NotAvailableValue()) - permissions = mft_entry.get('flags', renderers.NotAvailableValue()) - - si_creation_time = si.get('creation_time', renderers.NotAvailableValue()) - si_modified_time = si.get('modified_time', renderers.NotAvailableValue()) - si_updated_time = si.get('updated_time', renderers.NotAvailableValue()) - si_accessed_time = si.get('accessed_time', renderers.NotAvailableValue()) - - yield 0, ( - format_hints.Hex(offset), - signature, - record_number, - link_count, - permissions, - 'Standard Information', - renderers.NotApplicableValue(), - si_creation_time, - si_modified_time, - si_updated_time, - si_accessed_time) - - for entry in fn: - # As this is variable and may or may not exist - # And could have 0-6 entries lets do it per row. - yield 1, ( - format_hints.Hex(offset), - signature, - record_number, - link_count, - permissions, - 'FileName', - entry.get('file_name',renderers.NotAvailableValue()), - entry.get('creation_time', renderers.NotAvailableValue()), - entry.get('modified_time', renderers.NotAvailableValue()), - entry.get('updated_time', renderers.NotAvailableValue()), - entry.get('accessed_time', renderers.NotAvailableValue())) def run(self): return renderers.TreeGrid([ @@ -307,11 +128,12 @@ class MFTScan(interfaces.plugins.PluginInterface): ('Record Type', str), ('Record Number', int), ('Link Count', int), + ('MFT Type', str), ('Permissions', str), ('Attribute Type', str), - ('Filename', str), ('Created', datetime.datetime), ('Modified', datetime.datetime), ('Updated', datetime.datetime), - ('Accessed', datetime.datetime) + ('Accessed', datetime.datetime), + ('Filename', str), ],self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py new file mode 100644 index 000000000..09f6346cc --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -0,0 +1,104 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import enum + +from volatility3.framework import exceptions, objects, renderers +from volatility3.framework.objects import utility + + +class AttributeTypes(enum.Enum): + STANDARD_INFORMATION = 0x10 + ATTRIBUTE_LIST = 0x20 + FILE_NAME = 0x30 + OBJECT_ID = 0x40 + SECURITY_DESCRIPTOR = 0x50 + VOLUME_NAME = 0x60 + VOLUME_INFORMATION = 0x70 + DATA = 0x80 + INDEX_ROOT = 0x90 + INDEX_ALLOCATION = 0xa0 + BITMAP = 0xb0 + REPARSE_POINT = 0xc0 + EA_INFORMATION = 0xd0 + EA = 0xe0 + PROPERTY_SET = 0xf0 + LOGGED_UTILITY_STREAM = 0x100 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(AttributeTypes.Unknown) + +class NameSpace(enum.Enum): + POSIX = 0x0 + Win32 = 0x1 + DOS = 0x2 + Win32DOS = 0x3 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(NameSpace.Unknown) + + +class MFTFlags(enum.Enum): + Removed = 0x00 + File = 0x1 + Directory = 0x2 + DirInUse = 0x3 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(MFTFlags.Unknown) + + +class PermissionFlags(enum.Enum): + ReadOnly = 0x1 + Hidden = 0x2 + System = 0x4 + Archive = 0x20 + ArchiveHidden = 0x22 + ArchiveSystem = 0x24 + ArchiveHiddenSystem = 0x26 + Device = 0x40 + Normal = 0x80 + Temporary = 0x100 + TempArchive = 0x120 + SparseFile = 0x200 + ReparsePoint = 0x400 + Compressed = 0x800 + Offline = 0x1000 + NotIndexed = 0x2000 + Encrypted = 0x4000 + Directory = 0x10000000 + IndexView = 0x20000000 + unknown = None + + @classmethod + def _missing_(cls, value): + return cls(PermissionFlags.unknown) + + +class MFTEntry(objects.StructType): + """This represents the base MFT Record""" + + def get_signature(self) -> str: + signature = self.Signature.cast('string', max_length = 4, encoding = 'latin-1') + return signature + + +class MFTFileName(objects.StructType): + """This represents an MFT $FILE_NAME Attribute""" + + def get_full_name(self) -> str: + output = self.Name.cast("string", + encoding = "utf16", + max_length = self.NameLength*2, + errors = "replace") + return output + + def get_file_namespace(self) -> str: + pass diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json new file mode 100644 index 000000000..e045ed0fc --- /dev/null +++ b/volatility3/framework/symbols/windows/mft.json @@ -0,0 +1,371 @@ +{ + "metadata": { + "producer": { + "version": "0.0.1", + "name": "kevthehermit-by-hand", + "comment": "Using structures defined in File System Forensic Analysis pg 353+", + "datetime": "2022-01-03T13:37:00" + }, + "format": "6.1.0" + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": true, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "int", + "size": 1, + "signed": false, + "endian": "little" + }, + "wchar": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + } + }, + "symbols": {}, + "enums": {}, + "user_types": { + "MFT_ENTRY": { + "fields": { + "Signature": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "UpdateSequenceOffset": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "NumFixupEntries": { + "offset": 6, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "LSN": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "SequenceValue": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "LinkCount": { + "offset": 18, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "FirstAttrOffset": { + "offset": 20, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "Flags": { + "offset": 22, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "RealSize": { + "offset": 24, + "type":{ + "kind": "base", + "name": "unsigned int" + } + }, + "AlocatedSize": { + "offset": 28, + "type":{ + "kind": "base", + "name": "unsigned int" + } + }, + "BaseReference": { + "offset": 32, + "type":{ + "kind": "base", + "name": "unsigned long long" + } + }, + "NextAttrID": { + "offset": 40, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "RecordNumber": { + "offset": 44, + "type":{ + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 1024 + },"ATTR_HEADER": { + "fields": { + "AttrType": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned int" + } + },"Length": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "NonResidentFlag": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned char" } + }, + "NameLength": { + "offset": 9, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "NameOffset": { + "offset": 10, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "Flags": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "AttributeID": { + "offset": 14, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 16 + },"RESIDENT_HEADER": { + "fields": { + "AttrSize": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned int" + } + },"AttrOffset": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "IndexFlag": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned short" } + } + }, + "kind": "struct", + "size": 8 + }, + "STANDARD_INFORMATION_ENTRY": { + "fields": { + "CreationTime": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ModifiedTime": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "UpdatedTime": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AccessedTime": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "flags": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 1024 + }, + "FILE_NAME_ENTRY": { + "fields": { + "ParentDirectory": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "CreationTime": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ModifiedTime": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "UpdatedTime": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AccessedTime": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AllocatedFileSize": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "RealFileSize": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Flags": { + "offset": 56, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ReparseValue": { + "offset": 60, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "NameLength": { + "offset": 64, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "NameSpace": { + "offset": 65, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "Name": { + "offset": 66, + "type": { + "count": 10, + "kind": "array", + "subtype": { + "kind": "base", + "name": "wchar" + } + } + } + }, + "kind": "struct", + "size": 1024 + } + } +} \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/mft.py b/volatility3/framework/symbols/windows/mft.py new file mode 100644 index 000000000..921d75cd8 --- /dev/null +++ b/volatility3/framework/symbols/windows/mft.py @@ -0,0 +1,15 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mft + + +class MFTIntermedSymbols(intermed.IntermediateSymbolTable): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.set_type_class('FILE_NAME_ENTRY', mft.MFTFileName) + self.set_type_class('MFT_ENTRY', mft.MFTEntry) From 793d08faf487c1d440bae016d8ca1e87766da7cc Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 22:27:32 +0000 Subject: [PATCH 26/63] Add TimeLiner interface to MFTScan plugin --- volatility3/framework/plugins/windows/mftscan.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 484fcdb9a..616c0d738 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -14,11 +14,11 @@ from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols -from volatility3.plugins import yarascan +from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) -class MFTScan(interfaces.plugins.PluginInterface): +class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) @@ -121,6 +121,18 @@ class MFTScan(interfaces.plugins.PluginInterface): except exceptions.PagedInvalidAddressException: pass + def generate_timeline(self): + for row in self._generator(): + if row[-1] != 'N/A': + filename = row[-1] + created = f'File {row[-1]} Created' + updated = f'File {row[-1]} Updated' + modified = f'File {row[-1]} Modified' + accessed = f'File {row[-1]} Accessed' + yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) + yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) + yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) + yield (f'File {filename} accessed', timeliner.TimeLinerType.ACCESSED, row[10]) def run(self): return renderers.TreeGrid([ From a0b66f33f90968b24e1cefab9bd7d7efd0b12638 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jan 2022 21:06:20 +0000 Subject: [PATCH 27/63] Documentation: Update README.md before release --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 75c3c1c23..f7d326c33 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ technical and performance challenges associated with the original code base that became apparent over the previous 10 years. Another benefit of the rewrite is that Volatility 3 could be released under a custom license that was more aligned with the goals of the Volatility community, -the Volatility Software License (VSL). See the [LICENSE](LICENSE.txt) file for more details. +the Volatility Software License (VSL). See the +[LICENSE](https://www.volatilityfoundation.org/license/vsl-v1.0) file for +more details. ## Requirements @@ -102,7 +104,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 12 Jan 2022 21:06:20 +0000 Subject: [PATCH 28/63] Documentation: Update README.md before release --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 75c3c1c23..f7d326c33 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ technical and performance challenges associated with the original code base that became apparent over the previous 10 years. Another benefit of the rewrite is that Volatility 3 could be released under a custom license that was more aligned with the goals of the Volatility community, -the Volatility Software License (VSL). See the [LICENSE](LICENSE.txt) file for more details. +the Volatility Software License (VSL). See the +[LICENSE](https://www.volatilityfoundation.org/license/vsl-v1.0) file for +more details. ## Requirements @@ -102,7 +104,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 12 Jan 2022 21:13:05 +0000 Subject: [PATCH 29/63] Documentation: Update master branch to stable branch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7d326c33..9f9c1bbb7 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ pip3 install -r requirements.txt ## Downloading Volatility -The latest stable version of Volatility will always be the master branch of the GitHub repository. You can get the latest version of the code using the following command: +The latest stable version of Volatility will always be the stable branch of the GitHub repository. You can get the latest version of the code using the following command: ```shell git clone https://github.com/volatilityfoundation/volatility3.git From d469d9c597020fe52649e8a27b1b4738d805ccdc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jan 2022 21:13:05 +0000 Subject: [PATCH 30/63] Documentation: Update master branch to stable branch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7d326c33..9f9c1bbb7 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ pip3 install -r requirements.txt ## Downloading Volatility -The latest stable version of Volatility will always be the master branch of the GitHub repository. You can get the latest version of the code using the following command: +The latest stable version of Volatility will always be the stable branch of the GitHub repository. You can get the latest version of the code using the following command: ```shell git clone https://github.com/volatilityfoundation/volatility3.git From f67f1e242d7f5cf1571864d052a545161849c8e2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jan 2022 22:11:25 +0000 Subject: [PATCH 31/63] Documentation: Ensure the doc reqs are included in s_dist builds --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 504c7d89a..1cec729f6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ prune development include * .* -include doc/make.bat doc/Makefile +include doc/make.bat doc/Makefile doc/requirements.txt recursive-include doc/source * recursive-include volatility3 *.json recursive-exclude doc/source volatility3.*.rst From d91a6f94fbe015a4b837efcbd68407a93a31b682 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 13 Jan 2022 01:01:20 +0000 Subject: [PATCH 32/63] Automagic: Ensure linxu/mac are excluded from windows automagic --- volatility3/framework/automagic/pdbscan.py | 1 + volatility3/framework/automagic/windows.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 93d4337da..8179339c8 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -44,6 +44,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): """ priority = 30 max_pdb_size = 0x400000 + exclusion_list = ['linux', 'mac'] def find_virtual_layers_from_req(self, context: interfaces.context.ContextInterface, config_path: str, requirement: interfaces.configuration.RequirementInterface) -> List[str]: diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index eb63a75e5..71548ca40 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -238,6 +238,8 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): """Class to read swap_layers filenames from single-swap-layers, create the layers and populate the single-layers swap_layers.""" + exclusion_list = ['linux', 'mac'] + def __call__(self, context: interfaces.context.ContextInterface, config_path: str, From 1c6cd0fb528b02e8f35ac65f1173241ff84dfe26 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 15:53:35 +0000 Subject: [PATCH 33/63] Move mftscan enums to ISF file. --- .../framework/plugins/windows/mftscan.py | 33 +++++--- .../symbols/windows/extensions/mft.py | 77 ------------------- .../framework/symbols/windows/mft.json | 70 ++++++++++++++++- 3 files changed, 93 insertions(+), 87 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 616c0d738..991191d5b 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -11,7 +11,6 @@ from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols from volatility3.plugins import timeliner, yarascan @@ -53,6 +52,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Get the Enums + attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") + namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") + mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") + permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): @@ -70,14 +75,20 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.debug(f"Attr Type: {attr_header.AttrType}") # If this is not a valid type then exit the loop - if not AttributeTypes(attr_header.AttrType).value: + if attr_header.AttrType not in attr_types.choices.values(): break # Offset past the headers to the attribute data attr_data_offset = offset+attr_base_offset+24 + + # MFT Flags determine the file type or dir + if mft_record.Flags in mft_flags.choices.values(): + mft_flag = mft_flags.lookup(mft_record.Flags) + else: + mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType == 0x10: + if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) yield 0, ( @@ -85,9 +96,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, - MFTFlags(mft_record.Flags).name, + mft_flag, renderers.NotApplicableValue(), - AttributeTypes(attr_header.AttrType).name, + attr_types.lookup(attr_header.AttrType), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -96,18 +107,22 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType == 0x30: + if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) file_name = attr_data.get_full_name() + if attr_data.Flags in permission_flags.choices.values(): + permissions = permission_flags.lookup(attr_data.Flags) + else: + permissions = hex(attr_data.Flags) yield 1, ( format_hints.Hex(attr_data_offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, - MFTFlags(mft_record.Flags).name, - PermissionFlags(attr_data.Flags).name, - AttributeTypes(attr_header.AttrType).name, + mft_flag, + permissions, + attr_types.lookup(attr_header.AttrType), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 09f6346cc..0713c969a 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -8,80 +8,6 @@ from volatility3.framework import exceptions, objects, renderers from volatility3.framework.objects import utility -class AttributeTypes(enum.Enum): - STANDARD_INFORMATION = 0x10 - ATTRIBUTE_LIST = 0x20 - FILE_NAME = 0x30 - OBJECT_ID = 0x40 - SECURITY_DESCRIPTOR = 0x50 - VOLUME_NAME = 0x60 - VOLUME_INFORMATION = 0x70 - DATA = 0x80 - INDEX_ROOT = 0x90 - INDEX_ALLOCATION = 0xa0 - BITMAP = 0xb0 - REPARSE_POINT = 0xc0 - EA_INFORMATION = 0xd0 - EA = 0xe0 - PROPERTY_SET = 0xf0 - LOGGED_UTILITY_STREAM = 0x100 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(AttributeTypes.Unknown) - -class NameSpace(enum.Enum): - POSIX = 0x0 - Win32 = 0x1 - DOS = 0x2 - Win32DOS = 0x3 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(NameSpace.Unknown) - - -class MFTFlags(enum.Enum): - Removed = 0x00 - File = 0x1 - Directory = 0x2 - DirInUse = 0x3 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(MFTFlags.Unknown) - - -class PermissionFlags(enum.Enum): - ReadOnly = 0x1 - Hidden = 0x2 - System = 0x4 - Archive = 0x20 - ArchiveHidden = 0x22 - ArchiveSystem = 0x24 - ArchiveHiddenSystem = 0x26 - Device = 0x40 - Normal = 0x80 - Temporary = 0x100 - TempArchive = 0x120 - SparseFile = 0x200 - ReparsePoint = 0x400 - Compressed = 0x800 - Offline = 0x1000 - NotIndexed = 0x2000 - Encrypted = 0x4000 - Directory = 0x10000000 - IndexView = 0x20000000 - unknown = None - - @classmethod - def _missing_(cls, value): - return cls(PermissionFlags.unknown) - - class MFTEntry(objects.StructType): """This represents the base MFT Record""" @@ -99,6 +25,3 @@ class MFTFileName(objects.StructType): max_length = self.NameLength*2, errors = "replace") return output - - def get_file_namespace(self) -> str: - pass diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index e045ed0fc..a99b82e9e 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -53,7 +53,75 @@ } }, "symbols": {}, - "enums": {}, + "enums": { + "AttrTypeEnum": { + "base": "unsigned char", + "constants": { + "STANDARD_INFORMATION": 16, + "ATTRIBUTE_LIST": 32, + "FILE_NAME": 48, + "OBJECT_ID": 64, + "SECURITY_DESCRIPTOR": 80, + "VOLUME_NAME": 96, + "VOLUME_INFORMATION": 112, + "DATA": 128, + "INDEX_ROOT": 114, + "INDEX_ALLOCATION": 160, + "BITMAP": 176, + "REPARSE_POINT": 192, + "EA_INFORMATION": 208, + "EA": 224, + "PROPERTY_SET": 240, + "LOGGED_UTILITY_STREAM": 256 + }, + "size": 1 + }, + "NameSpaceEnum": { + "base":"unsigned char", + "constants": { + "POSIX": 0, + "Win32": 1, + "DOS": 2, + "Win32 DOS": 3 + }, + "size": 1 + }, + "MFTFlagsEnum": { + "base":"unsigned char", + "constants": { + "Removed": 0, + "File": 1, + "Directory": 2, + "DirInUse": 3 + }, + "size": 1 + }, + "PermissionFlagEnum": { + "base":"unsigned char", + "constants": { + "ReadOnly": 1, + "Hidden": 2, + "System": 4, + "Archive": 32, + "ArchiveHidden": 34, + "ArchiveSystem": 36, + "ArchiveHiddenSystem": 38, + "Device": 60, + "Normal": 128, + "Temporary": 256, + "TempArchive": 288, + "SparseFile": 512, + "ReparsePoint": 1024, + "Compressed": 2048, + "Offline": 4096, + "NotIndexed": 8192, + "Encrypted": 16384, + "Directory": 268435456, + "IndexView": 536870912 + }, + "size": 1 + } + }, "user_types": { "MFT_ENTRY": { "fields": { From a8f5b0381664b963a565161d88c64fc1b4053102 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 15:57:23 +0000 Subject: [PATCH 34/63] Apply yapf to mftscan plugin --- .../framework/plugins/windows/mftscan.py | 84 +++++++++---------- .../symbols/windows/extensions/mft.py | 10 +-- 2 files changed, 41 insertions(+), 53 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 991191d5b..070061b17 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -17,6 +17,7 @@ from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) + class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" @@ -32,7 +33,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): version = (2, 0, 0)), ] - def _generator(self): layer = self.context.layers[self.config['primary']] @@ -40,12 +40,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) # Read in the Symbol File - symbol_table = MFTIntermedSymbols.create( - self.context, - self.config_path, - "windows", - "mft" - ) + symbol_table = MFTIntermedSymbols.create(self.context, self.config_path, "windows", "mft") # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" @@ -57,20 +52,26 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") - permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") - + permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + + "PermissionFlagEnum") + # Scan the layer for Raw MFT records and parse the fields - for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + for offset, rule_name, name, value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): try: - mft_record = self.context.object(mft_object, offset=offset, layer_name=layer.name) + mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while True: - attr_header = self.context.object(header_object, offset=offset+attr_base_offset, layer_name=layer.name) - attr_resident_header = self.context.object(header_object, offset=offset+attr_base_offset+16, layer_name=layer.name) + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) + attr_resident_header = self.context.object(header_object, + offset = offset + attr_base_offset + 16, + layer_name = layer.name) vollog.debug(f"Attr Type: {attr_header.AttrType}") @@ -79,17 +80,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset+attr_base_offset+24 + attr_data_offset = offset + attr_base_offset + 24 # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): mft_flag = mft_flags.lookup(mft_record.Flags) else: mft_flag = hex(mft_record.Flags) - + # Standard Information Attribute if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): - attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) + attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name) yield 0, ( format_hints.Hex(attr_data_offset), @@ -108,28 +109,21 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # File Name Attribute if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): - attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) + attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name) file_name = attr_data.get_full_name() if attr_data.Flags in permission_flags.choices.values(): permissions = permission_flags.lookup(attr_data.Flags) else: permissions = hex(attr_data.Flags) - yield 1, ( - format_hints.Hex(attr_data_offset), - mft_record.get_signature(), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - permissions, - attr_types.lookup(attr_header.AttrType), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - file_name - ) - + yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(), + mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, + attr_types.lookup(attr_header.AttrType), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), file_name) + # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length @@ -151,16 +145,16 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): def run(self): return renderers.TreeGrid([ - ('Offset', format_hints.Hex), - ('Record Type', str), - ('Record Number', int), - ('Link Count', int), - ('MFT Type', str), - ('Permissions', str), - ('Attribute Type', str), - ('Created', datetime.datetime), - ('Modified', datetime.datetime), - ('Updated', datetime.datetime), - ('Accessed', datetime.datetime), - ('Filename', str), - ],self._generator()) + ('Offset', format_hints.Hex), + ('Record Type', str), + ('Record Number', int), + ('Link Count', int), + ('MFT Type', str), + ('Permissions', str), + ('Attribute Type', str), + ('Created', datetime.datetime), + ('Modified', datetime.datetime), + ('Updated', datetime.datetime), + ('Accessed', datetime.datetime), + ('Filename', str), + ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 0713c969a..ba79b7c8b 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,10 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import enum - -from volatility3.framework import exceptions, objects, renderers -from volatility3.framework.objects import utility +from volatility3.framework import objects class MFTEntry(objects.StructType): @@ -20,8 +17,5 @@ class MFTFileName(objects.StructType): """This represents an MFT $FILE_NAME Attribute""" def get_full_name(self) -> str: - output = self.Name.cast("string", - encoding = "utf16", - max_length = self.NameLength*2, - errors = "replace") + output = self.Name.cast("string", encoding = "utf16", max_length = self.NameLength * 2, errors = "replace") return output From c5987a45d2362562329ca0e977ec87cf76babca9 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 20:54:22 +0000 Subject: [PATCH 35/63] Relative Offset MFT Header --- .../framework/plugins/windows/mftscan.py | 13 +++----- .../framework/symbols/windows/mft.json | 30 ++++++++++++++++++- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 070061b17..3d8d96221 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -44,13 +44,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Get the Enums attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") - namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") + namespace_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") @@ -69,9 +70,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): attr_header = self.context.object(header_object, offset = offset + attr_base_offset, layer_name = layer.name) - attr_resident_header = self.context.object(header_object, - offset = offset + attr_base_offset + 16, - layer_name = layer.name) vollog.debug(f"Attr Type: {attr_header.AttrType}") @@ -80,7 +78,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + 24 + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( + attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -134,10 +133,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for row in self._generator(): if row[-1] != 'N/A': filename = row[-1] - created = f'File {row[-1]} Created' - updated = f'File {row[-1]} Updated' - modified = f'File {row[-1]} Modified' - accessed = f'File {row[-1]} Accessed' yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index a99b82e9e..2470dcbd5 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -223,7 +223,35 @@ }, "kind": "struct", "size": 1024 - },"ATTR_HEADER": { + }, + "ATTRIBUTE": { + "fields":{ + "Attr_Header": { + "offset": 0, + "type": { + "kind": "struct", + "name": "mft!ATTR_HEADER" + } + }, + "Resident_Header": { + "offset": 16, + "type": { + "kind": "struct", + "name": "mft!RESIDENT_HEADER" + } + }, + "Attr_Data": { + "offset": 24, + "type": { + "kind": "struct", + "name": "mft!ATTR_HEADER" + } + } + }, + "kind": "struct", + "size": 96 + }, + "ATTR_HEADER": { "fields": { "AttrType": { "offset": 0, From 1f7acf2779047f9b5bf39e2de2ecf214c0c678a7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 21:02:20 +0000 Subject: [PATCH 36/63] Documentation: Update sphinx requirement to 4.0.0 --- doc/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index d646d22ce..93d6ea70a 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,4 @@ # These packages are required for building the documentation. -sphinx>=1.8.2 +sphinx>=4.0.0 sphinx_autodoc_typehints>=1.4.0 -sphinx-rtd-theme>=0.4.3 \ No newline at end of file +sphinx-rtd-theme>=0.4.3 From c93e20ab36ae6a988d9690d9ac44b165368b76dd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 21:14:52 +0000 Subject: [PATCH 37/63] Plugins: linux.kmsg update documentation and reformat --- volatility3/framework/plugins/linux/kmsg.py | 66 ++++++++++++--------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index a729c1695..8f4540766 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -4,9 +4,9 @@ import logging from abc import ABC, abstractmethod from enum import Enum -from typing import List, Iterator, Tuple, Generator +from typing import Generator, Iterator, List, Tuple -from volatility3.framework import renderers, interfaces, constants, contexts, class_subclasses +from volatility3.framework import class_subclasses, constants, contexts, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -15,39 +15,39 @@ vollog = logging.getLogger(__name__) class DescStateEnum(Enum): - desc_miss = -1 # ID mismatch (pseudo state) - desc_reserved = 0x0 # reserved, in use by writer - desc_committed = 0x1 # committed by writer, could get reopened - desc_finalized = 0x2 # committed, no further modification allowed - desc_reusable = 0x3 # free, not yet used by any writer + desc_miss = -1 # ID mismatch (pseudo state) + desc_reserved = 0x0 # reserved, in use by writer + desc_committed = 0x1 # committed by writer, could get reopened + desc_finalized = 0x2 # committed, no further modification allowed + desc_reusable = 0x3 # free, not yet used by any writer class ABCKmsg(ABC): """Kernel log buffer reader""" LEVELS = ( - "emerg", # system is unusable - "alert", # action must be taken immediately - "crit", # critical conditions - "err", # error conditions - "warn", # warning conditions - "notice", # normal but significant condition - "info", # informational - "debug", # debug-level messages + "emerg", # system is unusable + "alert", # action must be taken immediately + "crit", # critical conditions + "err", # error conditions + "warn", # warning conditions + "notice", # normal but significant condition + "info", # informational + "debug", # debug-level messages ) FACILITIES = ( - "kern", # kernel messages - "user", # random user-level messages - "mail", # mail system - "daemon", # system daemons - "auth", # security/authorization messages - "syslog", # messages generated internally by syslogd - "lpr", # line printer subsystem - "news", # network news subsystem - "uucp", # UUCP subsystem - "cron", # clock daemon + "kern", # kernel messages + "user", # random user-level messages + "mail", # mail system + "daemon", # system daemons + "auth", # security/authorization messages + "syslog", # messages generated internally by syslogd + "lpr", # line printer subsystem + "news", # network news subsystem + "uucp", # UUCP subsystem + "cron", # clock daemon "authpriv", # security/authorization messages (private) - "ftp" # FTP daemon + "ftp" # FTP daemon ) def __init__( @@ -247,12 +247,20 @@ class KmsgFiveTen(ABCKmsg): The data block ring 'text_data_ring' contains the records' text strings. A pointer to the high level structure is kept in the prb pointer which is initialized to a static ringbuffer. + + .. code-block:: c + static struct printk_ringbuffer *prb = &printk_rb_static; + In SMP systems with more than 64 CPUs this ringbuffer size is dynamically allocated according the number of CPUs based on the value of CONFIG_LOG_CPU_MAX_BUF_SHIFT. The prb pointer is updated consequently to this dynamic ringbuffer in setup_log_buf(). + + .. code-block:: c + prb = &printk_rb_dynamic; + Behind scenes, log_buf is still used as external buffer. When the static printk_ringbuffer struct is initialized, _DEFINE_PRINTKRB sets text_data_ring.data pointer to the address in log_buf which points to @@ -262,12 +270,14 @@ class KmsgFiveTen(ABCKmsg): buffer via the prb_init function. In that case, the original external static buffer in __log_buf and printk_rb_static are unused. - ... + + .. code-block:: c + new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN); prb_init(&printk_rb_dynamic, new_log_buf, ...); log_buf = new_log_buf; prb = &printk_rb_dynamic; - ... + See printk.c and printk_ringbuffer.c in kernel/printk/ folder for more details. """ From 8791631db5168b5f85ee72304f7d8906f915586b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 21:40:25 +0000 Subject: [PATCH 38/63] Documentation: More minor fixes --- doc/source/conf.py | 9 +++++++++ doc/source/volshell.rst | 20 ++++++++++---------- volatility3/framework/objects/__init__.py | 4 ++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index eab5c2c94..54c5da5ed 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -84,6 +84,15 @@ def setup(app): for line in submodule_lines: contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins')) + # Clear up the framework.plugins page + with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "rb") as contents: + real_lines = contents.readlines() + + with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "wb") as contents: + for line in real_lines: + if b'volatility3.framework.plugins.' not in line: + contents.write(line) + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 1b6846e6b..3d4cad890 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -22,7 +22,7 @@ be run. When volshell starts, it will show the version of volshell, a brief message indicating how to get more help, the current operating system mode for volshell, and the current layer available for use. -.. code-block:: python +:: Volshell (Volatility 3 Framework) 1.0.1 Readline imported successfully PDB scanning finished @@ -53,7 +53,7 @@ run our examples against. We'll start by creating a process variable, and putting the first result from `ps()` in it. Since the shell is a python environment, we can do the following: -.. code-block:: python +:: (primary) >>> proc = ps()[0] (primary) >>> proc @@ -68,7 +68,7 @@ built-in mechanism for providing more information about a structure, called `dis either a type name (which if not prefixed with symbol table name, will use the kernel symbol table identified by the automagic). -.. code-block:: python +:: (primary) >>> dt('_EPROCESS') nt_symbols1!_EPROCESS (2624 bytes) @@ -80,7 +80,7 @@ automagic). It can also be provided with an object and will interpret the data for each in the process: -.. code-block:: python +:: (primary) >>> dt(proc) nt_symbols1!_EPROCESS (2624 bytes) @@ -92,7 +92,7 @@ It can also be provided with an object and will interpret the data for each in t These values can be accessed directory as attributes -.. code-block:: python +:: (primary) >>> proc.UniqueProcessId 356 @@ -100,7 +100,7 @@ These values can be accessed directory as attributes Pointer structures contain the value they point to, but attributes accessed are forwarded to the object they point to. This means that pointers do not need to be explicitly dereferenced to access underling objects. -.. code-block:: python +:: (primary) >>> proc.Pcb.DirectoryTableBase 4355817472 @@ -112,7 +112,7 @@ It's possible to run any plugin by importing it appropriately and passing it to method. In the following example we'll provide no additional parameters. Volatility will show us which parameters were required: -.. code-block:: python +:: (primary) >>> from volatility3.plugins.windows import pslist (primary) >>> display_plugin_output(pslist.PsList) @@ -124,14 +124,14 @@ was fulfilled. We can see all the options that the plugin can accept by access the `get_requirements()` method of the plugin. This is a classmethod, so can be called on an uninstantiated copy of the plugin. -.. code-block:: python +:: (primary) >>> pslist.PsList.get_requirements() [, , , , ] We can provide arguments via the `dpo` method call: -.. code-block:: python +:: (primary) >>> display_plugin_output(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) @@ -149,7 +149,7 @@ by the `dpo` method is always `context`. Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by using the `generate_treegrid` or `gt` command. -.. code-block:: python +:: (primary) >>> treegrid = gt(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) (primary) >>> treegrid.populate() diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 6ee24407f..b5a7db286 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -6,9 +6,9 @@ import collections import collections.abc import logging import struct -from typing import Any, ClassVar, Dict, List, Iterable, Optional, Tuple, Type, Union as TUnion, overload +from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.objects import templates, utility vollog = logging.getLogger(__name__) From 4aaba89d024a06a00760978b2f8bb7f099087c72 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 21:47:37 +0000 Subject: [PATCH 39/63] Unity timeliner output for mftscan --- .../framework/plugins/windows/mftscan.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 3d8d96221..03f269735 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -78,8 +78,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( - attribute_object).relative_child_offset("Attr_Data") + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type(attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -130,13 +129,18 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pass def generate_timeline(self): + print("MFT Timeline") for row in self._generator(): - if row[-1] != 'N/A': - filename = row[-1] - yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) - yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) - yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) - yield (f'File {filename} accessed', timeliner.TimeLinerType.ACCESSED, row[10]) + _depth, row_data = row + + # Only Output FN Records + if row_data[6] == 'FILE_NAME': + filename = row_data[-1] + description = f"MFT FILE_NAME entry for {filename}" + yield (description, timeliner.TimeLinerType.CREATED, row_data[7]) + yield (description, timeliner.TimeLinerType.MODIFIED, row_data[8]) + yield (description, timeliner.TimeLinerType.CHANGED, row_data[9]) + yield (description, timeliner.TimeLinerType.ACCESSED, row_data[10]) def run(self): return renderers.TreeGrid([ From a9e5589260710a22f47ea2612f5a27ae8ffe87fc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:02:02 +0000 Subject: [PATCH 40/63] Documentation: Add summary table for linux/mac ISF creation --- doc/source/symbol-tables.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 31329b9eb..245dd9c67 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -76,3 +76,21 @@ The banners available for volatility to use can be found using the `isfinfo` plu long time to run depending on the number of JSON files available. This will list all the JSON (ISF) files that volatility3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON file, the banners must match exactly (down to the compilation date). + +.. note:: + + Steps for constructing a new kernel ISF JSON file: + + * Run the `banners` plugin on the image to determine the necessary kernel + * Locate a copy of the debug kernel that matches the identified banner + + * Clone or update the dwarf2json repo: :code:`git clone https://github.com/volatilityfoundation/dwarf2json` + * Run :code:`go build` in the directory if the source has changed + + * Run :code:`dwarf2json linux --elf [path to debug kernel] > [kernel name].json` + + * For Mac change `linux` to `mac` + + * Copy the `.json` file to the symbols directory into `[symbols directory]/linux` + + * For Mac change `linux` to `mac` \ No newline at end of file From 32abab8733d697ed86e3a6208d9c03ce08233117 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 22:38:05 +0000 Subject: [PATCH 41/63] Remove debug print from mftscan --- volatility3/framework/plugins/windows/mftscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 03f269735..7cc57f111 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -129,7 +129,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pass def generate_timeline(self): - print("MFT Timeline") for row in self._generator(): _depth, row_data = row From dab746aff07bc7afe021bd620822640d4f026ce6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:44:24 +0000 Subject: [PATCH 42/63] Plugins: yara python module check improvement Fixes #616 --- volatility3/framework/plugins/yarascan.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index e51669b2c..0ef55ff4b 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -15,8 +15,11 @@ vollog = logging.getLogger(__name__) try: import yara + + if tuple([int(x) for x in yara.__version__.split('.')]) < (3, 8): + raise ImportError except ImportError: - vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") + vollog.info("Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available") raise From d9ba3e6bd9b71f0c9bc1973ada12c7bab104e729 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:50:21 +0000 Subject: [PATCH 43/63] Plugins: Timeliner improve support for body files --- volatility3/framework/plugins/timeliner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 6bf592504..0ae87e274 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -12,7 +12,7 @@ import traceback from typing import Generator, Iterable, List, Optional, Tuple, Type from volatility3 import framework -from volatility3.framework import renderers, automagic, interfaces, plugins, exceptions +from volatility3.framework import automagic, exceptions, interfaces, plugins, renderers from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -145,7 +145,7 @@ class Timeliner(interfaces.plugins.PluginInterface): # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime if self._any_time_present(times): - fp.write("|{} - {}||||||{}|{}|{}|{}\n".format( + fp.write("|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format( plugin_name, self._sanitize_body_format(item), self._text_format(times.get(TimeLinerType.ACCESSED, "")), self._text_format(times.get(TimeLinerType.MODIFIED, "")), @@ -202,7 +202,7 @@ class Timeliner(interfaces.plugins.PluginInterface): if isinstance(plugin, TimeLinerInterface): if not len(filter_list) or any( - [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]): + [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]): plugins_to_run.append(plugin) except exceptions.UnsatisfiedException as excp: # Remove the failed plugin from the list and continue From 742d46786bc0b48781343584204bd51a6c316da9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:56:36 +0000 Subject: [PATCH 44/63] Plugins: Don't forget missing values --- volatility3/framework/plugins/timeliner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 0ae87e274..faa99ff67 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -164,7 +164,7 @@ class Timeliner(interfaces.plugins.PluginInterface): def _text_format(self, value): """Formats a value as text, in case it is an AbsentValue""" if isinstance(value, interfaces.renderers.BaseAbsentValue): - return "" + return "0" if isinstance(value, datetime.datetime): return int(value.timestamp()) return value From f31bc853c4d7eb05041c854f26704dea21656d0d Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 23:30:39 +0000 Subject: [PATCH 45/63] remove MFTIntermedSymbols --- .../framework/plugins/windows/mftscan.py | 43 +++++++++++-------- volatility3/framework/symbols/windows/mft.py | 15 ------- 2 files changed, 25 insertions(+), 33 deletions(-) delete mode 100644 volatility3/framework/symbols/windows/mft.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 7cc57f111..6487ffe97 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,12 @@ import datetime import logging -from typing import Dict - from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mft from volatility3.plugins import timeliner, yarascan @@ -40,7 +39,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) # Read in the Symbol File - symbol_table = MFTIntermedSymbols.create(self.context, self.config_path, "windows", "mft") + symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, + config_path = self.config_path, + sub_path = "windows", + filename = "mft", + class_types = { + 'FILE_NAME_ENTRY': mft.MFTFileName, + 'MFT_ENTRY': mft.MFTEntry + }) # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" @@ -57,28 +63,25 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields - for offset, rule_name, name, value in layer.scan(context = self.context, - scanner = yarascan.YaraScanner(rules = rules)): + for offset, _rule_name, _name, _value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): try: mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) + # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while True: - attr_header = self.context.object(header_object, - offset = offset + attr_base_offset, - layer_name = layer.name) - + while attr_header.AttrType in attr_types.choices.values(): vollog.debug(f"Attr Type: {attr_header.AttrType}") - # If this is not a valid type then exit the loop - if attr_header.AttrType not in attr_types.choices.values(): - break - # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type(attribute_object).relative_child_offset("Attr_Data") + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( + attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -124,9 +127,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length + # Get the next attribute + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) - except exceptions.PagedInvalidAddressException: - pass + except Exception as err: + vollog.debug(f'Error Parsing MFT Record: {err}') def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/symbols/windows/mft.py b/volatility3/framework/symbols/windows/mft.py deleted file mode 100644 index 921d75cd8..000000000 --- a/volatility3/framework/symbols/windows/mft.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows.extensions import mft - - -class MFTIntermedSymbols(intermed.IntermediateSymbolTable): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.set_type_class('FILE_NAME_ENTRY', mft.MFTFileName) - self.set_type_class('MFT_ENTRY', mft.MFTEntry) From a31f846b14ffb4ffe529f1aae93256ba2c8e2d1e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Jan 2022 00:47:40 +0000 Subject: [PATCH 46/63] Documentation: Reorganize and consolidate pages --- doc/source/development.rst | 8 ++++++++ doc/source/index.rst | 6 ++---- 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 doc/source/development.rst diff --git a/doc/source/development.rst b/doc/source/development.rst new file mode 100644 index 000000000..ade068322 --- /dev/null +++ b/doc/source/development.rst @@ -0,0 +1,8 @@ +Writing Plugins +=============== + +.. toctree:: + + simple-plugin + complex-plugin + using-as-a-library diff --git a/doc/source/index.rst b/doc/source/index.rst index 0dc3b5025..3b5a5d2a8 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -12,11 +12,9 @@ Here are some guidelines for using Volatility 3 effectively: .. toctree:: basics - simple-plugin - vol2to3 - complex-plugin - using-as-a-library + development symbol-tables + vol2to3 volshell glossary From e3a7ac566840ee052ffbefe3fb370e4142704706 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 16 Jan 2022 02:20:55 +0000 Subject: [PATCH 47/63] Use lookups on mft enums instead of choices --- .../framework/plugins/windows/mftscan.py | 39 +++++++++---------- .../framework/symbols/windows/mft.json | 22 +++++------ 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 6487ffe97..16f0c9c95 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -55,12 +55,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" - # Get the Enums - attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") - namespace_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") - mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") - permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + - "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan(context = self.context, @@ -74,23 +68,26 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) + # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr_header.AttrType in attr_types.choices.values(): - vollog.debug(f"Attr Type: {attr_header.AttrType}") + + while attr_header.AttrType.is_valid_choice: + vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") # Offset past the headers to the attribute data attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir - if mft_record.Flags in mft_flags.choices.values(): - mft_flag = mft_flags.lookup(mft_record.Flags) - else: + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + mft_flag = mft_record.Flags.lookup() + except ValueError: mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): + if attr_header.AttrType.lookup() == 'STANDARD_INFORMATION': attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name) yield 0, ( @@ -100,7 +97,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_record.LinkCount, mft_flag, renderers.NotApplicableValue(), - attr_types.lookup(attr_header.AttrType), + attr_header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -109,17 +106,19 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): + if attr_header.AttrType.lookup() == 'FILE_NAME': attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name) file_name = attr_data.get_full_name() - if attr_data.Flags in permission_flags.choices.values(): - permissions = permission_flags.lookup(attr_data.Flags) - else: + + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + permissions = attr_data.Flags.lookup() + except ValueError: permissions = hex(attr_data.Flags) yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, - attr_types.lookup(attr_header.AttrType), + attr_header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -132,8 +131,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) - except Exception as err: - vollog.debug(f'Error Parsing MFT Record: {err}') + except exceptions.PagedInvalidAddressException: + pass def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index 2470dcbd5..b71be6444 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -181,8 +181,8 @@ "Flags": { "offset": 22, "type":{ - "kind": "base", - "name": "unsigned short" + "kind": "enum", + "name": "MFTFlagsEnum" } }, "RealSize": { @@ -256,8 +256,8 @@ "AttrType": { "offset": 0, "type": { - "kind": "base", - "name": "unsigned int" + "kind": "enum", + "name": "AttrTypeEnum" } },"Length": { "offset": 4, @@ -289,9 +289,9 @@ "Flags": { "offset": 12, "type": { - "kind": "base", - "name": "unsigned short" - } + "kind": "enum", + "name": "MFTFlagsEnum" + } }, "AttributeID": { "offset": 14, @@ -361,8 +361,8 @@ "flags": { "offset": 32, "type": { - "kind": "base", - "name": "unsigned short" + "kind": "enum", + "name": "PermissionFlagEnum" } } }, @@ -423,8 +423,8 @@ "Flags": { "offset": 56, "type": { - "kind": "base", - "name": "unsigned int" + "kind": "enum", + "name": "PermissionFlagEnum" } }, "ReparseValue": { From 57c9470f66f25726eaa2e6c5f87ac8a29e8dab68 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Jan 2022 22:09:02 +0000 Subject: [PATCH 48/63] Documentation: Fix building from different directories --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 54c5da5ed..6030c6308 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -85,10 +85,10 @@ def setup(app): contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins')) # Clear up the framework.plugins page - with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "rb") as contents: + with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "rb") as contents: real_lines = contents.readlines() - with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "wb") as contents: + with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "wb") as contents: for line in real_lines: if b'volatility3.framework.plugins.' not in line: contents.write(line) From 64acff1b59d7f85a1a187bf45875a8790029407e Mon Sep 17 00:00:00 2001 From: "Nick L. Petroni, Jr" Date: Sun, 16 Jan 2022 15:34:03 -0500 Subject: [PATCH 49/63] use doc/requirements.txt when building with readthedocs --- .readthedocs.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 764bb5a1a..4d21d9b40 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -16,7 +16,4 @@ formats: all python: version: 3.7 install: - - method: pip - path: . - extra_requirements: - - doc + - requirements: doc/requirements.txt From fa2a608cd296e925d9e9847c9ac55cff34bb5f6a Mon Sep 17 00:00:00 2001 From: "Nick L. Petroni, Jr" Date: Sun, 16 Jan 2022 17:32:02 -0500 Subject: [PATCH 50/63] update doc copyright --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 6030c6308..731a73d56 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # # @@ -135,7 +135,7 @@ master_doc = 'index' # General information about the project. project = 'Volatility 3' -copyright = '2012-2019, Volatility Foundation' +copyright = '2012-2022, Volatility Foundation' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 0f23089cb483484ce1eed54ca87d5e78825d02e3 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Mon, 17 Jan 2022 00:28:59 +0000 Subject: [PATCH 51/63] Create Sessions Plugin --- .../framework/plugins/windows/sessions.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 volatility3/framework/plugins/windows/sessions.py diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py new file mode 100644 index 000000000..cd02668ae --- /dev/null +++ b/volatility3/framework/plugins/windows/sessions.py @@ -0,0 +1,96 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import datetime +import logging + +from volatility3.framework import renderers, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + + +class Sessions(interfaces.plugins.PluginInterface): + """lists Processes with Session information extracted from Environmental Variables""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True) + ] + + def _generator(self, procs): + + # Collect all the values as we will want to group them later + sessions = {} + + for proc in procs: + + session_id = proc.get_session_id() + + # Detect RDP, Console or set default value + session_type = renderers.NotAvailableValue() + + # Construct Username from Process Env + user_domain = '' + user_name = '' + + for var, val in proc.environment_variables(): + if var.lower() == 'username': + user_name = val + elif var.lower() == 'userdomain': + user_domain = val + if var.lower() == 'sessionname': + session_type = val + + # Concat Domain and User + full_user = f'{user_domain}/{user_name}' + if full_user == '/': + full_user = renderers.NotAvailableValue() + + # Collect all the values in to a row we can yield after sorting. + row = { + "session_id": session_id, + "process_id": proc.UniqueProcessId, + "process_name": utility.array_to_string(proc.ImageFileName), + "user_name": full_user, + "process_start": proc.get_create_time(), + "session_type": session_type + } + + # Add row to correct session so we can sort it later + if session_id in sessions: + sessions[session_id].append(row) + else: + sessions[session_id] = [row] + + # Group and yield each row + for rows in sessions.values(): + for row in rows: + yield 0, (row.get('session_id'), row.get('session_type'), row.get('process_id'), + row.get('process_name'), row.get('user_name'), row.get('process_start')) + + def run(self): + + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + + return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), + ("User Name", str), ("Create Time", datetime.datetime)], + self._generator( + pslist.PsList.list_processes(self.context, + self.config['primary'], + self.config['nt_symbols'], + filter_func = filter_func))) From be0dd49d314bc6d4b2aa907928f0fb7123cb5f43 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Tue, 18 Jan 2022 19:34:06 +0000 Subject: [PATCH 52/63] Sessions Plugin use ModuleRequirement --- .../framework/plugins/windows/sessions.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py index cd02668ae..66a54c3f0 100644 --- a/volatility3/framework/plugins/windows/sessions.py +++ b/volatility3/framework/plugins/windows/sessions.py @@ -21,10 +21,9 @@ class Sessions(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', element_type = int, @@ -32,12 +31,17 @@ class Sessions(interfaces.plugins.PluginInterface): optional = True) ] - def _generator(self, procs): + def _generator(self): + kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) # Collect all the values as we will want to group them later sessions = {} - for proc in procs: + for proc in pslist.PsList.list_processes(self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func = filter_func): session_id = proc.get_session_id() @@ -56,7 +60,7 @@ class Sessions(interfaces.plugins.PluginInterface): if var.lower() == 'sessionname': session_type = val - # Concat Domain and User + # Concat Domain and User full_user = f'{user_domain}/{user_name}' if full_user == '/': full_user = renderers.NotAvailableValue() @@ -85,12 +89,5 @@ class Sessions(interfaces.plugins.PluginInterface): def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), - ("User Name", str), ("Create Time", datetime.datetime)], - self._generator( - pslist.PsList.list_processes(self.context, - self.config['primary'], - self.config['nt_symbols'], - filter_func = filter_func))) + ("User Name", str), ("Create Time", datetime.datetime)], self._generator()) From ac7cecf231e98c7f104d4c2b8fb569cafb977409 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Tue, 18 Jan 2022 20:42:38 +0000 Subject: [PATCH 53/63] add timeliner output to windows.sessions --- volatility3/framework/plugins/windows/sessions.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py index 66a54c3f0..6745e95ec 100644 --- a/volatility3/framework/plugins/windows/sessions.py +++ b/volatility3/framework/plugins/windows/sessions.py @@ -9,11 +9,12 @@ from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.plugins.windows import pslist +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class Sessions(interfaces.plugins.PluginInterface): +class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """lists Processes with Session information extracted from Environmental Variables""" _required_framework_version = (2, 0, 0) @@ -87,6 +88,15 @@ class Sessions(interfaces.plugins.PluginInterface): yield 0, (row.get('session_id'), row.get('session_type'), row.get('process_id'), row.get('process_name'), row.get('user_name'), row.get('process_start')) + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + # Only add to timeline if we have the username + # Without the user context PSList output is identical + if isinstance(row_data[4], str): + description = f"Process: {row_data[2]} {row_data[3]} started by user {row_data[4]}" + yield (description, timeliner.TimeLinerType.CREATED, row_data[5]) + def run(self): return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), From 0f90f7f6ca5b3b120f3c680aa2993ec4c5246b4c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 18 Jan 2022 20:58:48 +0000 Subject: [PATCH 54/63] Plugins: Remove unused timeliner parameter --- volatility3/framework/plugins/timeliner.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 6bf592504..f7ec340ec 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -74,10 +74,6 @@ class Timeliner(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.StringRequirement(name = 'plugins', - description = "Comma separated list of plugins to run", - optional = True, - default = None), requirements.BooleanRequirement( name = 'record-config', description = "Whether to record the state of all the plugins once complete", From 7c238f93a06b496826271e61a010be9d2c4a9661 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 31 Jan 2022 20:15:39 +0900 Subject: [PATCH 55/63] Update __init__.py Correcting typos for Windows Constants --- volatility3/framework/constants/windows/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/windows/__init__.py b/volatility3/framework/constants/windows/__init__.py index 372897598..a19216605 100644 --- a/volatility3/framework/constants/windows/__init__.py +++ b/volatility3/framework/constants/windows/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""Volatility 3 Linux Constants. +"""Volatility 3 Windows Constants. Windows-specific values that aren't found in debug symbols """ From a67121f1c17fd18acd13c4f7a63463b2b9fc5c8f Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 9 Feb 2022 10:40:37 +0200 Subject: [PATCH 56/63] Added data_offset to pattern matching result, fixes pdb scanning bug --- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 4c3788a56..ce492831e 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -357,4 +357,4 @@ class PdbSignatureScanner(interfaces.layers.ScannerInterface): guid = (16 * '{:02X}').format(g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, ga, gb, gc, gd, ge, gf) if match.start(0) < self.chunk_size: - yield (guid, a, pdb_name, match.start(0)) + yield (guid, a, pdb_name, data_offset + match.start(0)) From ce166b92637edb78abcc7e87139449019c6abaf6 Mon Sep 17 00:00:00 2001 From: trashcan122 Date: Wed, 9 Feb 2022 16:30:40 +0200 Subject: [PATCH 57/63] fix pdb mz parsing --- volatility3/framework/symbols/windows/pdbutil.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index ce492831e..cc7b22e03 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -131,8 +131,14 @@ class PDBUtility(interfaces.configuration.VersionableInterface): # Check it is actually the MZ header if mz_sig != b"MZ": return None - - nt_header_start = ord(layer.read(offset + 0x3C, 1)) + + nt_header_start = struct.unpack(" Date: Wed, 9 Feb 2022 20:07:13 +0000 Subject: [PATCH 58/63] Core: Slight speed-up for all single struct.unpack calls --- volatility3/framework/symbols/windows/extensions/pool.py | 4 ++-- .../framework/symbols/windows/extensions/registry.py | 9 ++++++--- volatility3/framework/symbols/windows/pdbutil.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index f75c6c417..d50d6e47a 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -128,8 +128,8 @@ class POOL_HEADER(objects.StructType): # --------------- if addr - optional_headers_length < 0: continue - padding_length = struct.unpack( - "L"): raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}") - return struct.unpack(">L", data)[0] + res, = struct.unpack(">L", data) + return res if self_type == RegValueTypes.REG_QWORD: if len(data) != struct.calcsize(" Date: Wed, 9 Feb 2022 20:19:17 +0000 Subject: [PATCH 59/63] Core: Bump the development version to 2.0.2 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 82ebd4936..665e62d30 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From cf11de174ab9ec019f09bf2bdc851619cf1ec509 Mon Sep 17 00:00:00 2001 From: pudii Date: Fri, 11 Feb 2022 18:19:53 +0100 Subject: [PATCH 60/63] Implement LDRmodules plugin --- .../framework/plugins/windows/ldrmodules.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 volatility3/framework/plugins/windows/ldrmodules.py diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py new file mode 100644 index 000000000..77678a4c0 --- /dev/null +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -0,0 +1,97 @@ +from volatility3.framework import interfaces, constants +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, vadinfo + +class LdrModules(interfaces.plugins.PluginInterface): + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), + requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), + requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True), + ] + + def _generator(self, procs): + + pe_table_name = intermed.IntermediateSymbolTable.create(self.context, + self.config_path, + "windows", + "pe", + class_types = pe.class_types) + + def filter_function(x: interfaces.objects.ObjectInterface) -> bool: + try: + return not (x.get_private_memory() == 0 and x.ControlArea) + except AttributeError: + return False + + filter_func = filter_function + + for proc in procs: + proc_layer_name = proc.add_process_layer() + + load_order_mod = dict((mod.DllBase, mod) + for mod in proc.load_order_modules()) + init_order_mod = dict((mod.DllBase, mod) + for mod in proc.init_order_modules()) + mem_order_mod = dict((mod.DllBase, mod) + for mod in proc.mem_order_modules()) + + mapped_files = {} + for vad in vadinfo.VadInfo.list_vads(proc, filter_func = filter_func): + dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset = vad.get_start(), + layer_name = proc_layer_name) + try: + if dos_header.e_magic != 0x5A4D: + continue + except exceptions.PagedInvalidAddressException: + continue + + mapped_files[int(vad.get_start())] = str(vad.get_file_name() or '') + + for base in mapped_files.keys(): + # Does the base address exist in the PEB DLL lists? + load_mod = load_order_mod.get(base, None) + init_mod = init_order_mod.get(base, None) + mem_mod = mem_order_mod.get(base, None) + + yield (0, [int(proc.UniqueProcessId), + str(proc.ImageFileName.cast("string", + max_length = proc.ImageFileName.vol.count, + errors = 'replace')), + format_hints.Hex(base), + str(load_mod != None), + str(init_mod != None), + str(mem_mod != None), + str(mapped_files[base])]) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] + + return renderers.TreeGrid([("Pid", int), + ("Process", str), + ("Base", format_hints.Hex), + ("InLoad", str), + ("InInit", str), + ("InMem", str), + ("MappedPath", str)], + self._generator( + pslist.PsList.list_processes(context = self.context, + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, + filter_func = filter_func))) \ No newline at end of file From 9cabf5362b66266a34f861af5e89d0315fe92124 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Feb 2022 00:19:27 +0000 Subject: [PATCH 61/63] Timeliner: Write out directly to the body file Since the body file doesn't need sorting, we can output it immediately, and this also means that partial results can be recorded even in the run is terminted before it compeletes. Goes someway to improving #646 --- volatility3/framework/plugins/timeliner.py | 33 +++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 01f775eb3..8785f62e1 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -106,6 +106,15 @@ class Timeliner(interfaces.plugins.PluginInterface): row from each plugin.""" # Generate the results for each plugin data = [] + + # Open the bodyfile now, so we can start outputting to it immediately + if self.config.get('create-bodyfile', True): + file_data = self.open("volatility.body") + fp = io.TextIOWrapper(file_data, write_through = True) + else: + file_data = None + fp = None + for plugin in runable_plugins: plugin_name = plugin.__class__.__name__ self._progress_callback((runable_plugins.index(plugin) * 100) // len(runable_plugins), @@ -126,17 +135,9 @@ class Timeliner(interfaces.plugins.PluginInterface): times.get(TimeLinerType.ACCESSED, renderers.NotApplicableValue()), times.get(TimeLinerType.CHANGED, renderers.NotApplicableValue()) ])) - except Exception: - vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}") - vollog.log(logging.DEBUG, traceback.format_exc()) - for data_item in sorted(data, key = self._sort_function): - yield data_item - # Write out a body file if necessary - if self.config.get('create-bodyfile', True): - with self.open("volatility.body") as file_data: - with io.TextIOWrapper(file_data, write_through = True) as fp: - for (plugin_name, item) in self.timeline: + # Write each entry because the body file doesn't need to be sorted + if fp: times = self.timeline[(plugin_name, item)] # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime @@ -147,6 +148,18 @@ class Timeliner(interfaces.plugins.PluginInterface): self._text_format(times.get(TimeLinerType.MODIFIED, "")), self._text_format(times.get(TimeLinerType.CHANGED, "")), self._text_format(times.get(TimeLinerType.CREATED, "")))) + except Exception: + vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}") + vollog.log(logging.DEBUG, traceback.format_exc()) + + for data_item in sorted(data, key = self._sort_function): + yield data_item + + # Write out a body file if necessary + if self.config.get('create-bodyfile', True): + if fp: + fp.close() + file_data.close() def _sanitize_body_format(self, value): return value.replace("|", "_") From 12c3f340370f5e4dd1cb34d6803154c23164d234 Mon Sep 17 00:00:00 2001 From: pudii Date: Sun, 13 Feb 2022 18:29:35 +0100 Subject: [PATCH 62/63] Add comments and fix minor code issues --- .../framework/plugins/windows/ldrmodules.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 77678a4c0..42eeacd4d 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -15,7 +15,6 @@ class LdrModules(interfaces.plugins.PluginInterface): def get_requirements(cls): return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', @@ -43,6 +42,7 @@ class LdrModules(interfaces.plugins.PluginInterface): for proc in procs: proc_layer_name = proc.add_process_layer() + # Build dictionaries from different module lists, where the DllBase address is the key and value is the module object load_order_mod = dict((mod.DllBase, mod) for mod in proc.load_order_modules()) init_order_mod = dict((mod.DllBase, mod) @@ -50,18 +50,20 @@ class LdrModules(interfaces.plugins.PluginInterface): mem_order_mod = dict((mod.DllBase, mod) for mod in proc.mem_order_modules()) + # Build dictionary of mapped files, where the VAD start address is the key and value is the file name of the mapped file mapped_files = {} for vad in vadinfo.VadInfo.list_vads(proc, filter_func = filter_func): dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", offset = vad.get_start(), layer_name = proc_layer_name) try: + # Filter out VADs that do not start with a MZ header if dos_header.e_magic != 0x5A4D: continue except exceptions.PagedInvalidAddressException: continue - mapped_files[int(vad.get_start())] = str(vad.get_file_name() or '') + mapped_files[vad.get_start()] = vad.get_file_name() for base in mapped_files.keys(): # Does the base address exist in the PEB DLL lists? @@ -74,10 +76,10 @@ class LdrModules(interfaces.plugins.PluginInterface): max_length = proc.ImageFileName.vol.count, errors = 'replace')), format_hints.Hex(base), - str(load_mod != None), - str(init_mod != None), - str(mem_mod != None), - str(mapped_files[base])]) + load_mod != None, + init_mod != None, + mem_mod != None, + mapped_files[base]]) def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) @@ -86,12 +88,12 @@ class LdrModules(interfaces.plugins.PluginInterface): return renderers.TreeGrid([("Pid", int), ("Process", str), ("Base", format_hints.Hex), - ("InLoad", str), - ("InInit", str), - ("InMem", str), + ("InLoad", bool), + ("InInit", bool), + ("InMem", bool), ("MappedPath", str)], self._generator( pslist.PsList.list_processes(context = self.context, layer_name = kernel.layer_name, symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) \ No newline at end of file + filter_func = filter_func))) From 22a7328e4cba83bbb6965d9705b22a1b5755d626 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Feb 2022 20:45:42 +0000 Subject: [PATCH 63/63] Windows: Prevent infinite loop in mftscan --- volatility3/framework/plugins/windows/mftscan.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 16f0c9c95..654e26db7 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,11 @@ import datetime import logging -from volatility3.framework import constants, renderers, interfaces +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import mft - from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) @@ -55,7 +53,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" - # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): @@ -68,10 +65,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) - # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - + while attr_header.AttrType.is_valid_choice: vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") @@ -124,6 +120,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): conversion.wintime_to_datetime(attr_data.UpdatedTime), conversion.wintime_to_datetime(attr_data.AccessedTime), file_name) + # If there's no advancement the loop will never end, so break it now + if attr_header.Length == 0: + break + # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length # Get the next attribute