From 441fff0d10b2053a1ee52e21dfb15ab0eb06b1a2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 13 Oct 2021 17:00:47 +0100 Subject: [PATCH 001/158] Windows: Reduce DTB false positive rate --- volatility3/framework/automagic/windows.py | 49 ++++++++++++++-------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index eb63a75e5..ce09ca6a4 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -202,31 +202,44 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): for description, tests, sections in cls.test_sets: vollog.debug(description) # There is a very high chance that the DTB will live in these very narrow segments, assuming we couldn't find them previously - hits = context.layers[layer_name].scan(context, - PageMapScanner(tests = tests), - sections = sections, - progress_callback = progress_callback) + hits = base_layer.scan(context, + PageMapScanner(tests = tests), + sections = sections, + progress_callback = progress_callback) # Flatten the generator def sort_by_tests(x): + """Key used to sort by tests""" return tests.index(x[0]), x[1] + def get_max_pointer(page_table, test, ptr_size: int): + """Determines a pointer from a page_table""" + max_ptr = 0 + for index in range(0, len(page_table), ptr_size): + max_ptr = max(max_ptr, + struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] & test.mask) + return max_ptr + hits = sorted(list(hits), key = sort_by_tests) - if hits: - # TODO: Decide which to use if there are multiple options - test, page_map_offset = hits[0] - vollog.debug(f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}") - new_layer_name = context.layers.free_layer_name("IntelLayer") - config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name) - context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name - context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] = page_map_offset - # TODO: Need to determine the layer type (chances are high it's x64, hence this default) - layer = test.layer_type(context, - config_path = config_path, - name = new_layer_name, - metadata = {'os': 'Windows'}) - break + for test, page_map_offset in hits: + # Turn the page tables into integers and find the largest one + page_table = base_layer.read(page_map_offset, 0x1000) + ptr_size = struct.calcsize(test.ptr_struct) + max_pointer = get_max_pointer(page_table, test, ptr_size) + + if max_pointer <= base_layer.maximum_address: + vollog.debug(f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}") + new_layer_name = context.layers.free_layer_name("IntelLayer") + config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name) + context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name + context.config[ + interfaces.configuration.path_join(config_path, "page_map_offset")] = page_map_offset + layer = test.layer_type(context, + config_path = config_path, + name = new_layer_name, + metadata = {'os': 'Windows'}) + break if layer is not None and config_path: vollog.debug("DTB was found at: 0x{:0x}".format(context.config[interfaces.configuration.path_join( From 865cb92527915490cebe35fd00763eb4693ac13d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 17 Oct 2021 00:25:42 +0100 Subject: [PATCH 002/158] Windows: Check only valid page table entries --- volatility3/framework/automagic/windows.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index ce09ca6a4..f90f20b52 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -216,8 +216,9 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): """Determines a pointer from a page_table""" max_ptr = 0 for index in range(0, len(page_table), ptr_size): - max_ptr = max(max_ptr, - struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] & test.mask) + pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] + if pointer & 0x1: + max_ptr = max(max_ptr, pointer & test.mask) return max_ptr hits = sorted(list(hits), key = sort_by_tests) From fb6610ff48d165527ad99b6e127eb09b05194452 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 17 Oct 2021 00:36:16 +0100 Subject: [PATCH 003/158] Windows: Limit pointers to layer maximum address --- volatility3/framework/automagic/windows.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index f90f20b52..6968a1ac2 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -218,7 +218,7 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): for index in range(0, len(page_table), ptr_size): pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] if pointer & 0x1: - max_ptr = max(max_ptr, pointer & test.mask) + max_ptr = max(max_ptr, pointer & test.layer_type.maximum_address) return max_ptr hits = sorted(list(hits), key = sort_by_tests) @@ -241,6 +241,9 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): name = new_layer_name, metadata = {'os': 'Windows'}) break + else: + vollog.debug( + f"Max pointer for hit with test {test.__class__.__name__} not met: {hex(max_pointer)} > {hex(base_layer.maximum_address)}") if layer is not None and config_path: vollog.debug("DTB was found at: 0x{:0x}".format(context.config[interfaces.configuration.path_join( From 49fe653d9849c48d7179774705c768ec1011f680 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 17 Oct 2021 00:53:20 +0100 Subject: [PATCH 004/158] Windows: Max pointers at maximum layer address --- volatility3/framework/automagic/windows.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 6968a1ac2..308ead157 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -217,8 +217,9 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): max_ptr = 0 for index in range(0, len(page_table), ptr_size): pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] - if pointer & 0x1: - max_ptr = max(max_ptr, pointer & test.layer_type.maximum_address) + # Make sure the pointer is valid, ignore large pages which would require more calculation + if pointer & 0x1 and not pointer & 0x80: + max_ptr = max(max_ptr, pointer % test.layer_type.maximum_address) return max_ptr hits = sorted(list(hits), key = sort_by_tests) From 9b4324adaae666e3649c27e8efced45853fc26eb Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 17 Oct 2021 01:53:41 +0100 Subject: [PATCH 005/158] Windows: Stop looking for DTBs when a good one is found --- volatility3/framework/automagic/windows.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 308ead157..beda8d97b 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -219,7 +219,7 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] # Make sure the pointer is valid, ignore large pages which would require more calculation if pointer & 0x1 and not pointer & 0x80: - max_ptr = max(max_ptr, pointer % test.layer_type.maximum_address) + max_ptr = max(max_ptr, (pointer ^ (pointer & 0xfff)) % test.layer_type.maximum_address) return max_ptr hits = sorted(list(hits), key = sort_by_tests) @@ -245,6 +245,8 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): else: vollog.debug( f"Max pointer for hit with test {test.__class__.__name__} not met: {hex(max_pointer)} > {hex(base_layer.maximum_address)}") + if layer is not None and config_path: + break if layer is not None and config_path: vollog.debug("DTB was found at: 0x{:0x}".format(context.config[interfaces.configuration.path_join( From 832bdc2ab19d795b7c4abe519ef4eea9b909e581 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 8 Dec 2021 23:34:23 +0000 Subject: [PATCH 006/158] Linux: Fix long standing typo (thanks to @gcmoreira) --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fbc02399f..0edd60608 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -108,7 +108,7 @@ class module(generic.GenericIntelProcess): "linux", "elf", native_types = None, - class_types = extensions.elf.class_types) + class_types = elf.class_types) syms = self._context.object( self.get_symbol_table().name + constants.BANG + "array", From 997b8572465ef811677c3e2775655e731cdb27fd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 19 Dec 2021 22:16:53 +0000 Subject: [PATCH 007/158] CLI: Add in None renderer to avoid text output --- volatility3/cli/text_renderer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 35b2468e8..e62f705ce 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -182,6 +182,17 @@ class QuickTextRenderer(CLIRenderer): outfd.write("\n") +class NoneRenderer(CLIRenderer): + """Outputs no results""" + name = "none" + + def get_render_options(self): + pass + + def render(self, grid: interfaces.renderers.TreeGrid) -> None: + if not grid.populated: + grid.populate(lambda x, y: True, True) + class CSVRenderer(CLIRenderer): _type_renderers = { format_hints.Bin: quoted_optional(lambda x: f"0b{x:b}"), From 3bc90b82009c8cb4774c635714a5c5d7009d2a5e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 20 Dec 2021 20:38:03 +0000 Subject: [PATCH 008/158] Plugins: Add more information to layerwriter --list --- volatility3/framework/plugins/layerwriter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index b2a02116e..0068ec224 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -110,9 +110,9 @@ class LayerWriter(plugins.PluginInterface): def _generate_layers(self): """List layer names from this run""" for name in self.context.layers: - yield (0, (name, )) + yield (0, (name, self.context.layers[name].__class__.__name__)) def run(self): if self.config['list']: - return renderers.TreeGrid([("Layer name", str)], self._generate_layers()) + return renderers.TreeGrid([("Layer name", str), ('Layer type', str)], self._generate_layers()) return renderers.TreeGrid([("Status", str)], self._generator()) From a96f5de6a0d635e381717b84f5bf643d7bbc6a3b Mon Sep 17 00:00:00 2001 From: cstation Date: Tue, 28 Dec 2021 18:45:08 +0100 Subject: [PATCH 009/158] QEMU: Add 'dirty-bitmap' and 'pbs-state', handle page_size more consistently --- volatility3/framework/layers/qemu.py | 31 +++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 1c5319dfd..df383a04a 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -54,7 +54,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, name: str) -> Any: """Reads the JSON configuration from the end of the file""" - chunk_size = 0x4096 + chunk_size = 4096 data = b'' for i in range(base_layer.maximum_address, base_layer.minimum_address, -chunk_size): if i != base_layer.maximum_address: @@ -65,6 +65,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if start_of_json >= 0: data = data[start_of_json:] return json.loads(data) + # No JSON configuration found at the end of the file, return empty dict return dict() raise exceptions.LayerException(name, "Invalid JSON configuration at the end of the file") @@ -79,9 +80,11 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): addr = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', offset = index, layer_name = self._base_layer) + # Flags are stored in the n least significant bits, where n equals the bit-length of pagesize flags = addr & (page_size - 1) - page_size_bits = int(math.log(page_size, 2)) - addr = (addr >> page_size_bits) << page_size_bits + # addr equals the highest multiple of pagesize <= offset + # (We assume that page_size is a power of 2) + addr = addr ^ (addr & (page_size - 1)) index += 8 if flags & self.SEGMENT_FLAG_MEM_SIZE: @@ -126,6 +129,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._configuration = self._read_configuration(base_layer, self.name) section_byte = -1 index = 8 + section_info = dict() current_section_id = -1 version_id = -1 name = None @@ -162,6 +166,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): offset = index, layer_name = self._base_layer) index += 4 + # Store section info for handling QEVM_SECTION_PARTs later on + section_info[current_section_id] = {'name': name, 'version_id': version_id} # Read additional data index = self.extract_data(index, name, version_id) elif section_byte == self.QEVM_SECTION_PART or section_byte == self.QEVM_SECTION_END: @@ -171,7 +177,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): current_section_id = section_id index += 4 # Read additional data - index = self.extract_data(index, name, version_id) + index = self.extract_data(index, section_info[current_section_id]['name'], + section_info[current_section_id]['version_id']) elif section_byte == self.QEVM_SECTION_FOOTER: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', offset = index, @@ -189,7 +196,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if name == 'ram': if version_id != 4: raise exceptions.LayerException(f"QEMU unknown RAM version_id {version_id}") - new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', None) or 4096) + new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', 4096)) self._segments += new_segments elif name == 'spapr/htab': if version_id != 1: @@ -208,6 +215,13 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): layer_name = self._base_layer) htab_index, htab_n_valid, htab_n_invalid = htab index += 8 + (htab_n_valid * self.HASH_PTE_SIZE_64) + elif name == 'dirty-bitmap': + index += 1 + elif name == 'pbs-state': + section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', + offset = index, + layer_name = self._base_layer) + index += 8 + section_len return index def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes: @@ -217,9 +231,12 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): of the starting data. It is the responsibility of the layer to turn the provided data chunk into the right portion of data necessary. """ - start_offset = offset ^ (offset & 0xfff) + page_size = self._configuration.get('page_size', 4096) + # start_offset equals the highest multiple of pagesize <= offset + # (We assume that page_size is a power of 2) + start_offset = offset ^ (offset & (page_size - 1)) if start_offset in self._compressed: - data = (data * 0x1000) + data = (data * page_size) result = data[offset - start_offset:output_length + offset - start_offset] return result From 7b0a90afa69f268db1a8b835f37d630f050c2fc5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Dec 2021 22:15:08 +0000 Subject: [PATCH 010/158] Automagic: Warn when multiple symbol files match a banner --- volatility3/framework/automagic/linux.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f9fa22c07..154f01749 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -45,6 +45,12 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): symbol_files = linux_banners.get(banner, None) if symbol_files: + if len(symbol_files) > 1: + using = "*" + vollog.warning(f"Multiple symbol files identified (using {using}):") + for symbol_file in symbol_files: + vollog.warning(f" {using} {symbol_file}") + using = " " isf_path = symbol_files[0] table_name = context.symbol_space.free_table_name('LintelStacker') table = linux.LinuxKernelIntermedSymbols(context, From c8dd8d08bda450d29cec91b797eb4094fbfed0e0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Dec 2021 22:26:28 +0000 Subject: [PATCH 011/158] Volshell: Synchronize with the standard CLI cache clearing --- volatility3/cli/volshell/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 94f735ba0..409123457 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -138,8 +138,7 @@ class VolShell(cli.CommandLine): console.setLevel(10 - (partial_args.verbosity - 2)) if partial_args.clear_cache: - for cache_filename in glob.glob(os.path.join(constants.CACHE_PATH, '*.cache')): - os.unlink(cache_filename) + framework.clear_cache() # Do the initialization ctx = contexts.Context() # Construct a blank context From f38fc22002714eb87734b1049cb3858e222d2fc0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Dec 2021 23:51:28 +0000 Subject: [PATCH 012/158] CLI: Support multi-line fields and tabstops in pretty renderer --- volatility3/cli/text_renderer.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index e62f705ce..8663bb995 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -283,9 +283,10 @@ class PrettyTextRenderer(CLIRenderer): column = grid.columns[column_index] renderer = self._type_renderers.get(column.type, self._type_renderers['default']) data = renderer(node.values[column_index]) + field_width = max([len(self.tab_stop(x)) for x in f"{data}".split("\n")]) max_column_widths[column.name] = max(max_column_widths.get(column.name, len(column.name)), - len(f"{data}")) - line[column] = data + field_width) + line[column] = data.split("\n") accumulator.append((node.path_depth, line)) return accumulator @@ -307,7 +308,25 @@ class PrettyTextRenderer(CLIRenderer): column_titles = [""] + [column.name for column in grid.columns] outfd.write(format_string.format(*column_titles)) for (depth, line) in final_output: - outfd.write(format_string.format("*" * depth, *[line[column] for column in grid.columns])) + nums_line = max([len(line[column]) for column in line]) + for column in line: + line[column] = line[column] + ([""] * (nums_line - len(line[column]))) + for index in range(nums_line): + if index == 0: + outfd.write(format_string.format("*" * depth, *[self.tab_stop(line[column][index]) for column in grid.columns])) + else: + outfd.write(format_string.format(" " * depth, *[self.tab_stop(line[column][index]) for column in grid.columns])) + + def tab_stop(self, line: str) -> str: + tab_width = 8 + while line.find('\t') >= 0: + i = line.find('\t') + if (tab_width > 0): + pad = " " * (tab_width - (i % tab_width)) + else: + pad = "" + line = line.replace("\t", pad, 1) + return line class JsonRenderer(CLIRenderer): From c13b49262d5ee5049d34d2781d319fbc9ac578d8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 30 Dec 2021 01:47:45 +0000 Subject: [PATCH 013/158] Renderers: Make appending rows massively more efficient Previously we were generating the list of children (for most likely the root node) for every single append statement, during which we were recalculating the length of the list, twice. In plugins that output a lot of rows this would add an enourmous overhead (that likely grew as the length of the output grew). Without this overhead, the time taken for the TreeGrid._append method went from 1230.0s to 2.4s. --- volatility3/framework/renderers/__init__.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index de1b14c91..5773861d9 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -272,20 +272,26 @@ class TreeGrid(interfaces.renderers.TreeGrid): def _append(self, parent: Optional[interfaces.renderers.TreeNode], values: Any) -> TreeNode: """Adds a new node at the top level if parent is None, or under the parent node otherwise, after all other children.""" - children = self.children(parent) - return self._insert(parent, len(children), values) + return self._insert(parent, None, values) - def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: int, values: Any) -> TreeNode: + def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: Optional[int], values: Any) -> TreeNode: """Inserts an element into the tree at a specific position.""" parent_path = "" children = self._find_children(parent) if parent is not None: parent_path = parent.path + self.path_sep - newpath = parent_path + str(position) + if position is None: + newpath = parent_path + str(len(children)) + else: + newpath = parent_path + str(position) + for node, _ in children[position:]: + self.visit(node, lambda child, _: child.path_changed(newpath, True), None) + tree_item = TreeNode(newpath, self, parent, values) - for node, _ in children[position:]: - self.visit(node, lambda child, _: child.path_changed(newpath, True), None) - children.insert(position, (tree_item, [])) + if position is None: + children.append((tree_item, [])) + else: + children.insert(position, (tree_item, [])) return tree_item def is_ancestor(self, node, descendant): From 571ab8f6590ec5306af8c6e391e3a481c2580560 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 31 Dec 2021 00:51:21 +0000 Subject: [PATCH 014/158] Volshell: Update the linux pslist requirement --- volatility3/cli/volshell/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 850e3111c..4338ae06f 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -17,7 +17,7 @@ class Volshell(generic.Volshell): def get_requirements(cls): return (super().get_requirements() + [ requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) From ac7127d05a5dc9a81870492d851b0c74c8eb8a43 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 31 Dec 2021 00:55:23 +0000 Subject: [PATCH 015/158] Volshell: Sync errors with the CLI concerning unsatisfied requirements Fixes #607 --- volatility3/cli/volshell/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 409123457..7b8a759a6 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -238,9 +238,14 @@ class VolShell(cli.CommandLine): vollog.debug("Writing out configuration data to config.json") with open("config.json", "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + except exceptions.UnsatisfiedException as excp: + self.process_unsatisfied_exceptions(excp) + parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") + try: # Construct and run the plugin - constructed.run() + if constructed: + constructed.run() except exceptions.VolatilityException as excp: self.process_exceptions(excp) parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") From dd13b427e011a84f94ebc571608a5375f5f9ccea Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 31 Dec 2021 00:51:21 +0000 Subject: [PATCH 016/158] Volshell: Update the linux pslist requirement --- volatility3/cli/volshell/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 850e3111c..4338ae06f 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -17,7 +17,7 @@ class Volshell(generic.Volshell): def get_requirements(cls): return (super().get_requirements() + [ requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) From e91e65d64b82968d466c879b1ac29c7f1d701e13 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 30 Dec 2021 01:47:45 +0000 Subject: [PATCH 017/158] Renderers: Make appending rows massively more efficient Previously we were generating the list of children (for most likely the root node) for every single append statement, during which we were recalculating the length of the list, twice. In plugins that output a lot of rows this would add an enourmous overhead (that likely grew as the length of the output grew). Without this overhead, the time taken for the TreeGrid._append method went from 1230.0s to 2.4s. --- volatility3/framework/renderers/__init__.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index de1b14c91..5773861d9 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -272,20 +272,26 @@ class TreeGrid(interfaces.renderers.TreeGrid): def _append(self, parent: Optional[interfaces.renderers.TreeNode], values: Any) -> TreeNode: """Adds a new node at the top level if parent is None, or under the parent node otherwise, after all other children.""" - children = self.children(parent) - return self._insert(parent, len(children), values) + return self._insert(parent, None, values) - def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: int, values: Any) -> TreeNode: + def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: Optional[int], values: Any) -> TreeNode: """Inserts an element into the tree at a specific position.""" parent_path = "" children = self._find_children(parent) if parent is not None: parent_path = parent.path + self.path_sep - newpath = parent_path + str(position) + if position is None: + newpath = parent_path + str(len(children)) + else: + newpath = parent_path + str(position) + for node, _ in children[position:]: + self.visit(node, lambda child, _: child.path_changed(newpath, True), None) + tree_item = TreeNode(newpath, self, parent, values) - for node, _ in children[position:]: - self.visit(node, lambda child, _: child.path_changed(newpath, True), None) - children.insert(position, (tree_item, [])) + if position is None: + children.append((tree_item, [])) + else: + children.insert(position, (tree_item, [])) return tree_item def is_ancestor(self, node, descendant): From 7aed488721153c7e786de7775a83f1056f3d8d16 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jan 2022 01:21:37 +0000 Subject: [PATCH 018/158] Automagic: Allow automagic to exclude unsupported OSes --- doc/source/using-as-a-library.rst | 3 ++- doc/source/vol2to3.rst | 4 ++++ volatility3/framework/automagic/__init__.py | 21 ++++++------------- volatility3/framework/automagic/linux.py | 2 ++ volatility3/framework/automagic/mac.py | 2 ++ volatility3/framework/constants/__init__.py | 1 + volatility3/framework/interfaces/automagic.py | 3 +++ 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index 95d2b1080..c63adcfc3 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -131,7 +131,8 @@ A suitable list of automagics for a particular plugin (based on operating system automagics = automagic.choose_automagic(available_automagics, plugin) This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just -the automagics which apply to the operating system. +the automagics which apply to the operating system. Each automagic can exclude itself from being used for specific +operating systems, so that an automagic designed for linux is not used for windows or mac plugins. These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that the plugin will be constructed on ('plugins' by default) and a progress_callback. This is a callable which takes diff --git a/doc/source/vol2to3.rst b/doc/source/vol2to3.rst index eb33b6618..e768df0c2 100644 --- a/doc/source/vol2to3.rst +++ b/doc/source/vol2to3.rst @@ -62,6 +62,10 @@ automagic processes are clearly defined and can be enabled or disabled as necess included a stacker automagic to emulate the most common feature of Volatility 2, automatically stacking address spaces (now translation layers) on top of each other. +By default the automagic chosen to be run are determined based on the plugin requested, so that linux plugins get linux +specific automagic and windows plugins get windows specific automagic. This should reduce unnecessarily searching for +linux kernels in a windows image, for example. At the moment this is not user configurableS. + Searching and Scanning ---------------------- Scanning is very similar to scanning in Volatility 2, a scanner object (such as a diff --git a/volatility3/framework/automagic/__init__.py b/volatility3/framework/automagic/__init__.py index e4d422c99..7567f206d 100644 --- a/volatility3/framework/automagic/__init__.py +++ b/volatility3/framework/automagic/__init__.py @@ -21,14 +21,6 @@ from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) -windows_automagic = [ - 'ConstructionMagic', 'LayerStacker', 'KernelPDBScanner', 'WinSwapLayers', 'KernelModule' -] - -linux_automagic = ['ConstructionMagic', 'LayerStacker', 'LinuxBannerCache', 'LinuxSymbolFinder', 'KernelModule'] - -mac_automagic = ['ConstructionMagic', 'LayerStacker', 'MacBannerCache', 'MacSymbolFinder', 'KernelModule'] - def available(context: interfaces.context.ContextInterface) -> List[interfaces.automagic.AutomagicInterface]: """Returns an ordered list of all subclasses of @@ -58,10 +50,7 @@ def choose_automagic( plugin_category = "None" plugin_categories = plugin.__module__.split('.') lowest_index = len(plugin_categories) - - automagic_categories = {'windows': windows_automagic, 'linux': linux_automagic, 'mac': mac_automagic} - - for os in automagic_categories: + for os in constants.OS_CATEGORIES: try: if plugin_categories.index(os) < lowest_index: lowest_index = plugin_categories.index(os) @@ -70,14 +59,16 @@ def choose_automagic( # The value wasn't found, try the next one pass - if plugin_category not in automagic_categories: + if plugin_category not in constants.OS_CATEGORIES: vollog.info("No plugin category detected") return automagics - vollog.info(f"Detected a {plugin_category} category plugin") + output = [] for amagic in automagics: - if amagic.__class__.__name__ in automagic_categories[plugin_category]: + if plugin_category not in amagic.exclusion_list: + # Only include uncategorized automagic, or platform specific automagic + # (This allows user defined/uncategorized automagic to be included) output += [amagic] return output diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f9fa22c07..a6577e322 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -147,6 +147,7 @@ class LinuxBannerCache(symbol_cache.SymbolBannerCache): os = "linux" symbol_name = "linux_banner" banner_path = constants.LINUX_BANNERS_PATH + exclusion_list = ['mac', 'windows'] class LinuxSymbolFinder(symbol_finder.SymbolFinder): @@ -156,3 +157,4 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder): banner_cache = LinuxBannerCache symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] + exclusion_list = ['mac', 'windows'] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index fb725a234..c37aef463 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -202,6 +202,7 @@ class MacBannerCache(symbol_cache.SymbolBannerCache): os = "mac" symbol_name = "version" banner_path = constants.MAC_BANNERS_PATH + exclusion_list = ['windows', 'linux'] class MacSymbolFinder(symbol_finder.SymbolFinder): @@ -211,3 +212,4 @@ class MacSymbolFinder(symbol_finder.SymbolFinder): banner_cache = MacBannerCache find_aslr = MacIntelStacker.find_aslr symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols" + exclusion_list = ['windows', 'linux'] diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 23598837b..82ebd4936 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -78,6 +78,7 @@ BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" ProgressCallback = Optional[Callable[[float, str], None]] """Type information for ProgressCallback objects""" +OS_CATEGORIES = ['windows', 'mac', 'linux'] class Parallelism(enum.IntEnum): """An enumeration listing the different types of parallelism applied to diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index c310f5b4a..c96c9bdbe 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -40,6 +40,9 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla priority = 10 """An ordering to indicate how soon this automagic should be run""" + exclusion_list = [] + """A list of plugin categories (typically operating systems) which the plugin will not operate on""" + def __init__(self, context: interfaces.context.ContextInterface, config_path: str, *args, **kwargs) -> None: super().__init__(context, config_path) for requirement in self.get_requirements(): From 2c949dfa50e2b530a3df9cc480933e583a1ca4e7 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Thu, 6 Jan 2022 22:53:24 +0000 Subject: [PATCH 019/158] Create MFTScanner plugin --- .../framework/plugins/windows/mftscan.py | 305 ++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 volatility3/framework/plugins/windows/mftscan.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py new file mode 100644 index 000000000..1d4ca954e --- /dev/null +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -0,0 +1,305 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from struct import unpack +from typing import Iterable + +from volatility3.framework import constants, renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework.renderers import conversion, format_hints +from volatility3.framework.symbols import intermed +from volatility3.plugins import yarascan + +vollog = logging.getLogger(__name__) + +try: + import yara +except ImportError: + vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") + raise + +signatures = { + 'mft_objects': """rule mft_headers + { + strings: + $header1 = "FILE0" + $header2 = "FILE*" + $header3 = "BAAD" + condition: + any of them + }""" +} + +# https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee9338b881c0fa25937/volatility/plugins/mftparser.py#L60 +ATTRIBUTE_TYPE_ID = { + 0x10:"STANDARD_INFORMATION", + 0x20:"ATTRIBUTE_LIST", + 0x30:"FILE_NAME", + 0x40:"OBJECT_ID", + 0x50:"SECURITY_DESCRIPTOR", + 0x60:"VOLUME_NAME", + 0x70:"VOLUME_INFORMATION", + 0x80:"DATA", + 0x90:"INDEX_ROOT", + 0xa0:"INDEX_ALLOCATION", + 0xb0:"BITMAP", + 0xc0:"REPARSE_POINT", + 0xd0:"EA_INFORMATION", #Extended Attribute + 0xe0:"EA", + 0xf0:"PROPERTY_SET", + 0x100:"LOGGED_UTILITY_STREAM", +} + +VERBOSE_STANDARD_INFO_FLAGS = { + 0x1:"Read Only", + 0x2:"Hidden", + 0x4:"System", + 0x20:"Archive", + 0x40:"Device", + 0x80:"Normal", + 0x100:"Temporary", + 0x200:"Sparse File", + 0x400:"Reparse Point", + 0x800:"Compressed", + 0x1000:"Offline", + 0x2000:"Content not indexed", + 0x4000:"Encrypted", + 0x10000000:"Directory", + 0x20000000:"Index view", +} + +FILE_NAME_NAMESPACE = { + 0x0:"POSIX", # Case sensitive, allows all Unicode chars except '/' and NULL + 0x1:"Win32", # Case insensitive, allows most Unicide except specials ('/', '\', ';', '>', '<', '?') + 0x2:"DOS", # Case insensitive, upper case, no special chars, name is 8 or fewer chars in name and 3 or less extension + 0x3:"Win32 & DOS", # Used when original name fits in DOS namespace and 2 names are not needed +} + +MFT_FLAGS = { + 0x0: "Removed", + 0x1: "File", # "In Use", + 0x2: "Directory", # if flag & 0x0002 == 0 this is a regular file + 0x3: "Directory" +} + +INDEX_ENTRY_FLAGS = { + 0x1:"Child Node Exists", + 0x2:"Last entry in list", +} + + +class MFTScan(interfaces.plugins.PluginInterface): + """Scans for MFT FILE objects present in a particular windows memory image.""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, + version = (2, 0, 0)), + ] + + # https://docs.python.org/3/library/struct.html + @classmethod + def unpack_data(self, mft_record, offset, data_type): + """Helper to unpack values from the raw mft_record""" + + if data_type == 'unsigned long': + return unpack(' 1000: + continue + + # attr_header + attr_type = self.unpack_data(mft_record, attr_offset, 'int') + attr_len = self.unpack_data(mft_record, attr_offset+4, 'int') + + # As we look for strucutres of header + 1K we can not unpack non resident structures + nr_flag = self.unpack_data(mft_record, attr_offset+8, 'unsigned char') + + # Skip headers + attr_data = attr_offset+24 # Len of Common and Resident Headers + + if attr_type in ATTRIBUTE_TYPE_ID: + vollog.debug(f'Found Attribute {ATTRIBUTE_TYPE_ID[attr_type]}') + + if ATTRIBUTE_TYPE_ID[attr_type] == 'STANDARD_INFORMATION': + creation_time_win = self.unpack_data(mft_record, attr_data, 'unsigned long long') + modified_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') + altered_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') + access_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') + flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') + permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') + + + mft_entry['attributes']['SI'] = { + "creation_time": self.human_date(creation_time_win), + "modified_time": self.human_date(modified_time_win), + "updated_time": self.human_date(altered_time_win), + "accessed_time": self.human_date(access_time_win), + "flags": permissions + } + + if ATTRIBUTE_TYPE_ID[attr_type] == 'FILE_NAME': + parent_record = self.unpack_data(mft_record, attr_data, 'unsigned long long') + creation_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') + modified_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') + altered_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') + access_time_win = self.unpack_data(mft_record, attr_data+32, 'unsigned long long') + + name_len = self.unpack_data(mft_record, attr_data+64, 'unsigned char') + name_space = self.unpack_data(mft_record, attr_data+65, 'unsigned char') + + # Unicode and partially corruprted records can break us here. + file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] + try: + file_name = file_name.replace(b'\x00', b'').decode() + except: + file_name = str(file_name.replace(b'\x00', b'')) + + flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') + permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') + + mft_entry['attributes']['FN'].append( + { + "creation_time": self.human_date(creation_time_win), + "modified_time": self.human_date(modified_time_win), + "updated_time": self.human_date(altered_time_win), + "accessed_time": self.human_date(access_time_win), + "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), + "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), + "flags": permissions, + "file_name": file_name, + "name_space": name_space + }) + + # Update Offset for next Attribute + attr_offset += attr_len + + return mft_entry + + def _generator(self): + rules = yara.compile(sources = signatures) + + layer = self.context.layers[self.config['primary']] + for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + + try: + mft_record = layer.read(offset, 1024, False) + mft_entry = self.parse_mft_record(mft_record) + except PagedInvalidAddressException: + mft_entry = None + except Exception as err: + vollog.error(err) + mft_entry = None + + if mft_entry: + vollog.debug(mft_entry) + + # Tree Grid is large and variable + si = mft_entry['attributes']['SI'] + fn = mft_entry['attributes']['FN'] + + signature = mft_entry.get('signature', 0) + record_number = mft_entry.get('record_number', 0) + link_count = mft_entry.get('link_count', 0) + permissions = mft_entry.get('flags', '') + + si_creation_time = si.get('creation_time', '') + si_modified_time = si.get('modified_time', '') + si_updated_time = si.get('updated_time', '') + si_accessed_time = si.get('accessed_time', '') + + yield 0, ( + format_hints.Hex(offset), + signature, + record_number, + link_count, + permissions, + 'Standard Information', + 'N/A', + si_creation_time, + si_modified_time, + si_updated_time, + si_accessed_time) + + for entry in fn: + # As this is variable and may or may not exist + # And could have 0-6 entries lets do it per row. + yield 0, ( + format_hints.Hex(offset), + signature, + record_number, + link_count, + permissions, + 'FileName', + entry.get('file_name', ''), + entry.get('creation_time', ''), + entry.get('modified_time', ''), + entry.get('updated_time', ''), + entry.get('accessed_time', '')) + + def run(self): + return renderers.TreeGrid([ + ('Offset', format_hints.Hex), + ('Record Type', str), + ('Record Number', int), + ('Link Count', int), + ('Permissions', str), + ('Attribute Type', str), + ('Filename', str), + ('Created', str), + ('Modified', str), + ('Updated', str), + ('Accessed', str) + ],self._generator()) From 899ec09ce3b620f75c9ecb74e117d8a3a918be3b Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Thu, 6 Jan 2022 23:09:50 +0000 Subject: [PATCH 020/158] Doc Strings --- .../framework/plugins/windows/mftscan.py | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 1d4ca954e..b75756a1d 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,13 @@ import logging from struct import unpack -from typing import Iterable +from typing import Dict -from volatility3.framework import constants, renderers, interfaces, exceptions +from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols import intermed from volatility3.plugins import yarascan vollog = logging.getLogger(__name__) @@ -109,8 +109,17 @@ class MFTScan(interfaces.plugins.PluginInterface): # https://docs.python.org/3/library/struct.html @classmethod - def unpack_data(self, mft_record, offset, data_type): - """Helper to unpack values from the raw mft_record""" + def unpack_data(self, mft_record: bytes, offset: int, data_type: str) -> bytes: + """Helper to unpack values from the raw mft_record + + Args: + mft_record: 1024 bytes starting from header value as returned by layer read + offset: how far in to the record to read + data_type: what is the data type to unpack + + Returns: + bytes: the unpacked data + """ if data_type == 'unsigned long': return unpack(' str: + """Converts a windows epoch to a date time string with a fixed format + + Args: + datetime_object: windows epoch time + + Returns: + str: strftime of the windows epoch in UTC + + """ dtg = conversion.wintime_to_datetime(datetime_object) return dtg.strftime('%Y-%m-%d %H:%M:%S %z') @classmethod - def parse_mft_record(self, mft_record): - """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes""" + def parse_mft_record(self, mft_record: bytes) -> Dict: + """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes + + Args: + mft_record: 1024 bytes starting from header value as returned by layer read + + Returns: + Dict: a Dictionary that contains the Parse MFT Record + """ # https://github.com/Invoke-IR/ForensicPosters flags = self.unpack_data(mft_record, 22, 'unsigned short') @@ -202,10 +226,11 @@ class MFTScan(interfaces.plugins.PluginInterface): # Unicode and partially corruprted records can break us here. file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - try: - file_name = file_name.replace(b'\x00', b'').decode() - except: - file_name = str(file_name.replace(b'\x00', b'')) + file_name = utility.array_to_string(file_name) + #try: + # # file_name = file_name.replace(b'\x00', b'').decode() + #except: + # file_name = str(file_name.replace(b'\x00', b'')) flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') @@ -234,6 +259,7 @@ class MFTScan(interfaces.plugins.PluginInterface): layer = self.context.layers[self.config['primary']] for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + # For each matching rule try to read 1024 bytes (size of an MFT record) at the offset. try: mft_record = layer.read(offset, 1024, False) mft_entry = self.parse_mft_record(mft_record) From 20a5868ff3df16710695b497069aba9ff0387842 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 00:23:31 +0000 Subject: [PATCH 021/158] Change return types for MFT Records DTGs --- .../framework/plugins/windows/mftscan.py | 101 ++++++++---------- 1 file changed, 44 insertions(+), 57 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index b75756a1d..ddc179fe1 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -2,14 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime import logging +import struct -from struct import unpack from typing import Dict from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework import exceptions from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints from volatility3.plugins import yarascan @@ -122,29 +123,15 @@ class MFTScan(interfaces.plugins.PluginInterface): """ if data_type == 'unsigned long': - return unpack(' str: - """Converts a windows epoch to a date time string with a fixed format - - Args: - datetime_object: windows epoch time - - Returns: - str: strftime of the windows epoch in UTC - - """ - dtg = conversion.wintime_to_datetime(datetime_object) - return dtg.strftime('%Y-%m-%d %H:%M:%S %z') + return struct.unpack(' Dict: @@ -207,10 +194,10 @@ class MFTScan(interfaces.plugins.PluginInterface): mft_entry['attributes']['SI'] = { - "creation_time": self.human_date(creation_time_win), - "modified_time": self.human_date(modified_time_win), - "updated_time": self.human_date(altered_time_win), - "accessed_time": self.human_date(access_time_win), + "creation_time": conversion.wintime_to_datetime(creation_time_win), + "modified_time": conversion.wintime_to_datetime(modified_time_win), + "updated_time": conversion.wintime_to_datetime(altered_time_win), + "accessed_time": conversion.wintime_to_datetime(access_time_win), "flags": permissions } @@ -226,21 +213,21 @@ class MFTScan(interfaces.plugins.PluginInterface): # Unicode and partially corruprted records can break us here. file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - file_name = utility.array_to_string(file_name) - #try: - # # file_name = file_name.replace(b'\x00', b'').decode() - #except: - # file_name = str(file_name.replace(b'\x00', b'')) + #file_name = utility.array_to_string(file_name) + try: + file_name = file_name.replace(b'\x00', b'').decode() + except: + file_name = str(file_name.replace(b'\x00', b'')) flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') mft_entry['attributes']['FN'].append( { - "creation_time": self.human_date(creation_time_win), - "modified_time": self.human_date(modified_time_win), - "updated_time": self.human_date(altered_time_win), - "accessed_time": self.human_date(access_time_win), + "creation_time": conversion.wintime_to_datetime(creation_time_win), + "modified_time": conversion.wintime_to_datetime(modified_time_win), + "updated_time": conversion.wintime_to_datetime(altered_time_win), + "accessed_time": conversion.wintime_to_datetime(access_time_win), "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), "flags": permissions, @@ -263,11 +250,11 @@ class MFTScan(interfaces.plugins.PluginInterface): try: mft_record = layer.read(offset, 1024, False) mft_entry = self.parse_mft_record(mft_record) - except PagedInvalidAddressException: - mft_entry = None - except Exception as err: - vollog.error(err) + except exceptions.PagedInvalidAddressException: mft_entry = None + #except Exception as err: + # vollog.error(err) + # mft_entry = None if mft_entry: vollog.debug(mft_entry) @@ -276,15 +263,15 @@ class MFTScan(interfaces.plugins.PluginInterface): si = mft_entry['attributes']['SI'] fn = mft_entry['attributes']['FN'] - signature = mft_entry.get('signature', 0) - record_number = mft_entry.get('record_number', 0) - link_count = mft_entry.get('link_count', 0) - permissions = mft_entry.get('flags', '') + signature = mft_entry.get('signature', renderers.NotAvailableValue()) + record_number = mft_entry.get('record_number', renderers.NotAvailableValue()) + link_count = mft_entry.get('link_count', renderers.NotAvailableValue()) + permissions = mft_entry.get('flags', renderers.NotAvailableValue()) - si_creation_time = si.get('creation_time', '') - si_modified_time = si.get('modified_time', '') - si_updated_time = si.get('updated_time', '') - si_accessed_time = si.get('accessed_time', '') + si_creation_time = si.get('creation_time', renderers.NotAvailableValue()) + si_modified_time = si.get('modified_time', renderers.NotAvailableValue()) + si_updated_time = si.get('updated_time', renderers.NotAvailableValue()) + si_accessed_time = si.get('accessed_time', renderers.NotAvailableValue()) yield 0, ( format_hints.Hex(offset), @@ -293,7 +280,7 @@ class MFTScan(interfaces.plugins.PluginInterface): link_count, permissions, 'Standard Information', - 'N/A', + renderers.NotApplicableValue(), si_creation_time, si_modified_time, si_updated_time, @@ -302,18 +289,18 @@ class MFTScan(interfaces.plugins.PluginInterface): for entry in fn: # As this is variable and may or may not exist # And could have 0-6 entries lets do it per row. - yield 0, ( + yield 1, ( format_hints.Hex(offset), signature, record_number, link_count, permissions, 'FileName', - entry.get('file_name', ''), - entry.get('creation_time', ''), - entry.get('modified_time', ''), - entry.get('updated_time', ''), - entry.get('accessed_time', '')) + entry.get('file_name',''), + entry.get('creation_time', renderers.NotAvailableValue()), + entry.get('modified_time', renderers.NotAvailableValue()), + entry.get('updated_time', renderers.NotAvailableValue()), + entry.get('accessed_time', renderers.NotAvailableValue())) def run(self): return renderers.TreeGrid([ @@ -324,8 +311,8 @@ class MFTScan(interfaces.plugins.PluginInterface): ('Permissions', str), ('Attribute Type', str), ('Filename', str), - ('Created', str), - ('Modified', str), - ('Updated', str), - ('Accessed', str) + ('Created', datetime.datetime), + ('Modified', datetime.datetime), + ('Updated', datetime.datetime), + ('Accessed', datetime.datetime) ],self._generator()) From 295fb453f5e73f65d39912336ef6517268e26d17 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 00:55:34 +0000 Subject: [PATCH 022/158] Add MFT Filename N/A type --- volatility3/framework/plugins/windows/mftscan.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index ddc179fe1..10db6f112 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -192,7 +192,6 @@ class MFTScan(interfaces.plugins.PluginInterface): flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - mft_entry['attributes']['SI'] = { "creation_time": conversion.wintime_to_datetime(creation_time_win), "modified_time": conversion.wintime_to_datetime(modified_time_win), @@ -296,7 +295,7 @@ class MFTScan(interfaces.plugins.PluginInterface): link_count, permissions, 'FileName', - entry.get('file_name',''), + entry.get('file_name',renderers.NotAvailableValue()), entry.get('creation_time', renderers.NotAvailableValue()), entry.get('modified_time', renderers.NotAvailableValue()), entry.get('updated_time', renderers.NotAvailableValue()), From 114a8d7c8d0195a83bfc0c0654fb24bd63202302 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 9 Jan 2022 20:32:34 +0000 Subject: [PATCH 023/158] Plugins: Update yarascan options Add in a yara_source option in the process method. Unfortunately yara_rules is still poorly named, but would require a major version bump, so to avoid major disruption, we're just adding the yara_source option instead. --- volatility3/framework/plugins/yarascan.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 94b3cba45..4d2ba88ee 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -3,7 +3,7 @@ # import logging -from typing import Iterable, Tuple, List, Dict, Any +from typing import Any, Dict, Iterable, List, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -40,7 +40,7 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -83,6 +83,8 @@ class YaraScan(plugins.PluginInterface): if config.get('wide', False): rule += " wide ascii" rules = yara.compile(sources = {'n': f'rule r1 {{strings: $a = {rule} condition: $a}}'}) + elif config.get('yara_source', None) is not None: + rules = yara.compile(source = config['yara_source']) elif config.get('yara_file', None) is not None: rules = yara.compile(file = resources.ResourceAccessor().open(config['yara_file'], "rb")) elif config.get('yara_compiled_file', None) is not None: From d34030e9ea44e81287c4836d9851280b3190d096 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 9 Jan 2022 20:52:49 +0000 Subject: [PATCH 024/158] Plugins: Add note to improve yarascan in the future --- volatility3/framework/plugins/yarascan.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 4d2ba88ee..e51669b2c 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -42,6 +42,9 @@ class YaraScan(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 1, 0) + # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string + # or something that makes more sense + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ From 9d86599e7f391c41f2fb7e06f6f0f811bd97a7b7 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 22:04:13 +0000 Subject: [PATCH 025/158] MFT Plugin use ISF instead of Struct --- .../framework/plugins/windows/mftscan.py | 352 ++++------------- .../symbols/windows/extensions/mft.py | 104 +++++ .../framework/symbols/windows/mft.json | 371 ++++++++++++++++++ volatility3/framework/symbols/windows/mft.py | 15 + 4 files changed, 577 insertions(+), 265 deletions(-) create mode 100644 volatility3/framework/symbols/windows/extensions/mft.py create mode 100644 volatility3/framework/symbols/windows/mft.json create mode 100644 volatility3/framework/symbols/windows/mft.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 10db6f112..484fcdb9a 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -4,95 +4,20 @@ import datetime import logging -import struct from typing import Dict from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions -from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints +from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags +from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols + from volatility3.plugins import yarascan vollog = logging.getLogger(__name__) -try: - import yara -except ImportError: - vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") - raise - -signatures = { - 'mft_objects': """rule mft_headers - { - strings: - $header1 = "FILE0" - $header2 = "FILE*" - $header3 = "BAAD" - condition: - any of them - }""" -} - -# https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee9338b881c0fa25937/volatility/plugins/mftparser.py#L60 -ATTRIBUTE_TYPE_ID = { - 0x10:"STANDARD_INFORMATION", - 0x20:"ATTRIBUTE_LIST", - 0x30:"FILE_NAME", - 0x40:"OBJECT_ID", - 0x50:"SECURITY_DESCRIPTOR", - 0x60:"VOLUME_NAME", - 0x70:"VOLUME_INFORMATION", - 0x80:"DATA", - 0x90:"INDEX_ROOT", - 0xa0:"INDEX_ALLOCATION", - 0xb0:"BITMAP", - 0xc0:"REPARSE_POINT", - 0xd0:"EA_INFORMATION", #Extended Attribute - 0xe0:"EA", - 0xf0:"PROPERTY_SET", - 0x100:"LOGGED_UTILITY_STREAM", -} - -VERBOSE_STANDARD_INFO_FLAGS = { - 0x1:"Read Only", - 0x2:"Hidden", - 0x4:"System", - 0x20:"Archive", - 0x40:"Device", - 0x80:"Normal", - 0x100:"Temporary", - 0x200:"Sparse File", - 0x400:"Reparse Point", - 0x800:"Compressed", - 0x1000:"Offline", - 0x2000:"Content not indexed", - 0x4000:"Encrypted", - 0x10000000:"Directory", - 0x20000000:"Index view", -} - -FILE_NAME_NAMESPACE = { - 0x0:"POSIX", # Case sensitive, allows all Unicode chars except '/' and NULL - 0x1:"Win32", # Case insensitive, allows most Unicide except specials ('/', '\', ';', '>', '<', '?') - 0x2:"DOS", # Case insensitive, upper case, no special chars, name is 8 or fewer chars in name and 3 or less extension - 0x3:"Win32 & DOS", # Used when original name fits in DOS namespace and 2 names are not needed -} - -MFT_FLAGS = { - 0x0: "Removed", - 0x1: "File", # "In Use", - 0x2: "Directory", # if flag & 0x0002 == 0 this is a regular file - 0x3: "Directory" -} - -INDEX_ENTRY_FLAGS = { - 0x1:"Child Node Exists", - 0x2:"Last entry in list", -} - - class MFTScan(interfaces.plugins.PluginInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" @@ -108,198 +33,94 @@ class MFTScan(interfaces.plugins.PluginInterface): version = (2, 0, 0)), ] - # https://docs.python.org/3/library/struct.html - @classmethod - def unpack_data(self, mft_record: bytes, offset: int, data_type: str) -> bytes: - """Helper to unpack values from the raw mft_record - - Args: - mft_record: 1024 bytes starting from header value as returned by layer read - offset: how far in to the record to read - data_type: what is the data type to unpack - - Returns: - bytes: the unpacked data - """ - - if data_type == 'unsigned long': - return struct.unpack(' Dict: - """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes - - Args: - mft_record: 1024 bytes starting from header value as returned by layer read - - Returns: - Dict: a Dictionary that contains the Parse MFT Record - """ - # https://github.com/Invoke-IR/ForensicPosters - - flags = self.unpack_data(mft_record, 22, 'unsigned short') - file_type = MFT_FLAGS.get(flags, 'Unknown') - - mft_entry = { - "signature": mft_record[:4].decode(), - "FixupArrayOffset": self.unpack_data(mft_record, 4, 'unsigned short'), - "NumFixupEntries": self.unpack_data(mft_record, 6, 'unsigned short'), - "LSN": self.unpack_data(mft_record, 8, 'unsigned long long'), - "SequenceValue": self.unpack_data(mft_record, 16, 'unsigned short'), - "link_count": self.unpack_data(mft_record, 18, 'unsigned short'), - "FirstAttrOffset": self.unpack_data(mft_record, 20, 'unsigned short'), - "flags": file_type, - "record_number": self.unpack_data(mft_record, 44, 'unsigned long'), - "attributes": { - "SI": {}, - "FN": [] - } - } - - attr_offset = mft_entry['FirstAttrOffset'] - # Check at most for 6 entries - for i in range(6): - # If we attempt to overread the entry continue out - if attr_offset > 1000: - continue - - # attr_header - attr_type = self.unpack_data(mft_record, attr_offset, 'int') - attr_len = self.unpack_data(mft_record, attr_offset+4, 'int') - - # As we look for strucutres of header + 1K we can not unpack non resident structures - nr_flag = self.unpack_data(mft_record, attr_offset+8, 'unsigned char') - - # Skip headers - attr_data = attr_offset+24 # Len of Common and Resident Headers - - if attr_type in ATTRIBUTE_TYPE_ID: - vollog.debug(f'Found Attribute {ATTRIBUTE_TYPE_ID[attr_type]}') - - if ATTRIBUTE_TYPE_ID[attr_type] == 'STANDARD_INFORMATION': - creation_time_win = self.unpack_data(mft_record, attr_data, 'unsigned long long') - modified_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') - altered_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') - access_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') - flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') - permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - - mft_entry['attributes']['SI'] = { - "creation_time": conversion.wintime_to_datetime(creation_time_win), - "modified_time": conversion.wintime_to_datetime(modified_time_win), - "updated_time": conversion.wintime_to_datetime(altered_time_win), - "accessed_time": conversion.wintime_to_datetime(access_time_win), - "flags": permissions - } - - if ATTRIBUTE_TYPE_ID[attr_type] == 'FILE_NAME': - parent_record = self.unpack_data(mft_record, attr_data, 'unsigned long long') - creation_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') - modified_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') - altered_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') - access_time_win = self.unpack_data(mft_record, attr_data+32, 'unsigned long long') - - name_len = self.unpack_data(mft_record, attr_data+64, 'unsigned char') - name_space = self.unpack_data(mft_record, attr_data+65, 'unsigned char') - - # Unicode and partially corruprted records can break us here. - file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - #file_name = utility.array_to_string(file_name) - try: - file_name = file_name.replace(b'\x00', b'').decode() - except: - file_name = str(file_name.replace(b'\x00', b'')) - - flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') - permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - - mft_entry['attributes']['FN'].append( - { - "creation_time": conversion.wintime_to_datetime(creation_time_win), - "modified_time": conversion.wintime_to_datetime(modified_time_win), - "updated_time": conversion.wintime_to_datetime(altered_time_win), - "accessed_time": conversion.wintime_to_datetime(access_time_win), - "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), - "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), - "flags": permissions, - "file_name": file_name, - "name_space": name_space - }) - - # Update Offset for next Attribute - attr_offset += attr_len - - return mft_entry def _generator(self): - rules = yara.compile(sources = signatures) - layer = self.context.layers[self.config['primary']] + + # Yara Rule to scan for MFT Header Signatures + rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) + + # Read in the Symbol File + symbol_table = MFTIntermedSymbols.create( + self.context, + self.config_path, + "windows", + "mft" + ) + + # get each of the individual Field Sets + mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + header_object = symbol_table + constants.BANG + "ATTR_HEADER" + si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Scan the layer for Raw MFT records and parse the fields for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): - - # For each matching rule try to read 1024 bytes (size of an MFT record) at the offset. try: - mft_record = layer.read(offset, 1024, False) - mft_entry = self.parse_mft_record(mft_record) + mft_record = self.context.object(mft_object, offset=offset, layer_name=layer.name) + # We will update this on each pass in the next loop and use it as the new offset. + attr_base_offset = mft_record.FirstAttrOffset + + # There is no field that has a count of Attributes + # Keep Attempting to read attributes until we get an invalid attr_header.AttrType + while True: + attr_header = self.context.object(header_object, offset=offset+attr_base_offset, layer_name=layer.name) + attr_resident_header = self.context.object(header_object, offset=offset+attr_base_offset+16, layer_name=layer.name) + + vollog.debug(f"Attr Type: {attr_header.AttrType}") + + # If this is not a valid type then exit the loop + if not AttributeTypes(attr_header.AttrType).value: + break + + # Offset past the headers to the attribute data + attr_data_offset = offset+attr_base_offset+24 + + # Standard Information Attribute + if attr_header.AttrType == 0x10: + attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) + + yield 0, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + MFTFlags(mft_record.Flags).name, + renderers.NotApplicableValue(), + AttributeTypes(attr_header.AttrType).name, + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + renderers.NotApplicableValue(), + ) + + # File Name Attribute + if attr_header.AttrType == 0x30: + attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) + file_name = attr_data.get_full_name() + + yield 1, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + MFTFlags(mft_record.Flags).name, + PermissionFlags(attr_data.Flags).name, + AttributeTypes(attr_header.AttrType).name, + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + file_name + ) + + # Update the base offset to point to the next attribute + attr_base_offset += attr_header.Length + except exceptions.PagedInvalidAddressException: - mft_entry = None - #except Exception as err: - # vollog.error(err) - # mft_entry = None + pass - if mft_entry: - vollog.debug(mft_entry) - - # Tree Grid is large and variable - si = mft_entry['attributes']['SI'] - fn = mft_entry['attributes']['FN'] - - signature = mft_entry.get('signature', renderers.NotAvailableValue()) - record_number = mft_entry.get('record_number', renderers.NotAvailableValue()) - link_count = mft_entry.get('link_count', renderers.NotAvailableValue()) - permissions = mft_entry.get('flags', renderers.NotAvailableValue()) - - si_creation_time = si.get('creation_time', renderers.NotAvailableValue()) - si_modified_time = si.get('modified_time', renderers.NotAvailableValue()) - si_updated_time = si.get('updated_time', renderers.NotAvailableValue()) - si_accessed_time = si.get('accessed_time', renderers.NotAvailableValue()) - - yield 0, ( - format_hints.Hex(offset), - signature, - record_number, - link_count, - permissions, - 'Standard Information', - renderers.NotApplicableValue(), - si_creation_time, - si_modified_time, - si_updated_time, - si_accessed_time) - - for entry in fn: - # As this is variable and may or may not exist - # And could have 0-6 entries lets do it per row. - yield 1, ( - format_hints.Hex(offset), - signature, - record_number, - link_count, - permissions, - 'FileName', - entry.get('file_name',renderers.NotAvailableValue()), - entry.get('creation_time', renderers.NotAvailableValue()), - entry.get('modified_time', renderers.NotAvailableValue()), - entry.get('updated_time', renderers.NotAvailableValue()), - entry.get('accessed_time', renderers.NotAvailableValue())) def run(self): return renderers.TreeGrid([ @@ -307,11 +128,12 @@ class MFTScan(interfaces.plugins.PluginInterface): ('Record Type', str), ('Record Number', int), ('Link Count', int), + ('MFT Type', str), ('Permissions', str), ('Attribute Type', str), - ('Filename', str), ('Created', datetime.datetime), ('Modified', datetime.datetime), ('Updated', datetime.datetime), - ('Accessed', datetime.datetime) + ('Accessed', datetime.datetime), + ('Filename', str), ],self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py new file mode 100644 index 000000000..09f6346cc --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -0,0 +1,104 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import enum + +from volatility3.framework import exceptions, objects, renderers +from volatility3.framework.objects import utility + + +class AttributeTypes(enum.Enum): + STANDARD_INFORMATION = 0x10 + ATTRIBUTE_LIST = 0x20 + FILE_NAME = 0x30 + OBJECT_ID = 0x40 + SECURITY_DESCRIPTOR = 0x50 + VOLUME_NAME = 0x60 + VOLUME_INFORMATION = 0x70 + DATA = 0x80 + INDEX_ROOT = 0x90 + INDEX_ALLOCATION = 0xa0 + BITMAP = 0xb0 + REPARSE_POINT = 0xc0 + EA_INFORMATION = 0xd0 + EA = 0xe0 + PROPERTY_SET = 0xf0 + LOGGED_UTILITY_STREAM = 0x100 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(AttributeTypes.Unknown) + +class NameSpace(enum.Enum): + POSIX = 0x0 + Win32 = 0x1 + DOS = 0x2 + Win32DOS = 0x3 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(NameSpace.Unknown) + + +class MFTFlags(enum.Enum): + Removed = 0x00 + File = 0x1 + Directory = 0x2 + DirInUse = 0x3 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(MFTFlags.Unknown) + + +class PermissionFlags(enum.Enum): + ReadOnly = 0x1 + Hidden = 0x2 + System = 0x4 + Archive = 0x20 + ArchiveHidden = 0x22 + ArchiveSystem = 0x24 + ArchiveHiddenSystem = 0x26 + Device = 0x40 + Normal = 0x80 + Temporary = 0x100 + TempArchive = 0x120 + SparseFile = 0x200 + ReparsePoint = 0x400 + Compressed = 0x800 + Offline = 0x1000 + NotIndexed = 0x2000 + Encrypted = 0x4000 + Directory = 0x10000000 + IndexView = 0x20000000 + unknown = None + + @classmethod + def _missing_(cls, value): + return cls(PermissionFlags.unknown) + + +class MFTEntry(objects.StructType): + """This represents the base MFT Record""" + + def get_signature(self) -> str: + signature = self.Signature.cast('string', max_length = 4, encoding = 'latin-1') + return signature + + +class MFTFileName(objects.StructType): + """This represents an MFT $FILE_NAME Attribute""" + + def get_full_name(self) -> str: + output = self.Name.cast("string", + encoding = "utf16", + max_length = self.NameLength*2, + errors = "replace") + return output + + def get_file_namespace(self) -> str: + pass diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json new file mode 100644 index 000000000..e045ed0fc --- /dev/null +++ b/volatility3/framework/symbols/windows/mft.json @@ -0,0 +1,371 @@ +{ + "metadata": { + "producer": { + "version": "0.0.1", + "name": "kevthehermit-by-hand", + "comment": "Using structures defined in File System Forensic Analysis pg 353+", + "datetime": "2022-01-03T13:37:00" + }, + "format": "6.1.0" + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": true, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "int", + "size": 1, + "signed": false, + "endian": "little" + }, + "wchar": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + } + }, + "symbols": {}, + "enums": {}, + "user_types": { + "MFT_ENTRY": { + "fields": { + "Signature": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "UpdateSequenceOffset": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "NumFixupEntries": { + "offset": 6, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "LSN": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "SequenceValue": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "LinkCount": { + "offset": 18, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "FirstAttrOffset": { + "offset": 20, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "Flags": { + "offset": 22, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "RealSize": { + "offset": 24, + "type":{ + "kind": "base", + "name": "unsigned int" + } + }, + "AlocatedSize": { + "offset": 28, + "type":{ + "kind": "base", + "name": "unsigned int" + } + }, + "BaseReference": { + "offset": 32, + "type":{ + "kind": "base", + "name": "unsigned long long" + } + }, + "NextAttrID": { + "offset": 40, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "RecordNumber": { + "offset": 44, + "type":{ + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 1024 + },"ATTR_HEADER": { + "fields": { + "AttrType": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned int" + } + },"Length": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "NonResidentFlag": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned char" } + }, + "NameLength": { + "offset": 9, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "NameOffset": { + "offset": 10, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "Flags": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "AttributeID": { + "offset": 14, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 16 + },"RESIDENT_HEADER": { + "fields": { + "AttrSize": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned int" + } + },"AttrOffset": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "IndexFlag": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned short" } + } + }, + "kind": "struct", + "size": 8 + }, + "STANDARD_INFORMATION_ENTRY": { + "fields": { + "CreationTime": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ModifiedTime": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "UpdatedTime": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AccessedTime": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "flags": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 1024 + }, + "FILE_NAME_ENTRY": { + "fields": { + "ParentDirectory": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "CreationTime": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ModifiedTime": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "UpdatedTime": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AccessedTime": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AllocatedFileSize": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "RealFileSize": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Flags": { + "offset": 56, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ReparseValue": { + "offset": 60, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "NameLength": { + "offset": 64, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "NameSpace": { + "offset": 65, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "Name": { + "offset": 66, + "type": { + "count": 10, + "kind": "array", + "subtype": { + "kind": "base", + "name": "wchar" + } + } + } + }, + "kind": "struct", + "size": 1024 + } + } +} \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/mft.py b/volatility3/framework/symbols/windows/mft.py new file mode 100644 index 000000000..921d75cd8 --- /dev/null +++ b/volatility3/framework/symbols/windows/mft.py @@ -0,0 +1,15 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mft + + +class MFTIntermedSymbols(intermed.IntermediateSymbolTable): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.set_type_class('FILE_NAME_ENTRY', mft.MFTFileName) + self.set_type_class('MFT_ENTRY', mft.MFTEntry) From 793d08faf487c1d440bae016d8ca1e87766da7cc Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 22:27:32 +0000 Subject: [PATCH 026/158] Add TimeLiner interface to MFTScan plugin --- volatility3/framework/plugins/windows/mftscan.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 484fcdb9a..616c0d738 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -14,11 +14,11 @@ from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols -from volatility3.plugins import yarascan +from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) -class MFTScan(interfaces.plugins.PluginInterface): +class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) @@ -121,6 +121,18 @@ class MFTScan(interfaces.plugins.PluginInterface): except exceptions.PagedInvalidAddressException: pass + def generate_timeline(self): + for row in self._generator(): + if row[-1] != 'N/A': + filename = row[-1] + created = f'File {row[-1]} Created' + updated = f'File {row[-1]} Updated' + modified = f'File {row[-1]} Modified' + accessed = f'File {row[-1]} Accessed' + yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) + yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) + yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) + yield (f'File {filename} accessed', timeliner.TimeLinerType.ACCESSED, row[10]) def run(self): return renderers.TreeGrid([ From a0b66f33f90968b24e1cefab9bd7d7efd0b12638 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jan 2022 21:06:20 +0000 Subject: [PATCH 027/158] Documentation: Update README.md before release --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 75c3c1c23..f7d326c33 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ technical and performance challenges associated with the original code base that became apparent over the previous 10 years. Another benefit of the rewrite is that Volatility 3 could be released under a custom license that was more aligned with the goals of the Volatility community, -the Volatility Software License (VSL). See the [LICENSE](LICENSE.txt) file for more details. +the Volatility Software License (VSL). See the +[LICENSE](https://www.volatilityfoundation.org/license/vsl-v1.0) file for +more details. ## Requirements @@ -102,7 +104,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 12 Jan 2022 21:06:20 +0000 Subject: [PATCH 028/158] Documentation: Update README.md before release --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 75c3c1c23..f7d326c33 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ technical and performance challenges associated with the original code base that became apparent over the previous 10 years. Another benefit of the rewrite is that Volatility 3 could be released under a custom license that was more aligned with the goals of the Volatility community, -the Volatility Software License (VSL). See the [LICENSE](LICENSE.txt) file for more details. +the Volatility Software License (VSL). See the +[LICENSE](https://www.volatilityfoundation.org/license/vsl-v1.0) file for +more details. ## Requirements @@ -102,7 +104,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 12 Jan 2022 21:13:05 +0000 Subject: [PATCH 029/158] Documentation: Update master branch to stable branch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7d326c33..9f9c1bbb7 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ pip3 install -r requirements.txt ## Downloading Volatility -The latest stable version of Volatility will always be the master branch of the GitHub repository. You can get the latest version of the code using the following command: +The latest stable version of Volatility will always be the stable branch of the GitHub repository. You can get the latest version of the code using the following command: ```shell git clone https://github.com/volatilityfoundation/volatility3.git From d469d9c597020fe52649e8a27b1b4738d805ccdc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jan 2022 21:13:05 +0000 Subject: [PATCH 030/158] Documentation: Update master branch to stable branch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7d326c33..9f9c1bbb7 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ pip3 install -r requirements.txt ## Downloading Volatility -The latest stable version of Volatility will always be the master branch of the GitHub repository. You can get the latest version of the code using the following command: +The latest stable version of Volatility will always be the stable branch of the GitHub repository. You can get the latest version of the code using the following command: ```shell git clone https://github.com/volatilityfoundation/volatility3.git From f67f1e242d7f5cf1571864d052a545161849c8e2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jan 2022 22:11:25 +0000 Subject: [PATCH 031/158] Documentation: Ensure the doc reqs are included in s_dist builds --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 504c7d89a..1cec729f6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ prune development include * .* -include doc/make.bat doc/Makefile +include doc/make.bat doc/Makefile doc/requirements.txt recursive-include doc/source * recursive-include volatility3 *.json recursive-exclude doc/source volatility3.*.rst From d91a6f94fbe015a4b837efcbd68407a93a31b682 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 13 Jan 2022 01:01:20 +0000 Subject: [PATCH 032/158] Automagic: Ensure linxu/mac are excluded from windows automagic --- volatility3/framework/automagic/pdbscan.py | 1 + volatility3/framework/automagic/windows.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 93d4337da..8179339c8 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -44,6 +44,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): """ priority = 30 max_pdb_size = 0x400000 + exclusion_list = ['linux', 'mac'] def find_virtual_layers_from_req(self, context: interfaces.context.ContextInterface, config_path: str, requirement: interfaces.configuration.RequirementInterface) -> List[str]: diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index eb63a75e5..71548ca40 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -238,6 +238,8 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): """Class to read swap_layers filenames from single-swap-layers, create the layers and populate the single-layers swap_layers.""" + exclusion_list = ['linux', 'mac'] + def __call__(self, context: interfaces.context.ContextInterface, config_path: str, From 1c6cd0fb528b02e8f35ac65f1173241ff84dfe26 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 15:53:35 +0000 Subject: [PATCH 033/158] Move mftscan enums to ISF file. --- .../framework/plugins/windows/mftscan.py | 33 +++++--- .../symbols/windows/extensions/mft.py | 77 ------------------- .../framework/symbols/windows/mft.json | 70 ++++++++++++++++- 3 files changed, 93 insertions(+), 87 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 616c0d738..991191d5b 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -11,7 +11,6 @@ from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols from volatility3.plugins import timeliner, yarascan @@ -53,6 +52,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Get the Enums + attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") + namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") + mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") + permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): @@ -70,14 +75,20 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.debug(f"Attr Type: {attr_header.AttrType}") # If this is not a valid type then exit the loop - if not AttributeTypes(attr_header.AttrType).value: + if attr_header.AttrType not in attr_types.choices.values(): break # Offset past the headers to the attribute data attr_data_offset = offset+attr_base_offset+24 + + # MFT Flags determine the file type or dir + if mft_record.Flags in mft_flags.choices.values(): + mft_flag = mft_flags.lookup(mft_record.Flags) + else: + mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType == 0x10: + if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) yield 0, ( @@ -85,9 +96,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, - MFTFlags(mft_record.Flags).name, + mft_flag, renderers.NotApplicableValue(), - AttributeTypes(attr_header.AttrType).name, + attr_types.lookup(attr_header.AttrType), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -96,18 +107,22 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType == 0x30: + if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) file_name = attr_data.get_full_name() + if attr_data.Flags in permission_flags.choices.values(): + permissions = permission_flags.lookup(attr_data.Flags) + else: + permissions = hex(attr_data.Flags) yield 1, ( format_hints.Hex(attr_data_offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, - MFTFlags(mft_record.Flags).name, - PermissionFlags(attr_data.Flags).name, - AttributeTypes(attr_header.AttrType).name, + mft_flag, + permissions, + attr_types.lookup(attr_header.AttrType), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 09f6346cc..0713c969a 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -8,80 +8,6 @@ from volatility3.framework import exceptions, objects, renderers from volatility3.framework.objects import utility -class AttributeTypes(enum.Enum): - STANDARD_INFORMATION = 0x10 - ATTRIBUTE_LIST = 0x20 - FILE_NAME = 0x30 - OBJECT_ID = 0x40 - SECURITY_DESCRIPTOR = 0x50 - VOLUME_NAME = 0x60 - VOLUME_INFORMATION = 0x70 - DATA = 0x80 - INDEX_ROOT = 0x90 - INDEX_ALLOCATION = 0xa0 - BITMAP = 0xb0 - REPARSE_POINT = 0xc0 - EA_INFORMATION = 0xd0 - EA = 0xe0 - PROPERTY_SET = 0xf0 - LOGGED_UTILITY_STREAM = 0x100 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(AttributeTypes.Unknown) - -class NameSpace(enum.Enum): - POSIX = 0x0 - Win32 = 0x1 - DOS = 0x2 - Win32DOS = 0x3 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(NameSpace.Unknown) - - -class MFTFlags(enum.Enum): - Removed = 0x00 - File = 0x1 - Directory = 0x2 - DirInUse = 0x3 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(MFTFlags.Unknown) - - -class PermissionFlags(enum.Enum): - ReadOnly = 0x1 - Hidden = 0x2 - System = 0x4 - Archive = 0x20 - ArchiveHidden = 0x22 - ArchiveSystem = 0x24 - ArchiveHiddenSystem = 0x26 - Device = 0x40 - Normal = 0x80 - Temporary = 0x100 - TempArchive = 0x120 - SparseFile = 0x200 - ReparsePoint = 0x400 - Compressed = 0x800 - Offline = 0x1000 - NotIndexed = 0x2000 - Encrypted = 0x4000 - Directory = 0x10000000 - IndexView = 0x20000000 - unknown = None - - @classmethod - def _missing_(cls, value): - return cls(PermissionFlags.unknown) - - class MFTEntry(objects.StructType): """This represents the base MFT Record""" @@ -99,6 +25,3 @@ class MFTFileName(objects.StructType): max_length = self.NameLength*2, errors = "replace") return output - - def get_file_namespace(self) -> str: - pass diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index e045ed0fc..a99b82e9e 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -53,7 +53,75 @@ } }, "symbols": {}, - "enums": {}, + "enums": { + "AttrTypeEnum": { + "base": "unsigned char", + "constants": { + "STANDARD_INFORMATION": 16, + "ATTRIBUTE_LIST": 32, + "FILE_NAME": 48, + "OBJECT_ID": 64, + "SECURITY_DESCRIPTOR": 80, + "VOLUME_NAME": 96, + "VOLUME_INFORMATION": 112, + "DATA": 128, + "INDEX_ROOT": 114, + "INDEX_ALLOCATION": 160, + "BITMAP": 176, + "REPARSE_POINT": 192, + "EA_INFORMATION": 208, + "EA": 224, + "PROPERTY_SET": 240, + "LOGGED_UTILITY_STREAM": 256 + }, + "size": 1 + }, + "NameSpaceEnum": { + "base":"unsigned char", + "constants": { + "POSIX": 0, + "Win32": 1, + "DOS": 2, + "Win32 DOS": 3 + }, + "size": 1 + }, + "MFTFlagsEnum": { + "base":"unsigned char", + "constants": { + "Removed": 0, + "File": 1, + "Directory": 2, + "DirInUse": 3 + }, + "size": 1 + }, + "PermissionFlagEnum": { + "base":"unsigned char", + "constants": { + "ReadOnly": 1, + "Hidden": 2, + "System": 4, + "Archive": 32, + "ArchiveHidden": 34, + "ArchiveSystem": 36, + "ArchiveHiddenSystem": 38, + "Device": 60, + "Normal": 128, + "Temporary": 256, + "TempArchive": 288, + "SparseFile": 512, + "ReparsePoint": 1024, + "Compressed": 2048, + "Offline": 4096, + "NotIndexed": 8192, + "Encrypted": 16384, + "Directory": 268435456, + "IndexView": 536870912 + }, + "size": 1 + } + }, "user_types": { "MFT_ENTRY": { "fields": { From a8f5b0381664b963a565161d88c64fc1b4053102 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 15:57:23 +0000 Subject: [PATCH 034/158] Apply yapf to mftscan plugin --- .../framework/plugins/windows/mftscan.py | 84 +++++++++---------- .../symbols/windows/extensions/mft.py | 10 +-- 2 files changed, 41 insertions(+), 53 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 991191d5b..070061b17 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -17,6 +17,7 @@ from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) + class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" @@ -32,7 +33,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): version = (2, 0, 0)), ] - def _generator(self): layer = self.context.layers[self.config['primary']] @@ -40,12 +40,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) # Read in the Symbol File - symbol_table = MFTIntermedSymbols.create( - self.context, - self.config_path, - "windows", - "mft" - ) + symbol_table = MFTIntermedSymbols.create(self.context, self.config_path, "windows", "mft") # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" @@ -57,20 +52,26 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") - permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") - + permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + + "PermissionFlagEnum") + # Scan the layer for Raw MFT records and parse the fields - for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + for offset, rule_name, name, value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): try: - mft_record = self.context.object(mft_object, offset=offset, layer_name=layer.name) + mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while True: - attr_header = self.context.object(header_object, offset=offset+attr_base_offset, layer_name=layer.name) - attr_resident_header = self.context.object(header_object, offset=offset+attr_base_offset+16, layer_name=layer.name) + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) + attr_resident_header = self.context.object(header_object, + offset = offset + attr_base_offset + 16, + layer_name = layer.name) vollog.debug(f"Attr Type: {attr_header.AttrType}") @@ -79,17 +80,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset+attr_base_offset+24 + attr_data_offset = offset + attr_base_offset + 24 # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): mft_flag = mft_flags.lookup(mft_record.Flags) else: mft_flag = hex(mft_record.Flags) - + # Standard Information Attribute if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): - attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) + attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name) yield 0, ( format_hints.Hex(attr_data_offset), @@ -108,28 +109,21 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # File Name Attribute if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): - attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) + attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name) file_name = attr_data.get_full_name() if attr_data.Flags in permission_flags.choices.values(): permissions = permission_flags.lookup(attr_data.Flags) else: permissions = hex(attr_data.Flags) - yield 1, ( - format_hints.Hex(attr_data_offset), - mft_record.get_signature(), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - permissions, - attr_types.lookup(attr_header.AttrType), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - file_name - ) - + yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(), + mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, + attr_types.lookup(attr_header.AttrType), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), file_name) + # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length @@ -151,16 +145,16 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): def run(self): return renderers.TreeGrid([ - ('Offset', format_hints.Hex), - ('Record Type', str), - ('Record Number', int), - ('Link Count', int), - ('MFT Type', str), - ('Permissions', str), - ('Attribute Type', str), - ('Created', datetime.datetime), - ('Modified', datetime.datetime), - ('Updated', datetime.datetime), - ('Accessed', datetime.datetime), - ('Filename', str), - ],self._generator()) + ('Offset', format_hints.Hex), + ('Record Type', str), + ('Record Number', int), + ('Link Count', int), + ('MFT Type', str), + ('Permissions', str), + ('Attribute Type', str), + ('Created', datetime.datetime), + ('Modified', datetime.datetime), + ('Updated', datetime.datetime), + ('Accessed', datetime.datetime), + ('Filename', str), + ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 0713c969a..ba79b7c8b 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,10 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import enum - -from volatility3.framework import exceptions, objects, renderers -from volatility3.framework.objects import utility +from volatility3.framework import objects class MFTEntry(objects.StructType): @@ -20,8 +17,5 @@ class MFTFileName(objects.StructType): """This represents an MFT $FILE_NAME Attribute""" def get_full_name(self) -> str: - output = self.Name.cast("string", - encoding = "utf16", - max_length = self.NameLength*2, - errors = "replace") + output = self.Name.cast("string", encoding = "utf16", max_length = self.NameLength * 2, errors = "replace") return output From c5987a45d2362562329ca0e977ec87cf76babca9 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 20:54:22 +0000 Subject: [PATCH 035/158] Relative Offset MFT Header --- .../framework/plugins/windows/mftscan.py | 13 +++----- .../framework/symbols/windows/mft.json | 30 ++++++++++++++++++- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 070061b17..3d8d96221 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -44,13 +44,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Get the Enums attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") - namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") + namespace_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") @@ -69,9 +70,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): attr_header = self.context.object(header_object, offset = offset + attr_base_offset, layer_name = layer.name) - attr_resident_header = self.context.object(header_object, - offset = offset + attr_base_offset + 16, - layer_name = layer.name) vollog.debug(f"Attr Type: {attr_header.AttrType}") @@ -80,7 +78,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + 24 + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( + attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -134,10 +133,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for row in self._generator(): if row[-1] != 'N/A': filename = row[-1] - created = f'File {row[-1]} Created' - updated = f'File {row[-1]} Updated' - modified = f'File {row[-1]} Modified' - accessed = f'File {row[-1]} Accessed' yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index a99b82e9e..2470dcbd5 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -223,7 +223,35 @@ }, "kind": "struct", "size": 1024 - },"ATTR_HEADER": { + }, + "ATTRIBUTE": { + "fields":{ + "Attr_Header": { + "offset": 0, + "type": { + "kind": "struct", + "name": "mft!ATTR_HEADER" + } + }, + "Resident_Header": { + "offset": 16, + "type": { + "kind": "struct", + "name": "mft!RESIDENT_HEADER" + } + }, + "Attr_Data": { + "offset": 24, + "type": { + "kind": "struct", + "name": "mft!ATTR_HEADER" + } + } + }, + "kind": "struct", + "size": 96 + }, + "ATTR_HEADER": { "fields": { "AttrType": { "offset": 0, From 1f7acf2779047f9b5bf39e2de2ecf214c0c678a7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 21:02:20 +0000 Subject: [PATCH 036/158] Documentation: Update sphinx requirement to 4.0.0 --- doc/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index d646d22ce..93d6ea70a 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,4 @@ # These packages are required for building the documentation. -sphinx>=1.8.2 +sphinx>=4.0.0 sphinx_autodoc_typehints>=1.4.0 -sphinx-rtd-theme>=0.4.3 \ No newline at end of file +sphinx-rtd-theme>=0.4.3 From c93e20ab36ae6a988d9690d9ac44b165368b76dd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 21:14:52 +0000 Subject: [PATCH 037/158] Plugins: linux.kmsg update documentation and reformat --- volatility3/framework/plugins/linux/kmsg.py | 66 ++++++++++++--------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index a729c1695..8f4540766 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -4,9 +4,9 @@ import logging from abc import ABC, abstractmethod from enum import Enum -from typing import List, Iterator, Tuple, Generator +from typing import Generator, Iterator, List, Tuple -from volatility3.framework import renderers, interfaces, constants, contexts, class_subclasses +from volatility3.framework import class_subclasses, constants, contexts, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -15,39 +15,39 @@ vollog = logging.getLogger(__name__) class DescStateEnum(Enum): - desc_miss = -1 # ID mismatch (pseudo state) - desc_reserved = 0x0 # reserved, in use by writer - desc_committed = 0x1 # committed by writer, could get reopened - desc_finalized = 0x2 # committed, no further modification allowed - desc_reusable = 0x3 # free, not yet used by any writer + desc_miss = -1 # ID mismatch (pseudo state) + desc_reserved = 0x0 # reserved, in use by writer + desc_committed = 0x1 # committed by writer, could get reopened + desc_finalized = 0x2 # committed, no further modification allowed + desc_reusable = 0x3 # free, not yet used by any writer class ABCKmsg(ABC): """Kernel log buffer reader""" LEVELS = ( - "emerg", # system is unusable - "alert", # action must be taken immediately - "crit", # critical conditions - "err", # error conditions - "warn", # warning conditions - "notice", # normal but significant condition - "info", # informational - "debug", # debug-level messages + "emerg", # system is unusable + "alert", # action must be taken immediately + "crit", # critical conditions + "err", # error conditions + "warn", # warning conditions + "notice", # normal but significant condition + "info", # informational + "debug", # debug-level messages ) FACILITIES = ( - "kern", # kernel messages - "user", # random user-level messages - "mail", # mail system - "daemon", # system daemons - "auth", # security/authorization messages - "syslog", # messages generated internally by syslogd - "lpr", # line printer subsystem - "news", # network news subsystem - "uucp", # UUCP subsystem - "cron", # clock daemon + "kern", # kernel messages + "user", # random user-level messages + "mail", # mail system + "daemon", # system daemons + "auth", # security/authorization messages + "syslog", # messages generated internally by syslogd + "lpr", # line printer subsystem + "news", # network news subsystem + "uucp", # UUCP subsystem + "cron", # clock daemon "authpriv", # security/authorization messages (private) - "ftp" # FTP daemon + "ftp" # FTP daemon ) def __init__( @@ -247,12 +247,20 @@ class KmsgFiveTen(ABCKmsg): The data block ring 'text_data_ring' contains the records' text strings. A pointer to the high level structure is kept in the prb pointer which is initialized to a static ringbuffer. + + .. code-block:: c + static struct printk_ringbuffer *prb = &printk_rb_static; + In SMP systems with more than 64 CPUs this ringbuffer size is dynamically allocated according the number of CPUs based on the value of CONFIG_LOG_CPU_MAX_BUF_SHIFT. The prb pointer is updated consequently to this dynamic ringbuffer in setup_log_buf(). + + .. code-block:: c + prb = &printk_rb_dynamic; + Behind scenes, log_buf is still used as external buffer. When the static printk_ringbuffer struct is initialized, _DEFINE_PRINTKRB sets text_data_ring.data pointer to the address in log_buf which points to @@ -262,12 +270,14 @@ class KmsgFiveTen(ABCKmsg): buffer via the prb_init function. In that case, the original external static buffer in __log_buf and printk_rb_static are unused. - ... + + .. code-block:: c + new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN); prb_init(&printk_rb_dynamic, new_log_buf, ...); log_buf = new_log_buf; prb = &printk_rb_dynamic; - ... + See printk.c and printk_ringbuffer.c in kernel/printk/ folder for more details. """ From 8791631db5168b5f85ee72304f7d8906f915586b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 21:40:25 +0000 Subject: [PATCH 038/158] Documentation: More minor fixes --- doc/source/conf.py | 9 +++++++++ doc/source/volshell.rst | 20 ++++++++++---------- volatility3/framework/objects/__init__.py | 4 ++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index eab5c2c94..54c5da5ed 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -84,6 +84,15 @@ def setup(app): for line in submodule_lines: contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins')) + # Clear up the framework.plugins page + with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "rb") as contents: + real_lines = contents.readlines() + + with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "wb") as contents: + for line in real_lines: + if b'volatility3.framework.plugins.' not in line: + contents.write(line) + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 1b6846e6b..3d4cad890 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -22,7 +22,7 @@ be run. When volshell starts, it will show the version of volshell, a brief message indicating how to get more help, the current operating system mode for volshell, and the current layer available for use. -.. code-block:: python +:: Volshell (Volatility 3 Framework) 1.0.1 Readline imported successfully PDB scanning finished @@ -53,7 +53,7 @@ run our examples against. We'll start by creating a process variable, and putting the first result from `ps()` in it. Since the shell is a python environment, we can do the following: -.. code-block:: python +:: (primary) >>> proc = ps()[0] (primary) >>> proc @@ -68,7 +68,7 @@ built-in mechanism for providing more information about a structure, called `dis either a type name (which if not prefixed with symbol table name, will use the kernel symbol table identified by the automagic). -.. code-block:: python +:: (primary) >>> dt('_EPROCESS') nt_symbols1!_EPROCESS (2624 bytes) @@ -80,7 +80,7 @@ automagic). It can also be provided with an object and will interpret the data for each in the process: -.. code-block:: python +:: (primary) >>> dt(proc) nt_symbols1!_EPROCESS (2624 bytes) @@ -92,7 +92,7 @@ It can also be provided with an object and will interpret the data for each in t These values can be accessed directory as attributes -.. code-block:: python +:: (primary) >>> proc.UniqueProcessId 356 @@ -100,7 +100,7 @@ These values can be accessed directory as attributes Pointer structures contain the value they point to, but attributes accessed are forwarded to the object they point to. This means that pointers do not need to be explicitly dereferenced to access underling objects. -.. code-block:: python +:: (primary) >>> proc.Pcb.DirectoryTableBase 4355817472 @@ -112,7 +112,7 @@ It's possible to run any plugin by importing it appropriately and passing it to method. In the following example we'll provide no additional parameters. Volatility will show us which parameters were required: -.. code-block:: python +:: (primary) >>> from volatility3.plugins.windows import pslist (primary) >>> display_plugin_output(pslist.PsList) @@ -124,14 +124,14 @@ was fulfilled. We can see all the options that the plugin can accept by access the `get_requirements()` method of the plugin. This is a classmethod, so can be called on an uninstantiated copy of the plugin. -.. code-block:: python +:: (primary) >>> pslist.PsList.get_requirements() [, , , , ] We can provide arguments via the `dpo` method call: -.. code-block:: python +:: (primary) >>> display_plugin_output(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) @@ -149,7 +149,7 @@ by the `dpo` method is always `context`. Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by using the `generate_treegrid` or `gt` command. -.. code-block:: python +:: (primary) >>> treegrid = gt(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) (primary) >>> treegrid.populate() diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 6ee24407f..b5a7db286 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -6,9 +6,9 @@ import collections import collections.abc import logging import struct -from typing import Any, ClassVar, Dict, List, Iterable, Optional, Tuple, Type, Union as TUnion, overload +from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.objects import templates, utility vollog = logging.getLogger(__name__) From 4aaba89d024a06a00760978b2f8bb7f099087c72 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 21:47:37 +0000 Subject: [PATCH 039/158] Unity timeliner output for mftscan --- .../framework/plugins/windows/mftscan.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 3d8d96221..03f269735 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -78,8 +78,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( - attribute_object).relative_child_offset("Attr_Data") + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type(attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -130,13 +129,18 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pass def generate_timeline(self): + print("MFT Timeline") for row in self._generator(): - if row[-1] != 'N/A': - filename = row[-1] - yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) - yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) - yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) - yield (f'File {filename} accessed', timeliner.TimeLinerType.ACCESSED, row[10]) + _depth, row_data = row + + # Only Output FN Records + if row_data[6] == 'FILE_NAME': + filename = row_data[-1] + description = f"MFT FILE_NAME entry for {filename}" + yield (description, timeliner.TimeLinerType.CREATED, row_data[7]) + yield (description, timeliner.TimeLinerType.MODIFIED, row_data[8]) + yield (description, timeliner.TimeLinerType.CHANGED, row_data[9]) + yield (description, timeliner.TimeLinerType.ACCESSED, row_data[10]) def run(self): return renderers.TreeGrid([ From a9e5589260710a22f47ea2612f5a27ae8ffe87fc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:02:02 +0000 Subject: [PATCH 040/158] Documentation: Add summary table for linux/mac ISF creation --- doc/source/symbol-tables.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 31329b9eb..245dd9c67 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -76,3 +76,21 @@ The banners available for volatility to use can be found using the `isfinfo` plu long time to run depending on the number of JSON files available. This will list all the JSON (ISF) files that volatility3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON file, the banners must match exactly (down to the compilation date). + +.. note:: + + Steps for constructing a new kernel ISF JSON file: + + * Run the `banners` plugin on the image to determine the necessary kernel + * Locate a copy of the debug kernel that matches the identified banner + + * Clone or update the dwarf2json repo: :code:`git clone https://github.com/volatilityfoundation/dwarf2json` + * Run :code:`go build` in the directory if the source has changed + + * Run :code:`dwarf2json linux --elf [path to debug kernel] > [kernel name].json` + + * For Mac change `linux` to `mac` + + * Copy the `.json` file to the symbols directory into `[symbols directory]/linux` + + * For Mac change `linux` to `mac` \ No newline at end of file From 32abab8733d697ed86e3a6208d9c03ce08233117 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 22:38:05 +0000 Subject: [PATCH 041/158] Remove debug print from mftscan --- volatility3/framework/plugins/windows/mftscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 03f269735..7cc57f111 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -129,7 +129,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pass def generate_timeline(self): - print("MFT Timeline") for row in self._generator(): _depth, row_data = row From dab746aff07bc7afe021bd620822640d4f026ce6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:44:24 +0000 Subject: [PATCH 042/158] Plugins: yara python module check improvement Fixes #616 --- volatility3/framework/plugins/yarascan.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index e51669b2c..0ef55ff4b 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -15,8 +15,11 @@ vollog = logging.getLogger(__name__) try: import yara + + if tuple([int(x) for x in yara.__version__.split('.')]) < (3, 8): + raise ImportError except ImportError: - vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") + vollog.info("Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available") raise From d9ba3e6bd9b71f0c9bc1973ada12c7bab104e729 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:50:21 +0000 Subject: [PATCH 043/158] Plugins: Timeliner improve support for body files --- volatility3/framework/plugins/timeliner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 6bf592504..0ae87e274 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -12,7 +12,7 @@ import traceback from typing import Generator, Iterable, List, Optional, Tuple, Type from volatility3 import framework -from volatility3.framework import renderers, automagic, interfaces, plugins, exceptions +from volatility3.framework import automagic, exceptions, interfaces, plugins, renderers from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -145,7 +145,7 @@ class Timeliner(interfaces.plugins.PluginInterface): # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime if self._any_time_present(times): - fp.write("|{} - {}||||||{}|{}|{}|{}\n".format( + fp.write("|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format( plugin_name, self._sanitize_body_format(item), self._text_format(times.get(TimeLinerType.ACCESSED, "")), self._text_format(times.get(TimeLinerType.MODIFIED, "")), @@ -202,7 +202,7 @@ class Timeliner(interfaces.plugins.PluginInterface): if isinstance(plugin, TimeLinerInterface): if not len(filter_list) or any( - [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]): + [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]): plugins_to_run.append(plugin) except exceptions.UnsatisfiedException as excp: # Remove the failed plugin from the list and continue From 742d46786bc0b48781343584204bd51a6c316da9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 15 Jan 2022 22:56:36 +0000 Subject: [PATCH 044/158] Plugins: Don't forget missing values --- volatility3/framework/plugins/timeliner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 0ae87e274..faa99ff67 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -164,7 +164,7 @@ class Timeliner(interfaces.plugins.PluginInterface): def _text_format(self, value): """Formats a value as text, in case it is an AbsentValue""" if isinstance(value, interfaces.renderers.BaseAbsentValue): - return "" + return "0" if isinstance(value, datetime.datetime): return int(value.timestamp()) return value From f31bc853c4d7eb05041c854f26704dea21656d0d Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 23:30:39 +0000 Subject: [PATCH 045/158] remove MFTIntermedSymbols --- .../framework/plugins/windows/mftscan.py | 43 +++++++++++-------- volatility3/framework/symbols/windows/mft.py | 15 ------- 2 files changed, 25 insertions(+), 33 deletions(-) delete mode 100644 volatility3/framework/symbols/windows/mft.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 7cc57f111..6487ffe97 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,12 @@ import datetime import logging -from typing import Dict - from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mft from volatility3.plugins import timeliner, yarascan @@ -40,7 +39,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) # Read in the Symbol File - symbol_table = MFTIntermedSymbols.create(self.context, self.config_path, "windows", "mft") + symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, + config_path = self.config_path, + sub_path = "windows", + filename = "mft", + class_types = { + 'FILE_NAME_ENTRY': mft.MFTFileName, + 'MFT_ENTRY': mft.MFTEntry + }) # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" @@ -57,28 +63,25 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields - for offset, rule_name, name, value in layer.scan(context = self.context, - scanner = yarascan.YaraScanner(rules = rules)): + for offset, _rule_name, _name, _value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): try: mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) + # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while True: - attr_header = self.context.object(header_object, - offset = offset + attr_base_offset, - layer_name = layer.name) - + while attr_header.AttrType in attr_types.choices.values(): vollog.debug(f"Attr Type: {attr_header.AttrType}") - # If this is not a valid type then exit the loop - if attr_header.AttrType not in attr_types.choices.values(): - break - # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type(attribute_object).relative_child_offset("Attr_Data") + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( + attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -124,9 +127,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length + # Get the next attribute + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) - except exceptions.PagedInvalidAddressException: - pass + except Exception as err: + vollog.debug(f'Error Parsing MFT Record: {err}') def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/symbols/windows/mft.py b/volatility3/framework/symbols/windows/mft.py deleted file mode 100644 index 921d75cd8..000000000 --- a/volatility3/framework/symbols/windows/mft.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows.extensions import mft - - -class MFTIntermedSymbols(intermed.IntermediateSymbolTable): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.set_type_class('FILE_NAME_ENTRY', mft.MFTFileName) - self.set_type_class('MFT_ENTRY', mft.MFTEntry) From a31f846b14ffb4ffe529f1aae93256ba2c8e2d1e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Jan 2022 00:47:40 +0000 Subject: [PATCH 046/158] Documentation: Reorganize and consolidate pages --- doc/source/development.rst | 8 ++++++++ doc/source/index.rst | 6 ++---- 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 doc/source/development.rst diff --git a/doc/source/development.rst b/doc/source/development.rst new file mode 100644 index 000000000..ade068322 --- /dev/null +++ b/doc/source/development.rst @@ -0,0 +1,8 @@ +Writing Plugins +=============== + +.. toctree:: + + simple-plugin + complex-plugin + using-as-a-library diff --git a/doc/source/index.rst b/doc/source/index.rst index 0dc3b5025..3b5a5d2a8 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -12,11 +12,9 @@ Here are some guidelines for using Volatility 3 effectively: .. toctree:: basics - simple-plugin - vol2to3 - complex-plugin - using-as-a-library + development symbol-tables + vol2to3 volshell glossary From e3a7ac566840ee052ffbefe3fb370e4142704706 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 16 Jan 2022 02:20:55 +0000 Subject: [PATCH 047/158] Use lookups on mft enums instead of choices --- .../framework/plugins/windows/mftscan.py | 39 +++++++++---------- .../framework/symbols/windows/mft.json | 22 +++++------ 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 6487ffe97..16f0c9c95 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -55,12 +55,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" - # Get the Enums - attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") - namespace_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") - mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") - permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + - "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan(context = self.context, @@ -74,23 +68,26 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) + # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr_header.AttrType in attr_types.choices.values(): - vollog.debug(f"Attr Type: {attr_header.AttrType}") + + while attr_header.AttrType.is_valid_choice: + vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") # Offset past the headers to the attribute data attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir - if mft_record.Flags in mft_flags.choices.values(): - mft_flag = mft_flags.lookup(mft_record.Flags) - else: + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + mft_flag = mft_record.Flags.lookup() + except ValueError: mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): + if attr_header.AttrType.lookup() == 'STANDARD_INFORMATION': attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name) yield 0, ( @@ -100,7 +97,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_record.LinkCount, mft_flag, renderers.NotApplicableValue(), - attr_types.lookup(attr_header.AttrType), + attr_header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -109,17 +106,19 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): + if attr_header.AttrType.lookup() == 'FILE_NAME': attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name) file_name = attr_data.get_full_name() - if attr_data.Flags in permission_flags.choices.values(): - permissions = permission_flags.lookup(attr_data.Flags) - else: + + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + permissions = attr_data.Flags.lookup() + except ValueError: permissions = hex(attr_data.Flags) yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, - attr_types.lookup(attr_header.AttrType), + attr_header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -132,8 +131,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) - except Exception as err: - vollog.debug(f'Error Parsing MFT Record: {err}') + except exceptions.PagedInvalidAddressException: + pass def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index 2470dcbd5..b71be6444 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -181,8 +181,8 @@ "Flags": { "offset": 22, "type":{ - "kind": "base", - "name": "unsigned short" + "kind": "enum", + "name": "MFTFlagsEnum" } }, "RealSize": { @@ -256,8 +256,8 @@ "AttrType": { "offset": 0, "type": { - "kind": "base", - "name": "unsigned int" + "kind": "enum", + "name": "AttrTypeEnum" } },"Length": { "offset": 4, @@ -289,9 +289,9 @@ "Flags": { "offset": 12, "type": { - "kind": "base", - "name": "unsigned short" - } + "kind": "enum", + "name": "MFTFlagsEnum" + } }, "AttributeID": { "offset": 14, @@ -361,8 +361,8 @@ "flags": { "offset": 32, "type": { - "kind": "base", - "name": "unsigned short" + "kind": "enum", + "name": "PermissionFlagEnum" } } }, @@ -423,8 +423,8 @@ "Flags": { "offset": 56, "type": { - "kind": "base", - "name": "unsigned int" + "kind": "enum", + "name": "PermissionFlagEnum" } }, "ReparseValue": { From 57c9470f66f25726eaa2e6c5f87ac8a29e8dab68 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Jan 2022 22:09:02 +0000 Subject: [PATCH 048/158] Documentation: Fix building from different directories --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 54c5da5ed..6030c6308 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -85,10 +85,10 @@ def setup(app): contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins')) # Clear up the framework.plugins page - with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "rb") as contents: + with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "rb") as contents: real_lines = contents.readlines() - with open(os.path.join('source', 'volatility3.framework.plugins.rst'), "wb") as contents: + with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "wb") as contents: for line in real_lines: if b'volatility3.framework.plugins.' not in line: contents.write(line) From 64acff1b59d7f85a1a187bf45875a8790029407e Mon Sep 17 00:00:00 2001 From: "Nick L. Petroni, Jr" Date: Sun, 16 Jan 2022 15:34:03 -0500 Subject: [PATCH 049/158] use doc/requirements.txt when building with readthedocs --- .readthedocs.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 764bb5a1a..4d21d9b40 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -16,7 +16,4 @@ formats: all python: version: 3.7 install: - - method: pip - path: . - extra_requirements: - - doc + - requirements: doc/requirements.txt From fa2a608cd296e925d9e9847c9ac55cff34bb5f6a Mon Sep 17 00:00:00 2001 From: "Nick L. Petroni, Jr" Date: Sun, 16 Jan 2022 17:32:02 -0500 Subject: [PATCH 050/158] update doc copyright --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 6030c6308..731a73d56 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # # @@ -135,7 +135,7 @@ master_doc = 'index' # General information about the project. project = 'Volatility 3' -copyright = '2012-2019, Volatility Foundation' +copyright = '2012-2022, Volatility Foundation' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 79a549f21170276cf1634b3599b89f5d8d2485e1 Mon Sep 17 00:00:00 2001 From: "Nick L. Petroni, Jr" Date: Sun, 16 Jan 2022 15:34:03 -0500 Subject: [PATCH 051/158] use doc/requirements.txt when building with readthedocs --- .readthedocs.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 764bb5a1a..4d21d9b40 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -16,7 +16,4 @@ formats: all python: version: 3.7 install: - - method: pip - path: . - extra_requirements: - - doc + - requirements: doc/requirements.txt From 0f23089cb483484ce1eed54ca87d5e78825d02e3 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Mon, 17 Jan 2022 00:28:59 +0000 Subject: [PATCH 052/158] Create Sessions Plugin --- .../framework/plugins/windows/sessions.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 volatility3/framework/plugins/windows/sessions.py diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py new file mode 100644 index 000000000..cd02668ae --- /dev/null +++ b/volatility3/framework/plugins/windows/sessions.py @@ -0,0 +1,96 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import datetime +import logging + +from volatility3.framework import renderers, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + + +class Sessions(interfaces.plugins.PluginInterface): + """lists Processes with Session information extracted from Environmental Variables""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True) + ] + + def _generator(self, procs): + + # Collect all the values as we will want to group them later + sessions = {} + + for proc in procs: + + session_id = proc.get_session_id() + + # Detect RDP, Console or set default value + session_type = renderers.NotAvailableValue() + + # Construct Username from Process Env + user_domain = '' + user_name = '' + + for var, val in proc.environment_variables(): + if var.lower() == 'username': + user_name = val + elif var.lower() == 'userdomain': + user_domain = val + if var.lower() == 'sessionname': + session_type = val + + # Concat Domain and User + full_user = f'{user_domain}/{user_name}' + if full_user == '/': + full_user = renderers.NotAvailableValue() + + # Collect all the values in to a row we can yield after sorting. + row = { + "session_id": session_id, + "process_id": proc.UniqueProcessId, + "process_name": utility.array_to_string(proc.ImageFileName), + "user_name": full_user, + "process_start": proc.get_create_time(), + "session_type": session_type + } + + # Add row to correct session so we can sort it later + if session_id in sessions: + sessions[session_id].append(row) + else: + sessions[session_id] = [row] + + # Group and yield each row + for rows in sessions.values(): + for row in rows: + yield 0, (row.get('session_id'), row.get('session_type'), row.get('process_id'), + row.get('process_name'), row.get('user_name'), row.get('process_start')) + + def run(self): + + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + + return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), + ("User Name", str), ("Create Time", datetime.datetime)], + self._generator( + pslist.PsList.list_processes(self.context, + self.config['primary'], + self.config['nt_symbols'], + filter_func = filter_func))) From be0dd49d314bc6d4b2aa907928f0fb7123cb5f43 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Tue, 18 Jan 2022 19:34:06 +0000 Subject: [PATCH 053/158] Sessions Plugin use ModuleRequirement --- .../framework/plugins/windows/sessions.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py index cd02668ae..66a54c3f0 100644 --- a/volatility3/framework/plugins/windows/sessions.py +++ b/volatility3/framework/plugins/windows/sessions.py @@ -21,10 +21,9 @@ class Sessions(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', element_type = int, @@ -32,12 +31,17 @@ class Sessions(interfaces.plugins.PluginInterface): optional = True) ] - def _generator(self, procs): + def _generator(self): + kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) # Collect all the values as we will want to group them later sessions = {} - for proc in procs: + for proc in pslist.PsList.list_processes(self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func = filter_func): session_id = proc.get_session_id() @@ -56,7 +60,7 @@ class Sessions(interfaces.plugins.PluginInterface): if var.lower() == 'sessionname': session_type = val - # Concat Domain and User + # Concat Domain and User full_user = f'{user_domain}/{user_name}' if full_user == '/': full_user = renderers.NotAvailableValue() @@ -85,12 +89,5 @@ class Sessions(interfaces.plugins.PluginInterface): def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) - return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), - ("User Name", str), ("Create Time", datetime.datetime)], - self._generator( - pslist.PsList.list_processes(self.context, - self.config['primary'], - self.config['nt_symbols'], - filter_func = filter_func))) + ("User Name", str), ("Create Time", datetime.datetime)], self._generator()) From ac7cecf231e98c7f104d4c2b8fb569cafb977409 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Tue, 18 Jan 2022 20:42:38 +0000 Subject: [PATCH 054/158] add timeliner output to windows.sessions --- volatility3/framework/plugins/windows/sessions.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py index 66a54c3f0..6745e95ec 100644 --- a/volatility3/framework/plugins/windows/sessions.py +++ b/volatility3/framework/plugins/windows/sessions.py @@ -9,11 +9,12 @@ from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.plugins.windows import pslist +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class Sessions(interfaces.plugins.PluginInterface): +class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """lists Processes with Session information extracted from Environmental Variables""" _required_framework_version = (2, 0, 0) @@ -87,6 +88,15 @@ class Sessions(interfaces.plugins.PluginInterface): yield 0, (row.get('session_id'), row.get('session_type'), row.get('process_id'), row.get('process_name'), row.get('user_name'), row.get('process_start')) + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + # Only add to timeline if we have the username + # Without the user context PSList output is identical + if isinstance(row_data[4], str): + description = f"Process: {row_data[2]} {row_data[3]} started by user {row_data[4]}" + yield (description, timeliner.TimeLinerType.CREATED, row_data[5]) + def run(self): return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), From 0f90f7f6ca5b3b120f3c680aa2993ec4c5246b4c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 18 Jan 2022 20:58:48 +0000 Subject: [PATCH 055/158] Plugins: Remove unused timeliner parameter --- volatility3/framework/plugins/timeliner.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 6bf592504..f7ec340ec 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -74,10 +74,6 @@ class Timeliner(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.StringRequirement(name = 'plugins', - description = "Comma separated list of plugins to run", - optional = True, - default = None), requirements.BooleanRequirement( name = 'record-config', description = "Whether to record the state of all the plugins once complete", From 7c238f93a06b496826271e61a010be9d2c4a9661 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 31 Jan 2022 20:15:39 +0900 Subject: [PATCH 056/158] Update __init__.py Correcting typos for Windows Constants --- volatility3/framework/constants/windows/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/windows/__init__.py b/volatility3/framework/constants/windows/__init__.py index 372897598..a19216605 100644 --- a/volatility3/framework/constants/windows/__init__.py +++ b/volatility3/framework/constants/windows/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""Volatility 3 Linux Constants. +"""Volatility 3 Windows Constants. Windows-specific values that aren't found in debug symbols """ From a67121f1c17fd18acd13c4f7a63463b2b9fc5c8f Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Wed, 9 Feb 2022 10:40:37 +0200 Subject: [PATCH 057/158] Added data_offset to pattern matching result, fixes pdb scanning bug --- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 4c3788a56..ce492831e 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -357,4 +357,4 @@ class PdbSignatureScanner(interfaces.layers.ScannerInterface): guid = (16 * '{:02X}').format(g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, ga, gb, gc, gd, ge, gf) if match.start(0) < self.chunk_size: - yield (guid, a, pdb_name, match.start(0)) + yield (guid, a, pdb_name, data_offset + match.start(0)) From ce166b92637edb78abcc7e87139449019c6abaf6 Mon Sep 17 00:00:00 2001 From: trashcan122 Date: Wed, 9 Feb 2022 16:30:40 +0200 Subject: [PATCH 058/158] fix pdb mz parsing --- volatility3/framework/symbols/windows/pdbutil.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index ce492831e..cc7b22e03 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -131,8 +131,14 @@ class PDBUtility(interfaces.configuration.VersionableInterface): # Check it is actually the MZ header if mz_sig != b"MZ": return None - - nt_header_start = ord(layer.read(offset + 0x3C, 1)) + + nt_header_start = struct.unpack(" Date: Wed, 9 Feb 2022 20:07:13 +0000 Subject: [PATCH 059/158] Core: Slight speed-up for all single struct.unpack calls --- volatility3/framework/symbols/windows/extensions/pool.py | 4 ++-- .../framework/symbols/windows/extensions/registry.py | 9 ++++++--- volatility3/framework/symbols/windows/pdbutil.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index f75c6c417..d50d6e47a 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -128,8 +128,8 @@ class POOL_HEADER(objects.StructType): # --------------- if addr - optional_headers_length < 0: continue - padding_length = struct.unpack( - "L"): raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}") - return struct.unpack(">L", data)[0] + res, = struct.unpack(">L", data) + return res if self_type == RegValueTypes.REG_QWORD: if len(data) != struct.calcsize(" Date: Wed, 9 Feb 2022 10:40:37 +0200 Subject: [PATCH 060/158] Added data_offset to pattern matching result, fixes pdb scanning bug --- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 4c3788a56..ce492831e 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -357,4 +357,4 @@ class PdbSignatureScanner(interfaces.layers.ScannerInterface): guid = (16 * '{:02X}').format(g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, ga, gb, gc, gd, ge, gf) if match.start(0) < self.chunk_size: - yield (guid, a, pdb_name, match.start(0)) + yield (guid, a, pdb_name, data_offset + match.start(0)) From de87fd0b226cfca60395f9f2e67248d7e343e770 Mon Sep 17 00:00:00 2001 From: trashcan122 Date: Wed, 9 Feb 2022 16:30:40 +0200 Subject: [PATCH 061/158] fix pdb mz parsing --- volatility3/framework/symbols/windows/pdbutil.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index ce492831e..cc7b22e03 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -131,8 +131,14 @@ class PDBUtility(interfaces.configuration.VersionableInterface): # Check it is actually the MZ header if mz_sig != b"MZ": return None - - nt_header_start = ord(layer.read(offset + 0x3C, 1)) + + nt_header_start = struct.unpack(" Date: Wed, 9 Feb 2022 20:18:04 +0000 Subject: [PATCH 062/158] Core: Prepare 2.0.1 release --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 23598837b..4ce2dbe2d 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 05b4eb22fad08f9b7db0192b193f4714e5f75c63 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 9 Feb 2022 20:19:17 +0000 Subject: [PATCH 063/158] Core: Bump the development version to 2.0.2 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 82ebd4936..665e62d30 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From edbbc0159fa7beeea60c72a6e32758ec79d9de17 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 10 Feb 2022 10:31:18 +0200 Subject: [PATCH 064/158] bug fix --- volatility3/framework/interfaces/objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 811327094..e589abd15 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -115,7 +115,7 @@ class ObjectInterface(metaclass = abc.ABCMeta): mask = context.layers[object_info.layer_name].address_mask normalized_offset = object_info.offset & mask - self._vol = collections.ChainMap({}, object_info, {'type_name': type_name, 'offset': normalized_offset}, kwargs) + self._vol = collections.ChainMap({}, {'type_name': type_name, 'offset': normalized_offset}, object_info, kwargs) self._context = context def __getattr__(self, attr: str) -> Any: From cf11de174ab9ec019f09bf2bdc851619cf1ec509 Mon Sep 17 00:00:00 2001 From: pudii Date: Fri, 11 Feb 2022 18:19:53 +0100 Subject: [PATCH 065/158] Implement LDRmodules plugin --- .../framework/plugins/windows/ldrmodules.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 volatility3/framework/plugins/windows/ldrmodules.py diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py new file mode 100644 index 000000000..77678a4c0 --- /dev/null +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -0,0 +1,97 @@ +from volatility3.framework import interfaces, constants +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, vadinfo + +class LdrModules(interfaces.plugins.PluginInterface): + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), + requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), + requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True), + ] + + def _generator(self, procs): + + pe_table_name = intermed.IntermediateSymbolTable.create(self.context, + self.config_path, + "windows", + "pe", + class_types = pe.class_types) + + def filter_function(x: interfaces.objects.ObjectInterface) -> bool: + try: + return not (x.get_private_memory() == 0 and x.ControlArea) + except AttributeError: + return False + + filter_func = filter_function + + for proc in procs: + proc_layer_name = proc.add_process_layer() + + load_order_mod = dict((mod.DllBase, mod) + for mod in proc.load_order_modules()) + init_order_mod = dict((mod.DllBase, mod) + for mod in proc.init_order_modules()) + mem_order_mod = dict((mod.DllBase, mod) + for mod in proc.mem_order_modules()) + + mapped_files = {} + for vad in vadinfo.VadInfo.list_vads(proc, filter_func = filter_func): + dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset = vad.get_start(), + layer_name = proc_layer_name) + try: + if dos_header.e_magic != 0x5A4D: + continue + except exceptions.PagedInvalidAddressException: + continue + + mapped_files[int(vad.get_start())] = str(vad.get_file_name() or '') + + for base in mapped_files.keys(): + # Does the base address exist in the PEB DLL lists? + load_mod = load_order_mod.get(base, None) + init_mod = init_order_mod.get(base, None) + mem_mod = mem_order_mod.get(base, None) + + yield (0, [int(proc.UniqueProcessId), + str(proc.ImageFileName.cast("string", + max_length = proc.ImageFileName.vol.count, + errors = 'replace')), + format_hints.Hex(base), + str(load_mod != None), + str(init_mod != None), + str(mem_mod != None), + str(mapped_files[base])]) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] + + return renderers.TreeGrid([("Pid", int), + ("Process", str), + ("Base", format_hints.Hex), + ("InLoad", str), + ("InInit", str), + ("InMem", str), + ("MappedPath", str)], + self._generator( + pslist.PsList.list_processes(context = self.context, + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, + filter_func = filter_func))) \ No newline at end of file From 9cabf5362b66266a34f861af5e89d0315fe92124 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Feb 2022 00:19:27 +0000 Subject: [PATCH 066/158] Timeliner: Write out directly to the body file Since the body file doesn't need sorting, we can output it immediately, and this also means that partial results can be recorded even in the run is terminted before it compeletes. Goes someway to improving #646 --- volatility3/framework/plugins/timeliner.py | 33 +++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 01f775eb3..8785f62e1 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -106,6 +106,15 @@ class Timeliner(interfaces.plugins.PluginInterface): row from each plugin.""" # Generate the results for each plugin data = [] + + # Open the bodyfile now, so we can start outputting to it immediately + if self.config.get('create-bodyfile', True): + file_data = self.open("volatility.body") + fp = io.TextIOWrapper(file_data, write_through = True) + else: + file_data = None + fp = None + for plugin in runable_plugins: plugin_name = plugin.__class__.__name__ self._progress_callback((runable_plugins.index(plugin) * 100) // len(runable_plugins), @@ -126,17 +135,9 @@ class Timeliner(interfaces.plugins.PluginInterface): times.get(TimeLinerType.ACCESSED, renderers.NotApplicableValue()), times.get(TimeLinerType.CHANGED, renderers.NotApplicableValue()) ])) - except Exception: - vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}") - vollog.log(logging.DEBUG, traceback.format_exc()) - for data_item in sorted(data, key = self._sort_function): - yield data_item - # Write out a body file if necessary - if self.config.get('create-bodyfile', True): - with self.open("volatility.body") as file_data: - with io.TextIOWrapper(file_data, write_through = True) as fp: - for (plugin_name, item) in self.timeline: + # Write each entry because the body file doesn't need to be sorted + if fp: times = self.timeline[(plugin_name, item)] # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime @@ -147,6 +148,18 @@ class Timeliner(interfaces.plugins.PluginInterface): self._text_format(times.get(TimeLinerType.MODIFIED, "")), self._text_format(times.get(TimeLinerType.CHANGED, "")), self._text_format(times.get(TimeLinerType.CREATED, "")))) + except Exception: + vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}") + vollog.log(logging.DEBUG, traceback.format_exc()) + + for data_item in sorted(data, key = self._sort_function): + yield data_item + + # Write out a body file if necessary + if self.config.get('create-bodyfile', True): + if fp: + fp.close() + file_data.close() def _sanitize_body_format(self, value): return value.replace("|", "_") From 12c3f340370f5e4dd1cb34d6803154c23164d234 Mon Sep 17 00:00:00 2001 From: pudii Date: Sun, 13 Feb 2022 18:29:35 +0100 Subject: [PATCH 067/158] Add comments and fix minor code issues --- .../framework/plugins/windows/ldrmodules.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 77678a4c0..42eeacd4d 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -15,7 +15,6 @@ class LdrModules(interfaces.plugins.PluginInterface): def get_requirements(cls): return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', @@ -43,6 +42,7 @@ class LdrModules(interfaces.plugins.PluginInterface): for proc in procs: proc_layer_name = proc.add_process_layer() + # Build dictionaries from different module lists, where the DllBase address is the key and value is the module object load_order_mod = dict((mod.DllBase, mod) for mod in proc.load_order_modules()) init_order_mod = dict((mod.DllBase, mod) @@ -50,18 +50,20 @@ class LdrModules(interfaces.plugins.PluginInterface): mem_order_mod = dict((mod.DllBase, mod) for mod in proc.mem_order_modules()) + # Build dictionary of mapped files, where the VAD start address is the key and value is the file name of the mapped file mapped_files = {} for vad in vadinfo.VadInfo.list_vads(proc, filter_func = filter_func): dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", offset = vad.get_start(), layer_name = proc_layer_name) try: + # Filter out VADs that do not start with a MZ header if dos_header.e_magic != 0x5A4D: continue except exceptions.PagedInvalidAddressException: continue - mapped_files[int(vad.get_start())] = str(vad.get_file_name() or '') + mapped_files[vad.get_start()] = vad.get_file_name() for base in mapped_files.keys(): # Does the base address exist in the PEB DLL lists? @@ -74,10 +76,10 @@ class LdrModules(interfaces.plugins.PluginInterface): max_length = proc.ImageFileName.vol.count, errors = 'replace')), format_hints.Hex(base), - str(load_mod != None), - str(init_mod != None), - str(mem_mod != None), - str(mapped_files[base])]) + load_mod != None, + init_mod != None, + mem_mod != None, + mapped_files[base]]) def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) @@ -86,12 +88,12 @@ class LdrModules(interfaces.plugins.PluginInterface): return renderers.TreeGrid([("Pid", int), ("Process", str), ("Base", format_hints.Hex), - ("InLoad", str), - ("InInit", str), - ("InMem", str), + ("InLoad", bool), + ("InInit", bool), + ("InMem", bool), ("MappedPath", str)], self._generator( pslist.PsList.list_processes(context = self.context, layer_name = kernel.layer_name, symbol_table = kernel.symbol_table_name, - filter_func = filter_func))) \ No newline at end of file + filter_func = filter_func))) From 22a7328e4cba83bbb6965d9705b22a1b5755d626 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Feb 2022 20:45:42 +0000 Subject: [PATCH 068/158] Windows: Prevent infinite loop in mftscan --- volatility3/framework/plugins/windows/mftscan.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 16f0c9c95..654e26db7 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,11 @@ import datetime import logging -from volatility3.framework import constants, renderers, interfaces +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import mft - from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) @@ -55,7 +53,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" - # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): @@ -68,10 +65,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) - # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - + while attr_header.AttrType.is_valid_choice: vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") @@ -124,6 +120,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): conversion.wintime_to_datetime(attr_data.UpdatedTime), conversion.wintime_to_datetime(attr_data.AccessedTime), file_name) + # If there's no advancement the loop will never end, so break it now + if attr_header.Length == 0: + break + # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length # Get the next attribute From 094a3c0a474dedc13f17314ef41b083ef0dbbeb2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 17 Feb 2022 00:21:59 +0000 Subject: [PATCH 069/158] Volshell: Update to use KernelRequirement --- doc/source/volshell.rst | 66 ++++++++++++++-------------- volatility3/cli/__init__.py | 4 +- volatility3/cli/volshell/__init__.py | 3 +- volatility3/cli/volshell/generic.py | 29 ++++++++---- volatility3/cli/volshell/linux.py | 26 ++++++++--- volatility3/cli/volshell/mac.py | 28 +++++++++--- volatility3/cli/volshell/windows.py | 30 ++++++++++--- 7 files changed, 123 insertions(+), 63 deletions(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 3d4cad890..1e51f90ba 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -24,13 +24,14 @@ operating system mode for volshell, and the current layer available for use. :: - Volshell (Volatility 3 Framework) 1.0.1 + Volshell (Volatility 3 Framework) 2.0.2 Readline imported successfully PDB scanning finished Call help() to see available functions - Volshell mode: Generic - Current Layer: primary + Volshell mode : Generic + Current Layer : primary + Current Symbol Table: None (primary) >>> @@ -55,9 +56,9 @@ python environment, we can do the following: :: - (primary) >>> proc = ps()[0] - (primary) >>> proc - + (layer_name) >>> proc = ps()[0] + (layer_name) >>> proc + When printing a volatility structure, various information is output, in this case the `type_name`, the `layer` and `offset` that it's been constructed on, and the size of the structure. @@ -70,31 +71,31 @@ automagic). :: - (primary) >>> dt('_EPROCESS') - nt_symbols1!_EPROCESS (2624 bytes) - 0x0 : Pcb nt_symbols1!_KPROCESS - 0x438 : ProcessLock nt_symbols1!_EX_PUSH_LOCK - 0x440 : UniqueProcessId nt_symbols1!pointer - 0x448 : ActiveProcessLinks nt_symbols1!_LIST_ENTRY + (layer_name) >>> dt('_EPROCESS') + symbol_table_name1!_EPROCESS (1968 bytes) + 0x0 : Pcb symbol_table_name1!_KPROCESS + 0x2d8 : ProcessLock symbol_table_name1!_EX_PUSH_LOCK + 0x2e0 : RundownProtect symbol_table_name1!_EX_RUNDOWN_REF + 0x2e8 : UniqueProcessId symbol_table_name1!pointer ... It can also be provided with an object and will interpret the data for each in the process: :: - (primary) >>> dt(proc) - nt_symbols1!_EPROCESS (2624 bytes) - 0x0 : Pcb nt_symbols1!_KPROCESS 0x8c0bccf8d040 - 0x438 : ProcessLock nt_symbols1!_EX_PUSH_LOCK 0x8c0bccf8d478 - 0x440 : UniqueProcessId nt_symbols1!pointer 356 - 0x448 : ActiveProcessLinks nt_symbols1!_LIST_ENTRY 0x8c0bccf8d488 + (layer_name) >>> dt(proc) + symbol_table_name1!_EPROCESS (1968 bytes) + 0x0 : Pcb symbol_table_name1!_KPROCESS 0xe08ff2459040 + 0x2d8 : ProcessLock symbol_table_name1!_EX_PUSH_LOCK 0xe08ff2459318 + 0x2e0 : RundownProtect symbol_table_name1!_EX_RUNDOWN_REF 0xe08ff2459320 + 0x2e8 : UniqueProcessId symbol_table_name1!pointer 4 ... These values can be accessed directory as attributes :: - (primary) >>> proc.UniqueProcessId + (layer_name) >>> proc.UniqueProcessId 356 Pointer structures contain the value they point to, but attributes accessed are forwarded to the object they point to. @@ -102,7 +103,7 @@ This means that pointers do not need to be explicitly dereferenced to access und :: - (primary) >>> proc.Pcb.DirectoryTableBase + (layer_name) >>> proc.Pcb.DirectoryTableBase 4355817472 Running plugins @@ -114,26 +115,26 @@ were required: :: - (primary) >>> from volatility3.plugins.windows import pslist - (primary) >>> display_plugin_output(pslist.PsList) - Unable to validate the plugin requirements: ['plugins.Volshell.9QZLXJKFWESI0BAP3M1U7Y5VCT468GRN.PsList.primary', 'plugins.Volshell.9QZLXJKFWESI0BAP3M1U7Y5VCT468GRN.PsList.nt_symbols'] + (layer_name) >>> from volatility3.plugins.windows import pslist + (layer_name) >>> display_plugin_output(pslist.PsList) + Unable to validate the plugin requirements: ['plugins.Volshell.VH3FSA1JBG0QP9E62Z8OT5UCIMLNYKW4.PsList.kernel'] -We can see that it's made a temporary configuration path for the plugin, and that neither `primary` nor `nt_symbols` -was fulfilled. +We can see that it's made a temporary configuration path for the plugin, and that the `kernel` requirement +was not fulfilled. We can see all the options that the plugin can accept by access the `get_requirements()` method of the plugin. This is a classmethod, so can be called on an uninstantiated copy of the plugin. :: - (primary) >>> pslist.PsList.get_requirements() - [, , , , ] + (layer_name) >>> pslist.PsList.get_requirements() + [, , , ] We can provide arguments via the `dpo` method call: :: - (primary) >>> display_plugin_output(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) + (layer_name) >>> display_plugin_output(pslist.PsList, kernel = self.config['kernel']) PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output @@ -142,8 +143,9 @@ We can provide arguments via the `dpo` method call: 356 4 smss.exe 0x8c0bccf8d040 3 - N/A False 2021-03-13 17:25:33.000000 N/A Disabled ... -Here's we've provided the current layer as the TranslationLayerRequirement, and used the symbol tables requirement -requested by the volshell plugin itself. A different table could be loaded and provided instead. The context used +Here's we've provided the kernel name that was requested by the volshell plugin itself (the generic volshell does not +load a kernel module, and instead only has a TranslationLayerRequirement). +A different module could be created and provided instead. The context used by the `dpo` method is always `context`. Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by @@ -151,8 +153,8 @@ using the `generate_treegrid` or `gt` command. :: - (primary) >>> treegrid = gt(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) - (primary) >>> treegrid.populate() + (layer_name) >>> treegrid = gt(pslist.PsList, kernel = self.config['kernel']) + (layer_name) >>> treegrid.populate() Treegrids must be populated before the data in them can be accessed. This is where the plugin actually runs and produces data. diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 2c5e13211..4cdbd26e8 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,14 +19,14 @@ import os import sys import tempfile import traceback -from typing import Dict, Type, Union, Any +from typing import Any, Dict, Type, Union from urllib import parse, request import volatility3.plugins import volatility3.symbols from volatility3 import framework from volatility3.cli import text_renderer, volargparse -from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins, configuration +from volatility3.framework import automagic, configuration, constants, contexts, exceptions, interfaces, plugins from volatility3.framework.automagic import stacker from volatility3.framework.configuration import requirements diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 7b8a759a6..812d44337 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -7,12 +7,11 @@ import json import logging import os import sys -import glob import volatility3.plugins import volatility3.symbols from volatility3 import cli, framework -from volatility3.cli.volshell import generic, windows, linux, mac +from volatility3.cli.volshell import generic, linux, mac, windows from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins # Make sure we log everything diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 8f81a0420..29accb529 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -8,11 +8,11 @@ import random import string import struct import sys -from typing import Any, Dict, List, Optional, Tuple, Union, Type, Iterable -from urllib import request, parse +from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union +from urllib import parse, request from volatility3.cli import text_renderer, volshell -from volatility3.framework import renderers, interfaces, objects, plugins, exceptions +from volatility3.framework import exceptions, interfaces, objects, plugins, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, physical, resources @@ -32,6 +32,7 @@ class Volshell(interfaces.plugins.PluginInterface): super().__init__(*args, **kwargs) self.__current_layer: Optional[str] = None self.__console = None + self.__kernel = None def random_string(self, length: int = 32) -> str: return ''.join(random.sample(string.ascii_uppercase + string.digits, length)) @@ -57,8 +58,6 @@ class Volshell(interfaces.plugins.PluginInterface): Return a TreeGrid but this is always empty since the point of this plugin is to run interactively """ - self.__current_layer = self.config['primary'] - # Try to enable tab completion try: import readline @@ -79,9 +78,10 @@ class Volshell(interfaces.plugins.PluginInterface): banner = f""" Call help() to see available functions - Volshell mode: {mode} - Current Layer: {self.current_layer} - """ + Volshell mode : {mode} + Current Layer : {self.current_layer} + Current Symbol Table: {self.current_symbol_table} +""" sys.ps1 = f"({self.current_layer}) >>> " self.__console = code.InteractiveConsole(locals = self._construct_locals_dict()) @@ -174,12 +174,23 @@ class Volshell(interfaces.plugins.PluginInterface): @property def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.config['primary'] return self.__current_layer + @property + def current_symbol_table(self): + return None + + @property + def kernel(self): + """No default kernel for generic volshell""" + return None + def change_layer(self, layer_name = None): """Changes the current default layer""" if not layer_name: - layer_name = self.config['primary'] + layer_name = self.current_layer self.__current_layer = layer_name sys.ps1 = f"({self.current_layer}) >>> " diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 4338ae06f..a58ff78f6 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -5,7 +5,7 @@ from typing import Any, List, Tuple, Union from volatility3.cli.volshell import generic -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.plugins.linux import pslist @@ -15,8 +15,8 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return (super().get_requirements() + [ - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), + return ([ + requirements.ModuleRequirement(name = "kernel", description = "Linux kernel module"), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) @@ -37,14 +37,14 @@ class Volshell(generic.Volshell): def list_tasks(self): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_tasks(self.context, self.config['primary'], self.config['vmlinux'])) + return list(pslist.PsList.list_tasks(self.context, self.current_layer, self.current_symbol_table)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (['ct', 'change_task', 'cp'], self.change_task), (['lt', 'list_tasks', 'ps'], self.list_tasks), - (['symbols'], self.context.symbol_space[self.config['vmlinux']]), + (['symbols'], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get('pid', None) is not None: self.change_task(self.config['pid']) @@ -64,3 +64,19 @@ class Volshell(generic.Volshell): if symbol_table is None: symbol_table = self.config['vmlinux'] return super().display_symbols(symbol_table) + + @property + def kernel(self): + if self.__kernel is None: + self.__kernel = self.context.modules[self.config['kernel']] + return self.__kernel + + @property + def current_symbol_table(self): + return self.kernel.symbol_table_name + + @property + def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.kernel.layer_name + return self.__current_layer diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 8218848ba..644a02bd2 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -15,9 +15,9 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return (super().get_requirements() + [ - requirements.SymbolTableRequirement(name = "darwin", description = "Darwin kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + return ([ + requirements.ModuleRequirement(name = "kernel", description = "Darwin kernel module"), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) @@ -37,14 +37,14 @@ class Volshell(generic.Volshell): def list_tasks(self): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_tasks(self.context, self.config['primary'], self.config['darwin'])) + return list(pslist.PsList.list_tasks(self.context, self.current_layer, self.current_symbol_table)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (['ct', 'change_task', 'cp'], self.change_task), (['lt', 'list_tasks', 'ps'], self.list_tasks), - (['symbols'], self.context.symbol_space[self.config['darwin']]), + (['symbols'], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get('pid', None) is not None: self.change_task(self.config['pid']) @@ -62,5 +62,21 @@ class Volshell(generic.Volshell): def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: - symbol_table = self.config['darwin'] + symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) + + @property + def kernel(self): + if self.__kernel is None: + self.__kernel = self.context.modules[self.config['kernel']] + return self.__kernel + + @property + def current_symbol_table(self): + return self.kernel.symbol_table_name + + @property + def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.kernel.layer_name + return self.__current_layer diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 6c191ad28..c35a8dfc7 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -5,7 +5,7 @@ from typing import Any, List, Tuple, Union from volatility3.cli.volshell import generic -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.plugins.windows import pslist @@ -15,8 +15,8 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return (super().get_requirements() + [ - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + return ([ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel'), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) @@ -34,14 +34,14 @@ class Volshell(generic.Volshell): def list_processes(self): """Returns a list of EPROCESS objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_processes(self.context, self.config['primary'], self.config['nt_symbols'])) + return list(pslist.PsList.list_processes(self.context, self.current_layer, self.current_symbol_table)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (['cp', 'change_process'], self.change_process), (['lp', 'list_processes', 'ps'], self.list_processes), - (['symbols'], self.context.symbol_space[self.config['nt_symbols']]), + (['symbols'], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get('pid', None) is not None: self.change_process(self.config['pid']) @@ -53,11 +53,27 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['nt_symbols'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: - symbol_table = self.config['nt_symbols'] + symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) + + @property + def kernel(self): + if self.__kernel is None: + self.__kernel = self.context.modules[self.config['kernel']] + return self.__kernel + + @property + def current_symbol_table(self): + return self.kernel.symbol_table_name + + @property + def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.kernel.layer_name + return self.__current_layer From 9fb59e4714f15bf571e4c70afa3c7fb290510040 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 17 Feb 2022 01:43:50 +0000 Subject: [PATCH 070/158] Volshell: Further improvements for mac/linux --- volatility3/cli/volshell/generic.py | 57 ++++++++++++++++++++++++----- volatility3/cli/volshell/linux.py | 12 +----- volatility3/cli/volshell/mac.py | 14 +------ volatility3/cli/volshell/windows.py | 10 ----- 4 files changed, 50 insertions(+), 43 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 29accb529..94634d005 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -31,8 +31,9 @@ class Volshell(interfaces.plugins.PluginInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__current_layer: Optional[str] = None + self.__current_symbol_table: Optional[str] = None + self.__current_kernel_name: Optional[str] = None self.__console = None - self.__kernel = None def random_string(self, length: int = 32) -> str: return ''.join(random.sample(string.ascii_uppercase + string.digits, length)) @@ -78,9 +79,10 @@ class Volshell(interfaces.plugins.PluginInterface): banner = f""" Call help() to see available functions - Volshell mode : {mode} - Current Layer : {self.current_layer} - Current Symbol Table: {self.current_symbol_table} + Volshell mode : {mode} + Current Layer : {self.current_layer} + Current Symbol Table : {self.current_symbol_table} + Current Kernel : {self.current_kernel_name} """ sys.ps1 = f"({self.current_layer}) >>> " @@ -121,7 +123,10 @@ class Volshell(interfaces.plugins.PluginInterface): (['dw', 'display_words'], self.display_words), (['dd', 'display_doublewords'], self.display_doublewords), (['dq', 'display_quadwords'], self.display_quadwords), (['dis', 'disassemble'], self.disassemble), - (['cl', 'change_layer'], self.change_layer), (['context'], self.context), (['self'], self), + (['cl', 'change_layer'], self.change_layer), + (['cs', 'change_symboltable'], self.change_symbol_table), + (['ck', 'change_kernel'], self.change_kernel), + (['context'], self.context), (['self'], self), (['dpo', 'display_plugin_output'], self.display_plugin_output), (['gt', 'generate_treegrid'], self.generate_treegrid), (['rt', 'render_treegrid'], self.render_treegrid), @@ -180,20 +185,52 @@ class Volshell(interfaces.plugins.PluginInterface): @property def current_symbol_table(self): - return None + if self.__current_symbol_table is None and self.kernel: + self.__current_symbol_table = self.kernel.symbol_table_name + return self.__current_symbol_table + + @property + def current_kernel_name(self): + if self.__current_kernel_name is None: + self.__current_kernel_name = self.config.get('kernel', None) + return self.__current_kernel_name @property def kernel(self): - """No default kernel for generic volshell""" - return None + """Returns the current kernel object""" + if self.current_kernel_name not in self.context.modules: + return None + return self.context.modules[self.current_kernel_name] - def change_layer(self, layer_name = None): + def change_layer(self, layer_name: str = None): """Changes the current default layer""" if not layer_name: layer_name = self.current_layer - self.__current_layer = layer_name + if layer_name not in self.context.layers: + print(f"Layer {layer_name} not present in context") + else: + self.__current_layer = layer_name sys.ps1 = f"({self.current_layer}) >>> " + def change_symbol_table(self, symbol_table_name: str = None): + """Changes the current_symbol_table""" + if not symbol_table_name: + print("No symbol table provided, not changing current symbol table") + if symbol_table_name not in self.context.symbol_space: + print(f"Symbol table {symbol_table_name} not present in context symbol_space") + else: + self.__current_symbol_table = symbol_table_name + print(f"Current Symbol Table: {self.current_symbol_table}") + + def change_kernel(self, kernel_name: str = None): + if not kernel_name: + print("No kernel module name provided, not changing current kernel") + if kernel_name not in self.context.modules: + print(f"Kernel module {kernel_name} not found in the context module list") + else: + self.__current_kernel_name = kernel_name + print(f"Current kernel : {self.current_kernel_name}") + def display_bytes(self, offset, count = 128, layer_name = None): """Displays byte values and ASCII characters""" remaining_data = self._read_data(offset, count = count, layer_name = layer_name) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index a58ff78f6..97a488743 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -37,7 +37,7 @@ class Volshell(generic.Volshell): def list_tasks(self): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_tasks(self.context, self.current_layer, self.current_symbol_table)) + return list(pslist.PsList.list_tasks(self.context, self.current_kernel_name)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() @@ -65,16 +65,6 @@ class Volshell(generic.Volshell): symbol_table = self.config['vmlinux'] return super().display_symbols(symbol_table) - @property - def kernel(self): - if self.__kernel is None: - self.__kernel = self.context.modules[self.config['kernel']] - return self.__kernel - - @property - def current_symbol_table(self): - return self.kernel.symbol_table_name - @property def current_layer(self): if self.__current_layer is None: diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 644a02bd2..305f80505 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -34,10 +34,10 @@ class Volshell(generic.Volshell): return print(f"No task with task ID {pid} found") - def list_tasks(self): + def list_tasks(self, method = None): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_tasks(self.context, self.current_layer, self.current_symbol_table)) + return list(pslist.PsList.get_list_tasks(method)(self.context, self.current_kernel_name)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() @@ -65,16 +65,6 @@ class Volshell(generic.Volshell): symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) - @property - def kernel(self): - if self.__kernel is None: - self.__kernel = self.context.modules[self.config['kernel']] - return self.__kernel - - @property - def current_symbol_table(self): - return self.kernel.symbol_table_name - @property def current_layer(self): if self.__current_layer is None: diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index c35a8dfc7..2cc5d3e1d 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -62,16 +62,6 @@ class Volshell(generic.Volshell): symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) - @property - def kernel(self): - if self.__kernel is None: - self.__kernel = self.context.modules[self.config['kernel']] - return self.__kernel - - @property - def current_symbol_table(self): - return self.kernel.symbol_table_name - @property def current_layer(self): if self.__current_layer is None: From e6c3c94a10a087a465c25f8f3b4a3e6e86b7eadb Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 17 Feb 2022 01:46:54 +0000 Subject: [PATCH 071/158] Volshell: Update docs slightly --- doc/source/volshell.rst | 7 ++++--- volatility3/cli/volshell/generic.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 1e51f90ba..de3c4398a 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -29,9 +29,10 @@ operating system mode for volshell, and the current layer available for use. Call help() to see available functions - Volshell mode : Generic - Current Layer : primary - Current Symbol Table: None + Volshell mode : Generic + Current Layer : primary + Current Symbol Table : None + Current Kernel Name : None (primary) >>> diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 94634d005..19e263a03 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -82,7 +82,7 @@ class Volshell(interfaces.plugins.PluginInterface): Volshell mode : {mode} Current Layer : {self.current_layer} Current Symbol Table : {self.current_symbol_table} - Current Kernel : {self.current_kernel_name} + Current Kernel Name : {self.current_kernel_name} """ sys.ps1 = f"({self.current_layer}) >>> " From da7dd322711187f6473eaf6f4fc116b021764ef3 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 17 Feb 2022 15:31:48 +0200 Subject: [PATCH 072/158] Improve slow pdb scanning --- volatility3/framework/automagic/pdbscan.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 8179339c8..3e350b071 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -192,9 +192,15 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): if not physical: layer_to_scan = virtual_layer_name + start_scan_address = 0 + if not physical and context.layers[layer_to_scan].metadata.architecture in ["Intel64"]: + # TODO: change this value accordingly when 5-Level paging is supported. + start_scan_address = (0x1f0 << 39) + kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] kernels = PDBUtility.pdbname_scan(ctx = context, layer_name = layer_to_scan, + start = start_scan_address, page_size = vlayer.page_size, pdb_names = kernel_pdb_names, progress_callback = progress_callback) From 30368751368e7874f452614d3f738e5da28ea8a8 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 17 Feb 2022 16:10:44 +0200 Subject: [PATCH 073/158] run optimized scan before slow scan --- volatility3/framework/automagic/pdbscan.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 3e350b071..5db66a3d0 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -146,8 +146,13 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): return None return (virtual_layer_name, kernel['mz_offset'], kernel) + vollog.debug("Kernel base determination - optimized scan virtual layer") + valid_kernel = self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, True, False, progress_callback) + if valid_kernel != None: + return valid_kernel + vollog.debug("Kernel base determination - slow scan virtual layer") - return self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, False, progress_callback) + return self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, False, False, progress_callback) def method_fixed_mapping(self, context: interfaces.context.ContextInterface, @@ -175,12 +180,13 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vollog.debug(f"Potential kernel_virtual_offset caused a page fault: {hex(kvo)}") vollog.debug("Kernel base determination - testing fixed base address") - return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, True, progress_callback) + return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, False, True, progress_callback) def _method_layer_pdb_scan(self, context: interfaces.context.ContextInterface, vlayer: layers.intel.Intel, test_kernel: Callable, + optimized: bool = False, physical: bool = True, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: # TODO: Verify this is a windows image @@ -193,7 +199,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): layer_to_scan = virtual_layer_name start_scan_address = 0 - if not physical and context.layers[layer_to_scan].metadata.architecture in ["Intel64"]: + if optimized and not physical and context.layers[layer_to_scan].metadata.architecture in ["Intel64"]: # TODO: change this value accordingly when 5-Level paging is supported. start_scan_address = (0x1f0 << 39) From bfc4c50e671404e3aa6d9262facd1a05951e3317 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Sun, 20 Feb 2022 14:49:06 +0200 Subject: [PATCH 074/158] related --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index b5a7db286..107689dc8 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -136,7 +136,7 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): if k not in ["context", "data_format", "object_info", "type_name"]: kwargs[k] = v kwargs['new_value'] = self.__new_value - return (self._context, self._vol.maps[-2]['type_name'], self._vol.maps[-3], self._data_format), kwargs + return (self._context, self._vol.maps[-3]['type_name'], self._vol.maps[-2], self._data_format), kwargs @classmethod def _unmarshall(cls, context: interfaces.context.ContextInterface, data_format: DataFormatInfo, From f2e3df27f48c09e75d42dc56419698ef8e5001af Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 21 Feb 2022 14:57:31 +0200 Subject: [PATCH 075/158] fix read whole module --- volatility3/framework/symbols/windows/pdbutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 35bc62d18..585e96b6d 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -146,7 +146,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): max_size = pe_data.OPTIONAL_HEADER.SizeOfImage # Proper data - virtual_data = layer.read(offset, max_size) + virtual_data = layer.read(offset, max_size, pad=True) pe_data = pefile.PE(data = virtual_data) # De-virtualize the memory From 58697479bb819fe6c3f17182bb419a03bc4541d7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 23 Feb 2022 00:00:07 +0000 Subject: [PATCH 076/158] Layers: Fix opening UNC paths on windows --- volatility3/framework/layers/resources.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 7ace25290..f8705edca 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -10,10 +10,11 @@ import logging import lzma import os import ssl +import sys import urllib.parse import urllib.request import zipfile -from typing import Optional, Any, IO, List +from typing import Any, IO, List, Optional from urllib import error from volatility3 import framework @@ -100,6 +101,19 @@ class ResourceAccessor(object): """ urllib.request.install_opener(urllib.request.build_opener(*self._handlers)) + # Python bug 46654 + if sys.platform == 'win32': + # We only need to worry about UNC paths on windows, on linux they'd be smb:// and need pysmb or similar + parsed_url = urllib.parse.urlparse(url, scheme = 'file') + if parsed_url.scheme == 'file' and parsed_url.netloc: + # Change the netloc to '/' and then prepend the netloc to the path + # Urlunparse will remove extra initial slashes from path, hence setting netloc + new_url = urllib.parse.urlunparse((parsed_url.scheme, '/', + '/' + parsed_url.netloc + parsed_url.path, parsed_url.params, + parsed_url.query, parsed_url.fragment)) + vollog.log(constants.LOGLEVEL_VVVV, f'UNC path detected, converted path {url} to {new_url}') + url = new_url + try: fp = urllib.request.urlopen(url, context = self._context) except error.URLError as excp: From 579a0b873515dc94795f9bef0efddc4f743cd372 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 23 Feb 2022 00:08:10 +0000 Subject: [PATCH 077/158] Layers: More documentation and don't break correct URLs --- volatility3/framework/layers/resources.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index f8705edca..ac25b5cc2 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -105,7 +105,9 @@ class ResourceAccessor(object): if sys.platform == 'win32': # We only need to worry about UNC paths on windows, on linux they'd be smb:// and need pysmb or similar parsed_url = urllib.parse.urlparse(url, scheme = 'file') - if parsed_url.scheme == 'file' and parsed_url.netloc: + # Only worry about file scheme URLs, make sure that there's either a host or + # the unparsing left an extra slash at the start (which will get lost with urlunparse) + if parsed_url.scheme == 'file' and (parsed_url.netloc or parsed_url.path.startswith('//')): # Change the netloc to '/' and then prepend the netloc to the path # Urlunparse will remove extra initial slashes from path, hence setting netloc new_url = urllib.parse.urlunparse((parsed_url.scheme, '/', From 265b2825697ecb8c94ba8654acd6191ba0055fdd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 23 Feb 2022 22:53:54 +0000 Subject: [PATCH 078/158] Objects: Don't try to read 0 bytes when unmarshalling --- volatility3/framework/objects/__init__.py | 6 +++++- .../symbols/windows/extensions/__init__.py | 16 ++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 107689dc8..c191d5562 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -141,7 +141,11 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): @classmethod def _unmarshall(cls, context: interfaces.context.ContextInterface, data_format: DataFormatInfo, object_info: interfaces.objects.ObjectInformation) -> TUnion[int, float, bool, bytes, str]: - data = context.layers.read(object_info.layer_name, object_info.offset, data_format.length) + # Don't try to lookup a 0 length data format, incase it's at an invalid offset. Length 0 means b'' + if data_format.length > 0: + data = context.layers.read(object_info.layer_name, object_info.offset, data_format.length) + else: + data = b'' return convert_data_to_value(data, cls._struct_type, data_format) class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 84c47e733..616744093 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -7,16 +7,17 @@ import datetime import functools import logging import math -from typing import Iterable, Iterator, Optional, Union, Tuple, List +from typing import Iterable, Iterator, List, Optional, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols from volatility3.framework.layers import intel from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic -from volatility3.framework.symbols.windows.extensions import pool, pe, kdbg +from volatility3.framework.symbols.windows.extensions import kdbg, pe, pool vollog = logging.getLogger(__name__) + # Keep these in a basic module, to prevent import cycles when symbol providers require them @@ -461,10 +462,13 @@ class UNICODE_STRING(objects.StructType): # We explicitly do *not* catch errors here, we allow an exception to be thrown # (otherwise there's no way to determine anything went wrong) # It's up to the user of this method to catch exceptions - return self.Buffer.dereference().cast("string", - max_length = self.Length, - errors = "replace", - encoding = "utf16") + + # We manually construct an object rather than casting a dereferenced pointer in case + # the buffer length is 0 and the pointer is a NULL pointer + return self._context.object(self.vol.type_name.split(constants.BANG)[0] + constants.BANG + 'string', + layer_name = self.Buffer.vol.layer_name, + offset = self.Buffer, + max_length = self.Length, errors = 'replace', encoding = 'utf16') String = property(get_string) From 78b3553b2ab8d4a318df021d09b191b9192add5b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 25 Feb 2022 16:33:54 +0000 Subject: [PATCH 079/158] Objects: Implement minor code optimization by @paulkermann --- volatility3/framework/objects/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index c191d5562..472370e6b 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -142,10 +142,9 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): def _unmarshall(cls, context: interfaces.context.ContextInterface, data_format: DataFormatInfo, object_info: interfaces.objects.ObjectInformation) -> TUnion[int, float, bool, bytes, str]: # Don't try to lookup a 0 length data format, incase it's at an invalid offset. Length 0 means b'' + data = b'' if data_format.length > 0: data = context.layers.read(object_info.layer_name, object_info.offset, data_format.length) - else: - data = b'' return convert_data_to_value(data, cls._struct_type, data_format) class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): From 1b09f20b5c226622408dc26494364195b9705d88 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 13:21:34 +0000 Subject: [PATCH 080/158] Windows: Raise PE extraction size and make it a constant --- .../framework/constants/windows/__init__.py | 2 ++ .../framework/symbols/windows/extensions/pe.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/constants/windows/__init__.py b/volatility3/framework/constants/windows/__init__.py index a19216605..7face984a 100644 --- a/volatility3/framework/constants/windows/__init__.py +++ b/volatility3/framework/constants/windows/__init__.py @@ -8,3 +8,5 @@ Windows-specific values that aren't found in debug symbols KERNEL_MODULE_NAMES = ["ntkrnlmp", "ntkrnlpa", "ntkrpamp", "ntoskrnl"] """The list of names that kernel modules can have within the windows OS""" + +PE_MAX_EXTRACTION_SIZE = 1024 * 1024 * 256 diff --git a/volatility3/framework/symbols/windows/extensions/pe.py b/volatility3/framework/symbols/windows/extensions/pe.py index df461318f..2f271da5d 100644 --- a/volatility3/framework/symbols/windows/extensions/pe.py +++ b/volatility3/framework/symbols/windows/extensions/pe.py @@ -2,15 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Generator, Tuple import logging +from typing import Generator, Tuple -from volatility3.framework import constants -from volatility3.framework import objects, interfaces +from volatility3.framework import constants, interfaces, objects from volatility3.framework.renderers import conversion vollog = logging.getLogger(__name__) + class IMAGE_DOS_HEADER(objects.StructType): def get_nt_header(self) -> interfaces.objects.ObjectInterface: @@ -77,12 +77,13 @@ class IMAGE_DOS_HEADER(objects.StructType): image_base_type = nt_header.OptionalHeader.ImageBase.vol.type_name member_size = self._context.symbol_space.get_type(image_base_type).size try: - newval = objects.convert_value_to_data(self.vol.offset, int, nt_header.OptionalHeader.ImageBase.vol.data_format) + newval = objects.convert_value_to_data(self.vol.offset, int, + nt_header.OptionalHeader.ImageBase.vol.data_format) new_pe = raw_data[:image_base_offset] + newval + raw_data[image_base_offset + member_size:] except OverflowError: vollog.warning("Volatility was unable to fix the image base for the PE file at base address {:#x}. " \ - "This will cause issues with many static analysis tools if you do not inform the " \ - "tool of the in-memory load address.".format(self.vol.offset)) + "This will cause issues with many static analysis tools if you do not inform the " \ + "tool of the in-memory load address.".format(self.vol.offset)) new_pe = raw_data return new_pe @@ -109,7 +110,7 @@ class IMAGE_DOS_HEADER(objects.StructType): size_of_image = nt_header.OptionalHeader.SizeOfImage # no legitimate PE is going to be larger than this - if size_of_image > (1024 * 1024 * 100): + if size_of_image > constants.windows.PE_MAX_EXTRACTION_SIZE: raise ValueError(f"The claimed SizeOfImage is too large: {size_of_image}") read_layer = self._context.layers[layer_name] From f156d237a4129da9394f81e72949bef58f3e0b76 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Feb 2022 13:36:54 +0900 Subject: [PATCH 081/158] Add 'ImportError' handling of the capstone module on malfind plugin. --- volatility3/framework/plugins/windows/malfind.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index bfd29a254..7fd032ef5 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -13,6 +13,12 @@ from volatility3.plugins.windows import pslist, vadinfo vollog = logging.getLogger(__name__) +try: + import capstone + + has_capstone = True +except ImportError: + has_capstone = False class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" From 9868aeb9060687b240637470d690de4327cab3c3 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Feb 2022 14:12:25 +0900 Subject: [PATCH 082/158] Add Error Raise point --- volatility3/framework/plugins/windows/malfind.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 7fd032ef5..ae10a6c65 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -134,7 +134,11 @@ class Malfind(interfaces.plugins.PluginInterface): else: architecture = "intel64" - disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) + if has_capstone: + disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) + else: + raise exceptions.MissingModuleException( + "capstone", "Requires capstone to disassembly data") file_output = "Disabled" if self.config['dump']: From 58782fcfe1ab7f495897571f9fd1a682b6fc48f9 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 3 Mar 2022 02:00:32 +0900 Subject: [PATCH 083/158] Typo Error Fix - Context module object Args code comment --- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 518215ab4..ab81beb5e 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -141,7 +141,7 @@ class Context(interfaces.context.ContextInterface): layer_name: The layer within the context in which the module exists offset: The offset at which the module exists in the layer native_layer_name: The default native layer for objects constructed by the module - size: The size, in bytes, that the module occupys from offset location within the layer named layer_name + size: The size, in bytes, that the module occupies from offset location within the layer named layer_name """ if size: return SizedModule.create(self, diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index c52e1aaa5..b8470ae47 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -129,7 +129,7 @@ class ContextInterface(metaclass = ABCMeta): layer_name: The layer the module is associated with (which layer the module lives within) offset: The initial/base offset of the module (used as the offset for relative symbols) native_layer_name: The default native_layer_name to use when the module constructs objects - size: The size, in bytes, that the module occupys from offset location within the layer named layer_name + size: The size, in bytes, that the module occupies from offset location within the layer named layer_name Returns: A module object From dae88605778a7b627ccd924e0a1ea7b8742ca0e0 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 3 Mar 2022 02:02:43 +0900 Subject: [PATCH 084/158] Restore PR --- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index ab81beb5e..2fd00c531 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -141,7 +141,7 @@ class Context(interfaces.context.ContextInterface): layer_name: The layer within the context in which the module exists offset: The offset at which the module exists in the layer native_layer_name: The default native layer for objects constructed by the module - size: The size, in bytes, that the module occupies from offset location within the layer named layer_name + size: The size, in bytes, that the module occupy from offset location within the layer named layer_name """ if size: return SizedModule.create(self, diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index b8470ae47..b70d55baa 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -129,7 +129,7 @@ class ContextInterface(metaclass = ABCMeta): layer_name: The layer the module is associated with (which layer the module lives within) offset: The initial/base offset of the module (used as the offset for relative symbols) native_layer_name: The default native_layer_name to use when the module constructs objects - size: The size, in bytes, that the module occupies from offset location within the layer named layer_name + size: The size, in bytes, that the module occupy from offset location within the layer named layer_name Returns: A module object From 49308eb18dd0035188ab7939dc825904d477066f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 3 Mar 2022 02:03:31 +0900 Subject: [PATCH 085/158] Restore PR --- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 2fd00c531..518215ab4 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -141,7 +141,7 @@ class Context(interfaces.context.ContextInterface): layer_name: The layer within the context in which the module exists offset: The offset at which the module exists in the layer native_layer_name: The default native layer for objects constructed by the module - size: The size, in bytes, that the module occupy from offset location within the layer named layer_name + size: The size, in bytes, that the module occupys from offset location within the layer named layer_name """ if size: return SizedModule.create(self, diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index b70d55baa..c52e1aaa5 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -129,7 +129,7 @@ class ContextInterface(metaclass = ABCMeta): layer_name: The layer the module is associated with (which layer the module lives within) offset: The initial/base offset of the module (used as the offset for relative symbols) native_layer_name: The default native_layer_name to use when the module constructs objects - size: The size, in bytes, that the module occupy from offset location within the layer named layer_name + size: The size, in bytes, that the module occupys from offset location within the layer named layer_name Returns: A module object From 670401eac71d39cd24cea9a17ef0062bb9722756 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 3 Mar 2022 20:35:39 +0000 Subject: [PATCH 086/158] Windows: Test unicode strings for length 0 In some tests we were checking whether asking for the string value threw an InvalidAddressException through an error as to whether we should look elsewhere for the data. As of commit 265b2825 we now treat 0-length strings as valid (as per #652), meaning we need to check for length 0 as well as invalid pointers. If this crops up often, we may need to revisit the decision to make sure its in keeping with how windows treats zero length strings, but for now we only did it once for registry keys. Closes #665 --- .../symbols/windows/extensions/__init__.py | 19 +++++++++++-------- .../symbols/windows/extensions/registry.py | 18 ++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 616744093..dc0de1dda 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -307,12 +307,15 @@ class MMVAD(MMVAD_SHORT): try: # this is for xp and 2003 if self.has_member("ControlArea"): - file_name = self.ControlArea.FilePointer.FileName.get_string() + filename_obj = self.ControlArea.FilePointer.FileName # this is for vista through windows 7 else: - file_name = self.Subsection.ControlArea.FilePointer.dereference().cast( - "_FILE_OBJECT").FileName.get_string() + filename_obj = self.Subsection.ControlArea.FilePointer.dereference().cast( + "_FILE_OBJECT").FileName + + if filename_obj.Length > 0: + file_name = filename_obj.get_string() except exceptions.InvalidAddressException: pass @@ -902,8 +905,8 @@ class CONTROL_AREA(objects.StructType): return False # The first SubsectionBase should not be page aligned - #subsection = self.get_subsection() - #if subsection.SubsectionBase & self.PAGE_MASK == 0: + # subsection = self.get_subsection() + # if subsection.SubsectionBase & self.PAGE_MASK == 0: # return False except exceptions.InvalidAddressException: return False @@ -952,7 +955,7 @@ class CONTROL_AREA(objects.StructType): subsection_offset = starting_sector * 0x200 # Similar to the check in is_valid(), make sure the SubsectionBase is not page aligned. - #if subsection.SubsectionBase & self.PAGE_MASK == 0: + # if subsection.SubsectionBase & self.PAGE_MASK == 0: # break ptecount = 0 @@ -983,8 +986,8 @@ class CONTROL_AREA(objects.StructType): # Currently just a temporary workaround to deal with custom bit flag # in the PFN field for pages in transition state. # See https://github.com/volatilityfoundation/volatility3/pull/475 - physoffset = (mmpte.u.Trans.PageFrameNumber & (( 1 << 33 ) - 1 ) ) << 12 - + physoffset = (mmpte.u.Trans.PageFrameNumber & ((1 << 33) - 1)) << 12 + yield physoffset, file_offset, self.PAGE_SIZE # Go to the next PTE entry diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index f30bb5eb0..47ff24506 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -5,10 +5,10 @@ import enum import logging import struct -from typing import Optional, Iterable, Union +from typing import Iterable, Optional, Union -from volatility3.framework import constants, exceptions, objects, interfaces -from volatility3.framework.layers.registry import RegistryHive, RegistryInvalidIndex, RegistryFormatException +from volatility3.framework import constants, exceptions, interfaces, objects +from volatility3.framework.layers.registry import RegistryFormatException, RegistryHive, RegistryInvalidIndex vollog = logging.getLogger(__name__) @@ -76,7 +76,9 @@ class CMHIVE(objects.StructType): for attr in ["FileFullPath", "FileUserName", "HiveRootPath"]: try: - return getattr(self, attr).get_string() + name = getattr(self, attr) + if name.Length > 0: + return name.get_string() except (AttributeError, exceptions.InvalidAddressException): pass @@ -269,7 +271,7 @@ class CM_KEY_VALUE(objects.StructType): if self_type == RegValueTypes.REG_DWORD_BIG_ENDIAN: if len(data) != struct.calcsize(">L"): raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}") - res, = struct.unpack(">L", data) + res, = struct.unpack(">L", data) return res if self_type == RegValueTypes.REG_QWORD: if len(data) != struct.calcsize(" Date: Sat, 5 Mar 2022 16:27:07 +0900 Subject: [PATCH 087/158] Context Typo Error, MFT Symbol JSON Prettier --- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/context.py | 2 +- volatility3/framework/symbols/windows/mft.json | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 518215ab4..ab81beb5e 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -141,7 +141,7 @@ class Context(interfaces.context.ContextInterface): layer_name: The layer within the context in which the module exists offset: The offset at which the module exists in the layer native_layer_name: The default native layer for objects constructed by the module - size: The size, in bytes, that the module occupys from offset location within the layer named layer_name + size: The size, in bytes, that the module occupies from offset location within the layer named layer_name """ if size: return SizedModule.create(self, diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index c52e1aaa5..b8470ae47 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -129,7 +129,7 @@ class ContextInterface(metaclass = ABCMeta): layer_name: The layer the module is associated with (which layer the module lives within) offset: The initial/base offset of the module (used as the offset for relative symbols) native_layer_name: The default native_layer_name to use when the module constructs objects - size: The size, in bytes, that the module occupys from offset location within the layer named layer_name + size: The size, in bytes, that the module occupies from offset location within the layer named layer_name Returns: A module object diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index b71be6444..17edd45dd 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -270,7 +270,8 @@ "offset": 8, "type": { "kind": "base", - "name": "unsigned char" } + "name": "unsigned char" + } }, "NameLength": { "offset": 9, @@ -322,7 +323,8 @@ "offset": 8, "type": { "kind": "base", - "name": "unsigned short" } + "name": "unsigned short" + } } }, "kind": "struct", From f060562b278352e8c3cfadd65850688717c47b15 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Mar 2022 16:29:09 +0900 Subject: [PATCH 088/158] Rebase --- volatility3/framework/plugins/windows/malfind.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index ae10a6c65..e70fd0f75 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -13,12 +13,6 @@ from volatility3.plugins.windows import pslist, vadinfo vollog = logging.getLogger(__name__) -try: - import capstone - - has_capstone = True -except ImportError: - has_capstone = False class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" @@ -134,12 +128,8 @@ class Malfind(interfaces.plugins.PluginInterface): else: architecture = "intel64" - if has_capstone: - disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) - else: - raise exceptions.MissingModuleException( - "capstone", "Requires capstone to disassembly data") - + disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) + file_output = "Disabled" if self.config['dump']: file_output = "Error outputting to file" From 639f87a0a4e642ce02c848b4ae2639efc64639ff Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Mar 2022 16:29:40 +0900 Subject: [PATCH 089/158] Remove Tab --- volatility3/framework/plugins/windows/malfind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index e70fd0f75..bfd29a254 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -129,7 +129,7 @@ class Malfind(interfaces.plugins.PluginInterface): architecture = "intel64" disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture) - + file_output = "Disabled" if self.config['dump']: file_output = "Error outputting to file" From 06961ce53742a4f266d4892f7b7d8120dc34388b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Mar 2022 16:32:38 +0900 Subject: [PATCH 090/158] Initialize MBR Parser --- .../framework/plugins/windows/mbrparser.py | 76 ++++++++++++ .../symbols/windows/extensions/mbr.py | 111 ++++++++++++++++++ .../framework/symbols/windows/mbr.json | 30 +++++ 3 files changed, 217 insertions(+) create mode 100644 volatility3/framework/plugins/windows/mbrparser.py create mode 100644 volatility3/framework/symbols/windows/extensions/mbr.py create mode 100644 volatility3/framework/symbols/windows/mbr.json diff --git a/volatility3/framework/plugins/windows/mbrparser.py b/volatility3/framework/plugins/windows/mbrparser.py new file mode 100644 index 000000000..37a4612f8 --- /dev/null +++ b/volatility3/framework/plugins/windows/mbrparser.py @@ -0,0 +1,76 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import datetime +import logging + +from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mbr +from volatility3.plugins import yarascan + +vollog = logging.getLogger(__name__) + + +class MBRParser(interfaces.plugins.PluginInterface): + """ Scans for and parses potential Master Boot Records (MBRs) """ + + _required_framework_version = (2, 0, 1) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, + version = (2, 0, 0)), + ] + + @classmethod + def levenshtein(self, s1, s2): + if len(s1) < len(s2): + return self.levenshtein(s2, s1) + + if len(s2) == 0: + return len(s1) + + previous_row = range(len(s2) + 1) + for i, c1 in enumerate(s1): + current_row = [i + 1] + for j, c2 in enumerate(s2): + insertions = previous_row[j + 1] + 1 + deletions = current_row[j] + 1 + substitutions = previous_row[j] + (c1 != c2) + current_row.append(min(insertions, deletions, substitutions)) + previous_row = current_row + + return previous_row[-1] + + def _generator(self): + layer = self.context.layers[self.config['primary']] + rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/\x55\xaa/'}) + symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, + config_path = self.config_path, + sub_path = "windows", + filename = "mbr", + class_types = { + 'PARTITION_ENTRY': mbr.PARTITION_ENTRY, + }) + + for offset, _rule_name, _name, _value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): + try: + yield 1, (format_hints.Hex(offset), _value) + + except exceptions.PagedInvalidAddressException: + pass + + def run(self): + return renderers.TreeGrid([ + ('Offset', format_hints.Hex), + ('Record Type', str), + ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py new file mode 100644 index 000000000..eeb97d332 --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -0,0 +1,111 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework import objects + +import struct + +PartitionTypes = { + 0x00:"Empty", + 0x01:"FAT12,CHS", + 0x04:"FAT16 16-32MB,CHS", + 0x05:"Microsoft Extended", + 0x06:"FAT16 32MB,CHS", + 0x07:"NTFS", + 0x0b:"FAT32,CHS", + 0x0c:"FAT32,LBA", + 0x0e:"FAT16, 32MB-2GB,LBA", + 0x0f:"Microsoft Extended, LBA", + 0x11:"Hidden FAT12,CHS", + 0x14:"Hidden FAT16,16-32MB,CHS", + 0x16:"Hidden FAT16,32MB-2GB,CHS", + 0x18:"AST SmartSleep Partition", + 0x1b:"Hidden FAT32,CHS", + 0x1c:"Hidden FAT32,LBA", + 0x1e:"Hidden FAT16,32MB-2GB,LBA", + 0x27:"PQservice", + 0x39:"Plan 9 partition", + 0x3c:"PartitionMagic recovery partition", + 0x42:"Microsoft MBR,Dynamic Disk", + 0x44:"GoBack partition", + 0x51:"Novell", + 0x52:"CP/M", + 0x63:"Unix System V", + 0x64:"PC-ARMOUR protected partition", + 0x82:"Solaris x86 or Linux Swap", + 0x83:"Linux", + 0x84:"Hibernation", + 0x85:"Linux Extended", + 0x86:"NTFS Volume Set", + 0x87:"NTFS Volume Set", + 0x9f:"BSD/OS", + 0xa0:"Hibernation", + 0xa1:"Hibernation", + 0xa5:"FreeBSD", + 0xa6:"OpenBSD", + 0xa8:"Mac OSX", + 0xa9:"NetBSD", + 0xab:"Mac OSX Boot", + 0xaf:"MacOS X HFS", + 0xb7:"BSDI", + 0xb8:"BSDI Swap", + 0xbb:"Boot Wizard hidden", + 0xbe:"Solaris 8 boot partition", + 0xd8:"CP/M-86", + 0xde:"Dell PowerEdge Server utilities (FAT fs)", + 0xdf:"DG/UX virtual disk manager partition", + 0xeb:"BeOS BFS", + 0xee:"EFI GPT Disk", + 0xef:"EFI System Partition", + 0xfb:"VMWare File System", + 0xfc:"VMWare Swap", +} + +class PARTITION_ENTRY(objects.StructType): + def get_value(self, char): + padded = "\x00\x00\x00" + str(char) + val = int(struct.unpack('>I', padded)[0]) + return val + + def get_type(self): + return PartitionTypes.get(self.get_value(self.PartitionType), "Invalid") + + def is_bootable(self): + return self.get_value(self.BootableFlag) == 0x80 + + def is_bootable_and_used(self): + return self.is_bootable() and self.is_used() + + def is_valid(self): + return self.get_type() != "Invalid" + + def is_used(self): + return self.get_type() != "Empty" and self.is_valid() + + def StartingSector(self): + return self.StartingCHS[1] % 64 + + def StartingCylinder(self): + return (self.StartingCHS[1] - self.StartingSector()) * 4 + self.StartingCHS[2] + + def EndingSector(self): + return self.EndingCHS[1] % 64 + + def EndingCylinder(self): + return (self.EndingCHS[1] - self.EndingSector()) * 4 + self.EndingCHS[2] + + def __str__(self): + processed_entry = "" + bootable = self.get_value(self.BootableFlag) + processed_entry = "Boot flag: {0:#x} {1}\n".format(bootable, "(Bootable)" if self.is_bootable() else '') + processed_entry += "Partition type: {0:#x} ({1})\n".format(self.get_value(self.PartitionType), self.get_type()) + processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.StartingLBA) + processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.StartingCylinder(), + self.StartingCHS[0], + self.StartingSector()) + processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.EndingCylinder(), + self.EndingCHS[0], + self.EndingSector()) + processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.SizeInSectors) + return processed_entry \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json new file mode 100644 index 000000000..83bd48f13 --- /dev/null +++ b/volatility3/framework/symbols/windows/mbr.json @@ -0,0 +1,30 @@ +{ + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Donghyun Kim", + "comment": "Using structures defined in File System Forensic Analysis pg 353+", + "datetime": "2022-01-03T13:37:00" + }, + "format": "6.1.0" + }, + { + 'PARTITION_ENTRY': [ 0x10, { + 'BootableFlag': [0x0, ['char']], # 0x80 is bootable + 'StartingCHS': [0x1, ['array', 3, ['unsigned char']]], + 'PartitionType': [0x4, ['char']], + 'EndingCHS': [0x5, ['array', 3, ['unsigned char']]], + 'StartingLBA': [0x8, ['unsigned int']], + 'SizeInSectors': [0xc, ['int']], + }], + 'PARTITION_TABLE': [ 0x200, { + 'DiskSignature': [ 0x1b8, ['array', 4, ['unsigned char']]], + 'Unused': [ 0x1bc, ['unsigned short']], + 'Entry1': [ 0x1be, ['PARTITION_ENTRY']], + 'Entry2': [ 0x1ce, ['PARTITION_ENTRY']], + 'Entry3': [ 0x1de, ['PARTITION_ENTRY']], + 'Entry4': [ 0x1ee, ['PARTITION_ENTRY']], + 'Signature': [0x1fe, ['unsigned short']], + }] + } +} \ No newline at end of file From 7570e82786f49db3e0aed591ce6a13b17a97570a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Mar 2022 17:47:37 +0900 Subject: [PATCH 091/158] Configuration Yara Rules --- .../framework/plugins/windows/mbrparser.py | 16 ++--- .../framework/symbols/windows/mbr.json | 64 +++++++++++++------ 2 files changed, 49 insertions(+), 31 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrparser.py b/volatility3/framework/plugins/windows/mbrparser.py index 37a4612f8..10afa53d2 100644 --- a/volatility3/framework/plugins/windows/mbrparser.py +++ b/volatility3/framework/plugins/windows/mbrparser.py @@ -2,13 +2,11 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import datetime import logging from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import mbr from volatility3.plugins import yarascan @@ -52,19 +50,13 @@ class MBRParser(interfaces.plugins.PluginInterface): def _generator(self): layer = self.context.layers[self.config['primary']] - rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/\x55\xaa/'}) - symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, - config_path = self.config_path, - sub_path = "windows", - filename = "mbr", - class_types = { - 'PARTITION_ENTRY': mbr.PARTITION_ENTRY, - }) + # TODO : YARA RULE HEX + rules = yarascan.YaraScan.process_yara_options({'yara_rules': "55 aa"}) for offset, _rule_name, _name, _value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): try: - yield 1, (format_hints.Hex(offset), _value) + yield 0, (format_hints.Hex(offset), _name) except exceptions.PagedInvalidAddressException: pass @@ -72,5 +64,5 @@ class MBRParser(interfaces.plugins.PluginInterface): def run(self): return renderers.TreeGrid([ ('Offset', format_hints.Hex), - ('Record Type', str), + ("Name", str) ], self._generator()) diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 83bd48f13..84162c96d 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -8,23 +8,49 @@ }, "format": "6.1.0" }, - { - 'PARTITION_ENTRY': [ 0x10, { - 'BootableFlag': [0x0, ['char']], # 0x80 is bootable - 'StartingCHS': [0x1, ['array', 3, ['unsigned char']]], - 'PartitionType': [0x4, ['char']], - 'EndingCHS': [0x5, ['array', 3, ['unsigned char']]], - 'StartingLBA': [0x8, ['unsigned int']], - 'SizeInSectors': [0xc, ['int']], - }], - 'PARTITION_TABLE': [ 0x200, { - 'DiskSignature': [ 0x1b8, ['array', 4, ['unsigned char']]], - 'Unused': [ 0x1bc, ['unsigned short']], - 'Entry1': [ 0x1be, ['PARTITION_ENTRY']], - 'Entry2': [ 0x1ce, ['PARTITION_ENTRY']], - 'Entry3': [ 0x1de, ['PARTITION_ENTRY']], - 'Entry4': [ 0x1ee, ['PARTITION_ENTRY']], - 'Signature': [0x1fe, ['unsigned short']], - }] - } + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": true, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "int", + "size": 1, + "signed": false, + "endian": "little" + }, + "wchar": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + } + }, + "symbols": {} } \ No newline at end of file From 244751e9aebf30c2f592c576b3a57ddbbe24b9ed Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 6 Mar 2022 18:48:00 +0000 Subject: [PATCH 092/158] Layers: Better checks on PAE page tables This checks that the very top level table points to the next four pages, as we'd expected in general. This relies on the same assumptions as the existing PAE detection did, ie that the PAE page_map maps the next four pages immediately. Previously we didn't check that the top page was valid, once we found the self-referential pointer. This adds in an appropriate check to reduce false positives. Closes #631. --- volatility3/framework/automagic/windows.py | 27 ++++++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 71548ca40..b93ee244c 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -28,9 +28,9 @@ The self-referential indices for older versions of windows are listed below: """ import logging import struct -from typing import Generator, List, Optional, Tuple, Type, Iterable +from typing import Generator, Iterable, List, Optional, Tuple, Type -from volatility3.framework import interfaces, layers, constants +from volatility3.framework import constants, interfaces, layers from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel @@ -116,10 +116,27 @@ class DtbSelfRefPae(DtbSelfReferential): mask = 0x3FFFFFFFFFF000, reserved_bits = 0x0) - def __call__(self, *args, **kwargs): - dtb = super().__call__(*args, **kwargs) + @staticmethod + def _and_bytes(abytes, bbytes): + return bytes([a & b for a, b in zip(abytes[::-1], bbytes[::-1])][::-1]) + + def __call__(self, data: bytes, data_offset: int, page_offset: int) -> Optional[Tuple[int, int]]: + dtb = super().__call__(data, data_offset, page_offset) if dtb: - return dtb[0] - 0x4000, dtb[1] + # Find the top page + top_pae_page = dtb[0] - 0x4000 + # The top page should map to the next four pages after it + # Build what we expect the page table to be + expected_table = b''.join([struct.pack(self.ptr_struct, top_pae_page + (i * 0x1000)) for i in range(1, 5)]) + # Mask off the page bits of top level page map + page_table_mask = b"\x00\xf0\xff\xff\xff\xff\xff\xff" * 4 + page_table = data[top_pae_page - data_offset: top_pae_page - data_offset + (4 * self.ptr_size)] + # Compare them + anded_bytes = self._and_bytes(page_table, page_table_mask) + if (anded_bytes == expected_table): + return top_pae_page, dtb[1] + # Return None since the dtb value *isn't* None + return None return dtb From 68903c63df92c2726fb9cfe75f902ada808a1d30 Mon Sep 17 00:00:00 2001 From: Samuel Zurowski Date: Sun, 6 Mar 2022 20:20:24 -0500 Subject: [PATCH 093/158] Added task_struct function to get each task_struct from the thread_nodes structure --- .../symbols/linux/extensions/__init__.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0edd60608..b48027bf5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -201,6 +201,24 @@ class task_struct(generic.GenericIntelProcess): yield (start, end - start) + def get_thread_nodes(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns a list of the task_struct based on the list_head + thread_node structure.""" + + task_symbol_table_name = self.get_symbol_table_name() + + parent = self.group_leader + for task in self.thread_node.to_list( + f"{task_symbol_table_name}{constants.BANG}task_struct", + "thread_node" + ): + + if task.group_leader != parent: continue + + yield task + + + class fs_struct(objects.StructType): From a1023e51f59aaf85b50d318b36a072772efaef1d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 7 Mar 2022 13:54:35 +0900 Subject: [PATCH 094/158] Fix Renderes, Scanners, MFT Symbol Typo Error --- volatility3/framework/interfaces/renderers.py | 2 +- volatility3/framework/layers/scanners/__init__.py | 4 ++-- volatility3/framework/symbols/windows/mft.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index 7f80425a4..9368009a9 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""All plugins output a TreeGrid object which must then be rendered (eithe by a +"""All plugins output a TreeGrid object which must then be rendered (either by a GUI, or as text output, html output or in some other form. This module defines both the output format (:class:`TreeGrid`) and the diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index 85c8390d9..ec66f2708 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -31,7 +31,7 @@ class BytesScanner(layers.ScannerInterface): class RegExScanner(layers.ScannerInterface): """A scanner that can be provided with a bytes-object regular expression pattern - The scanner will scqn all blocks for the regular expression and report the absolute offset of any finds + The scanner will scan all blocks for the regular expression and report the absolute offset of any finds The default flags include DOTALL, since the searches are through binary data and the newline character should have no specific significance in such searches""" @@ -95,7 +95,7 @@ class MultiStringScanner(layers.ScannerInterface): else: suffixes.append(re.escape(bytes([entry]))) else: - # If we've fininshed one of the strings at this point, remember it for later + # If we've finished one of the strings at this point, remember it for later finished = True if len(suffixes) == 1: diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index 17edd45dd..e5de8f3fa 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -134,7 +134,7 @@ "kind": "base", "name": "unsigned char" } - } + } }, "UpdateSequenceOffset": { "offset": 4, @@ -192,7 +192,7 @@ "name": "unsigned int" } }, - "AlocatedSize": { + "AllocatedSize": { "offset": 28, "type":{ "kind": "base", From 34a732a4f09f123746753c5b77661d44c92aeb2b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 7 Mar 2022 14:00:03 +0900 Subject: [PATCH 095/158] Fix Object Typo Error --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 472370e6b..1fa6dd62a 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -704,7 +704,7 @@ class AggregateType(interfaces.objects.ObjectInterface): tmp_list[member] = (relative_offset, new_child) # If there's trouble with mutability, consider making update_vol return a clone with the changes # (there will be a few other places that will be necessary) and/or making these part of the - # permanent dictionaries rather than the non-clonable ones + # permanent dictionaries rather than the non-cloneable ones template.update_vol(members = tmp_list) @classmethod From 4087236957d2567d9054f1ceb41700b054e3b835 Mon Sep 17 00:00:00 2001 From: Samuel Zurowski Date: Mon, 7 Mar 2022 19:18:17 -0500 Subject: [PATCH 096/158] Changed named and used thread_group instead to ensure all threads are grabbed --- .../symbols/linux/extensions/__init__.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b48027bf5..2fb832930 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -201,25 +201,22 @@ class task_struct(generic.GenericIntelProcess): yield (start, end - start) - def get_thread_nodes(self) -> Iterable[interfaces.objects.ObjectInterface]: + def get_threads(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns a list of the task_struct based on the list_head thread_node structure.""" task_symbol_table_name = self.get_symbol_table_name() - parent = self.group_leader - for task in self.thread_node.to_list( + # iterating through the thread_list from thread_group + # this allows iterating through pointers to grab the + # threads and using the thread_group offset to get the + # corresponding task_struct + for task in self.thread_group.to_list( f"{task_symbol_table_name}{constants.BANG}task_struct", - "thread_node" + "thread_group" ): - - if task.group_leader != parent: continue - yield task - - - class fs_struct(objects.StructType): def get_root_dentry(self): From 6c1fe42a3791f1702df1c2733c886cb96c7a1100 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 01:53:04 +0900 Subject: [PATCH 097/158] Fix Docs, Framework, Windows Plugin Typo Error --- doc/source/complex-plugin.rst | 2 +- doc/source/simple-plugin.rst | 4 ++-- volatility3/framework/interfaces/configuration.py | 2 +- volatility3/framework/objects/__init__.py | 2 +- volatility3/framework/plugins/windows/privileges.py | 4 ++-- volatility3/framework/plugins/windows/psscan.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/complex-plugin.rst b/doc/source/complex-plugin.rst index f06b398e8..8ab8a5186 100644 --- a/doc/source/complex-plugin.rst +++ b/doc/source/complex-plugin.rst @@ -300,7 +300,7 @@ This will mean that when a specific structure is loaded from the symbol_space, i `StructType`, but instead is instantiated using the NewStructureClass, meaning new methods can be called directly on it. If the situation really calls for an entirely new object, that isn't covered by one of the existing -:py:class:`~volatility3.framework.objects.PrimativeObject` objects (such as +:py:class:`~volatility3.framework.objects.PrimitiveObject` objects (such as :py:class:`~volatility3.framework.objects.Integer`, :py:class:`~volatility3.framework.objects.Boolean`, :py:class:`~volatility3.framework.objects.Float`, diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 4e499b186..9360ccf40 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -206,9 +206,9 @@ information may not be provided. The plugin then takes the process's ``BaseDllName`` value, and calls :py:meth:`~volatility3.framework.symbols.windows.extensions.UNICODE_STRING.get_string` on it. All structure attributes, as defined by the symbols, are directly accessible and use the case-style of the symbol library it came from (in Windows, -attributes are CamelCase), such as ``entry.BaseDllName`` in this instance. Any attribtues not defined by the symbol but added +attributes are CamelCase), such as ``entry.BaseDllName`` in this instance. Any attributes not defined by the symbol but added by Volatility extensions cannot be properties (in case they overlap with the attributes defined in the symbol libraries) -and are therefore always methods and prepended with ``get_``, in this example ``BaseDllName.get_string()``. +and are therefore always methods and pretended with ``get_``, in this example ``BaseDllName.get_string()``. Finally, ``FullDllName`` is populated. These operations read from memory, and as such, the memory image may be unable to read the data at a particular offset. This will cause an exception to be thrown. In Volatility 3, exceptions are thrown diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 7dc046a3e..c39dba680 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -73,7 +73,7 @@ class HierarchicalDict(collections.abc.Mapping): separator: str = CONFIG_SEPARATOR) -> None: """ Args: - initial_dict: A dictionary to populate the HierachicalDict with initially + initial_dict: A dictionary to populate the HierarchicalDict with initially separator: A custom hierarchy separator (defaults to CONFIG_SEPARATOR) """ if not (isinstance(separator, str) and len(separator) == 1): diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 1fa6dd62a..e0f927ec9 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -206,7 +206,7 @@ class Bytes(PrimitiveObject, bytes): length: int = 1, **kwargs) -> 'Bytes': """Creates the appropriate class and returns it so that the native type - is inherritted. + is inherited. The only reason the kwargs is added, is so that the inheriting types can override __init__ without needing to diff --git a/volatility3/framework/plugins/windows/privileges.py b/volatility3/framework/plugins/windows/privileges.py index eaafbeae6..2d48a30f7 100644 --- a/volatility3/framework/plugins/windows/privileges.py +++ b/volatility3/framework/plugins/windows/privileges.py @@ -55,14 +55,14 @@ class Privs(interfaces.plugins.PluginInterface): try: process_token = task.Token.dereference().cast("_TOKEN") except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, 'Skeep invalid token.') + vollog.log(constants.LOGLEVEL_VVV, 'Skip invalid token.') continue for value, present, enabled, default in process_token.privileges(): # Skip privileges whose bit positions cannot be # translated to a privilege name if not self.privilege_info.get(int(value)): - vollog.log(constants.LOGLEVEL_VVV, f'Skeep invalid privilege ({value}).') + vollog.log(constants.LOGLEVEL_VVV, f'Skip invalid privilege ({value}).') continue name, desc = self.privilege_info.get(int(value)) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 237c0edcd..a0601aef1 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -85,7 +85,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): context: The context to retrieve required elements (layers, symbol tables) from layer_name: The name of the layer on which to operate symbol_table: The name of the table containing the kernel symbols - proc: the process object with phisical address + proc: the process object with physical address Returns: A process object on virtual address layer From 18770d0cd3b0dff138a33b98d39e77743086ac3c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 02:09:47 +0900 Subject: [PATCH 098/158] Fix glossary.rst Typo Error --- doc/source/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 68bc41e4c..66dabfafe 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -145,7 +145,7 @@ Struct, Structure Symbol This is used in many different contexts, as a short term for many things. Within Volatility, a symbol is a - construct that usually encompasses a specific type :ref:`type` at a specfific :ref:`offset`, + construct that usually encompasses a specific type :ref:`type` at a specific :ref:`offset`, representing a particular instance of that type within the memory of a compiled and running program. An example would be the location in memory of a list of active tcp endpoints maintained by the networking stack within an operating system. From acc3f6f352d9446c773fd3ebf5891f67ba9d214b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 13:27:53 +0900 Subject: [PATCH 099/158] Update Symbol Table, Load Physical Layer --- .../framework/plugins/windows/mbrparser.py | 68 ------ .../framework/plugins/windows/mbrscan.py | 80 +++++++ .../symbols/windows/extensions/mbr.py | 110 +-------- .../framework/symbols/windows/mbr.json | 208 +++++++++++++++++- 4 files changed, 292 insertions(+), 174 deletions(-) delete mode 100644 volatility3/framework/plugins/windows/mbrparser.py create mode 100644 volatility3/framework/plugins/windows/mbrscan.py diff --git a/volatility3/framework/plugins/windows/mbrparser.py b/volatility3/framework/plugins/windows/mbrparser.py deleted file mode 100644 index 10afa53d2..000000000 --- a/volatility3/framework/plugins/windows/mbrparser.py +++ /dev/null @@ -1,68 +0,0 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -import logging - -from volatility3.framework import exceptions, interfaces, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols.windows.extensions import mbr -from volatility3.plugins import yarascan - -vollog = logging.getLogger(__name__) - - -class MBRParser(interfaces.plugins.PluginInterface): - """ Scans for and parses potential Master Boot Records (MBRs) """ - - _required_framework_version = (2, 0, 1) - - @classmethod - def get_requirements(cls): - return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, - version = (2, 0, 0)), - ] - - @classmethod - def levenshtein(self, s1, s2): - if len(s1) < len(s2): - return self.levenshtein(s2, s1) - - if len(s2) == 0: - return len(s1) - - previous_row = range(len(s2) + 1) - for i, c1 in enumerate(s1): - current_row = [i + 1] - for j, c2 in enumerate(s2): - insertions = previous_row[j + 1] + 1 - deletions = current_row[j] + 1 - substitutions = previous_row[j] + (c1 != c2) - current_row.append(min(insertions, deletions, substitutions)) - previous_row = current_row - - return previous_row[-1] - - def _generator(self): - layer = self.context.layers[self.config['primary']] - # TODO : YARA RULE HEX - rules = yarascan.YaraScan.process_yara_options({'yara_rules': "55 aa"}) - - for offset, _rule_name, _name, _value in layer.scan(context = self.context, - scanner = yarascan.YaraScanner(rules = rules)): - try: - yield 0, (format_hints.Hex(offset), _name) - - except exceptions.PagedInvalidAddressException: - pass - - def run(self): - return renderers.TreeGrid([ - ('Offset', format_hints.Hex), - ("Name", str) - ], self._generator()) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py new file mode 100644 index 000000000..3876d2ac8 --- /dev/null +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -0,0 +1,80 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from volatility3.framework import constants, interfaces, renderers, symbols +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mbr + +vollog = logging.getLogger(__name__) + +class MBRScan(interfaces.plugins.PluginInterface): + """ Scans for and parses potential Master Boot Records (MBRs) """ + + _required_framework_version = (2, 0, 1) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]) + ] + + def _generator(self): + kernel = self.context.modules[self.config['kernel']] + physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) + + layer = self.context.layers[physical_layer_name] + architecture = "intel" if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) else "intel64" + + symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, + config_path = self.config_path, + sub_path = "windows", + filename = "mbr", + class_types = { + 'PARTITION_TABLE': mbr.PARTITION_TABLE, + 'PARTITION_ENTRY': mbr.PARTITION_ENTRY + }) + + partition_table_object = symbol_table + constants.BANG + "PARTITION_TABLE" + + mbr_signature = b"\x55\xAA" + mbr_length = 0x200 + boot_code_length = 0x1B8 + + for offset, _value in layer.scan(context = self.context, scanner = scanners.MultiStringScanner(patterns = [mbr_signature])): + mbr_start_offset = offset - (mbr_length - len(mbr_signature)) + partition_table = self.context.object(partition_table_object, offset = mbr_start_offset, layer_name = layer.name) + + boot_code = layer.read(mbr_start_offset, boot_code_length, pad = True) + + if boot_code: + all_zeros = boot_code.count(b"\x00") == len(boot_code) + + if not all_zeros: + partition_type = partition_table.FirstEntry.PartitionType + + + if partition_type.is_valid_choice: + yield 0, ( + format_hints.Hex(offset), + partition_type.lookup(), + interfaces.renderers.Disassembly(boot_code, 0, architecture), + format_hints.HexBytes(boot_code) + ) + else: + vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + + def run(self): + return renderers.TreeGrid([ + ('Offset', format_hints.Hex), + ('PartitionType', str), + ("Disasm", interfaces.renderers.Disassembly), + ("Hexdump", format_hints.HexBytes) + ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index eeb97d332..9bcefe12e 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -6,106 +6,14 @@ from volatility3.framework import objects import struct -PartitionTypes = { - 0x00:"Empty", - 0x01:"FAT12,CHS", - 0x04:"FAT16 16-32MB,CHS", - 0x05:"Microsoft Extended", - 0x06:"FAT16 32MB,CHS", - 0x07:"NTFS", - 0x0b:"FAT32,CHS", - 0x0c:"FAT32,LBA", - 0x0e:"FAT16, 32MB-2GB,LBA", - 0x0f:"Microsoft Extended, LBA", - 0x11:"Hidden FAT12,CHS", - 0x14:"Hidden FAT16,16-32MB,CHS", - 0x16:"Hidden FAT16,32MB-2GB,CHS", - 0x18:"AST SmartSleep Partition", - 0x1b:"Hidden FAT32,CHS", - 0x1c:"Hidden FAT32,LBA", - 0x1e:"Hidden FAT16,32MB-2GB,LBA", - 0x27:"PQservice", - 0x39:"Plan 9 partition", - 0x3c:"PartitionMagic recovery partition", - 0x42:"Microsoft MBR,Dynamic Disk", - 0x44:"GoBack partition", - 0x51:"Novell", - 0x52:"CP/M", - 0x63:"Unix System V", - 0x64:"PC-ARMOUR protected partition", - 0x82:"Solaris x86 or Linux Swap", - 0x83:"Linux", - 0x84:"Hibernation", - 0x85:"Linux Extended", - 0x86:"NTFS Volume Set", - 0x87:"NTFS Volume Set", - 0x9f:"BSD/OS", - 0xa0:"Hibernation", - 0xa1:"Hibernation", - 0xa5:"FreeBSD", - 0xa6:"OpenBSD", - 0xa8:"Mac OSX", - 0xa9:"NetBSD", - 0xab:"Mac OSX Boot", - 0xaf:"MacOS X HFS", - 0xb7:"BSDI", - 0xb8:"BSDI Swap", - 0xbb:"Boot Wizard hidden", - 0xbe:"Solaris 8 boot partition", - 0xd8:"CP/M-86", - 0xde:"Dell PowerEdge Server utilities (FAT fs)", - 0xdf:"DG/UX virtual disk manager partition", - 0xeb:"BeOS BFS", - 0xee:"EFI GPT Disk", - 0xef:"EFI System Partition", - 0xfb:"VMWare File System", - 0xfc:"VMWare Swap", -} +class PARTITION_TABLE(objects.StructType): + + def get_disk_signature(self) -> str: + signature = self.DiskSignature.values + return signature class PARTITION_ENTRY(objects.StructType): - def get_value(self, char): - padded = "\x00\x00\x00" + str(char) - val = int(struct.unpack('>I', padded)[0]) - return val - - def get_type(self): - return PartitionTypes.get(self.get_value(self.PartitionType), "Invalid") - - def is_bootable(self): - return self.get_value(self.BootableFlag) == 0x80 - - def is_bootable_and_used(self): - return self.is_bootable() and self.is_used() - - def is_valid(self): - return self.get_type() != "Invalid" - - def is_used(self): - return self.get_type() != "Empty" and self.is_valid() - - def StartingSector(self): - return self.StartingCHS[1] % 64 - - def StartingCylinder(self): - return (self.StartingCHS[1] - self.StartingSector()) * 4 + self.StartingCHS[2] - - def EndingSector(self): - return self.EndingCHS[1] % 64 - - def EndingCylinder(self): - return (self.EndingCHS[1] - self.EndingSector()) * 4 + self.EndingCHS[2] - - def __str__(self): - processed_entry = "" - bootable = self.get_value(self.BootableFlag) - processed_entry = "Boot flag: {0:#x} {1}\n".format(bootable, "(Bootable)" if self.is_bootable() else '') - processed_entry += "Partition type: {0:#x} ({1})\n".format(self.get_value(self.PartitionType), self.get_type()) - processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.StartingLBA) - processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.StartingCylinder(), - self.StartingCHS[0], - self.StartingSector()) - processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.EndingCylinder(), - self.EndingCHS[0], - self.EndingSector()) - processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.SizeInSectors) - return processed_entry \ No newline at end of file + + def get_partition_type(self, type: int) -> str: + + return "Hello" diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 84162c96d..382af403e 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -2,9 +2,9 @@ "metadata": { "producer": { "version": "0.0.1", - "name": "Donghyun Kim", - "comment": "Using structures defined in File System Forensic Analysis pg 353+", - "datetime": "2022-01-03T13:37:00" + "name": "Donghyun Kim (@digitalisx99)", + "comment": "Using structures defined in File System Forensic Analysis pg 88+", + "datetime": "2022-03-05T10:53:00" }, "format": "6.1.0" }, @@ -32,6 +32,12 @@ "size": 4, "signed": false, "endian": "little" + }, + "int": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 4 }, "unsigned short": { "kind": "int", @@ -45,12 +51,204 @@ "signed": false, "endian": "little" }, + "char": { + "endian": "little", + "kind": "char", + "signed": true, + "size": 1 + }, "wchar": { "kind": "int", "size": 2, "signed": true, "endian": "little" } - }, - "symbols": {} + }, + "symbols": {}, + "enums": { + "BootableFlag":{ + "base": "unsigned char", + "constants": { + "Bootable": 0, + "Non-Bootable": 128 + }, + "size": 1 + }, + "PartitionTypes": { + "base": "unsigned char", + "constants": { + "Empty": 0, + "FAT12,CHS": 1, + "FAT16 16-32MB,CHS": 4, + "Microsoft Extended": 5, + "FAT16 32MB,CHS": 6, + "NTFS": 7, + "FAT32,CHS": 11, + "FAT32,LBA": 12, + "FAT16, 32MB-2GB,LBA": 14, + "Microsoft Extended, LBA": 15, + "Hidden FAT12,CHS": 17, + "Hidden FAT16,16-32MB,CHS": 20, + "Hidden FAT16,32MB-2GB,CHS": 22, + "AST SmartSleep Partition": 24, + "Hidden FAT32,CHS": 27, + "Hidden FAT32,LBA": 28, + "Hidden FAT16,32MB-2GB,LBA": 30, + "PQservice": 39, + "Plan 9 partition": 57, + "PartitionMagic recovery partition": 60, + "Microsoft MBR,Dynamic Disk": 66, + "GoBack partition": 68, + "Novell": 81, + "CP/M": 82, + "Unix System V": 99, + "PC-ARMOUR protected partition": 100, + "Solaris x86 or Linux Swap": 130, + "Linux": 131, + "Hibernation": 132, + "Linux Extended": 133, + "NTFS Volume Set": 134, + "NTFS Volume Set": 135, + "BSD/OS": 159, + "Hibernation": 160, + "Hibernation": 161, + "FreeBSD": 165, + "OpenBSD": 166, + "Mac OSX": 168, + "NetBSD": 169, + "Mac OSX Boot": 171, + "MacOS X HFS": 175, + "BSDI": 183, + "BSDI Swap": 184, + "Boot Wizard hidden": 187, + "Solaris 8 boot partition": 190, + "CP/M-86": 216, + "Dell PowerEdge Server utilities (FAT fs)": 222, + "DG/UX virtual disk manager partition": 223, + "BeOS BFS": 235, + "EFI GPT Disk": 238, + "EFI System Partition": 239, + "VMWare File System": 251, + "VMWare Swap": 252 + }, + "size": 1 + } + }, + "user_types": { + "PARTITION_ENTRY":{ + "fields": { + "BootableFlag": { + "offset": 0, + "type": { + "kind": "enum", + "name": "BootableFlag" + } + }, + "StartingCHS": { + "offset": 1, + "type": { + "count": 3, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "PartitionType": { + "offset": 4, + "type": { + "kind": "enum", + "name": "PartitionTypes" + } + }, + "EndingCHS": { + "offset": 5, + "type": { + "count": 3, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "StartingLBA": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "SizeInSectors": { + "offset": 12, + "type": { + "kind": "base", + "name": "int" + } + } + }, + "kind": "struct", + "size": 16 + }, + "PARTITION_TABLE":{ + "fields":{ + "DiskSignature": { + "offset": 440, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "Unused": { + "offset": 444, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "FirstEntry":{ + "offset": 446, + "type": { + "kind": "struct", + "name": "PARTITION_ENTRY" + } + }, + "SecondEntry":{ + "offset": 462, + "type": { + "kind": "struct", + "name": "PARTITION_ENTRY" + } + }, + "ThirdEntry":{ + "offset": 478, + "type": { + "kind": "struct", + "name": "PARTITION_ENTRY" + } + }, + "FourthEntry":{ + "offset": 494, + "type": { + "kind": "struct", + "name": "PARTITION_ENTRY" + } + }, + "Signature":{ + "offset": 510, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 512 + } + } } \ No newline at end of file From eda765d61d316d1d49d0008164a310743d710d5a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 16:12:32 +0900 Subject: [PATCH 100/158] Update MBR Partition Entry Object Function --- .../framework/plugins/windows/mbrscan.py | 32 ++++++------ .../symbols/windows/extensions/mbr.py | 51 +++++++++++++++++-- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 3876d2ac8..6bfed2bde 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -56,25 +56,27 @@ class MBRScan(interfaces.plugins.PluginInterface): if boot_code: all_zeros = boot_code.count(b"\x00") == len(boot_code) - - if not all_zeros: - partition_type = partition_table.FirstEntry.PartitionType - - if partition_type.is_valid_choice: - yield 0, ( - format_hints.Hex(offset), - partition_type.lookup(), - interfaces.renderers.Disassembly(boot_code, 0, architecture), - format_hints.HexBytes(boot_code) - ) + if not all_zeros: + partition_entry_list = ["FirstEntry", "SecondEntry", "ThirdEntry", "FourthEntry"] + #partition_type = getattr(partition_table, "FirstEntry").PartitionType + yield 0, ( + format_hints.Hex(offset), + partition_table.FirstEntry.get_bootable_flag(), + partition_table.FirstEntry.get_partition_type(), + format_hints.Hex(partition_table.FirstEntry.get_starting_chs()) + #interfaces.renderers.Disassembly(boot_code, 0, architecture), + #format_hints.HexBytes(boot_code) + ) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") def run(self): return renderers.TreeGrid([ - ('Offset', format_hints.Hex), - ('PartitionType', str), - ("Disasm", interfaces.renderers.Disassembly), - ("Hexdump", format_hints.HexBytes) + ("Offset", format_hints.Hex), + ("Bootable", bool), + ("Partition Type", str), + ("Starting CHS",format_hints.Hex) + #("Disasm", interfaces.renderers.Disassembly), + #("Hexdump", format_hints.HexBytes) ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 9bcefe12e..c02d741df 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -4,8 +4,6 @@ from volatility3.framework import objects -import struct - class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: @@ -14,6 +12,49 @@ class PARTITION_TABLE(objects.StructType): class PARTITION_ENTRY(objects.StructType): - def get_partition_type(self, type: int) -> str: - - return "Hello" + def get_bootable_flag(self) -> int: + return self.BootableFlag + + def is_bootable(self) -> bool: + return False if not (self.BootableFlag == 0x80) else True + + def get_partition_type(self) -> str: + return self.PartitionType.lookup() if self.PartitionType.is_valid_choice else "Not Defined PartitionType" + + def get_starting_chs(self): + return self.StartingCHS[0] + + def get_ending_chs(self): + return self.EndingCHS[0] + + def get_starting_sector(self): + return self.StartingCHS[1] % 64 + + def get_starting_cylinder(self): + return (self.StartingCHS[1] - self.get_starting_sector()) * 4 + self.StartingCHS[2] + + def get_ending_sector(self): + return self.EndingCHS[1] % 64 + + def get_ending_cylinder(self): + return (self.EndingCHS[1] - self.get_ending_sector()) * 4 + self.EndingCHS[2] + + def get_starting_lba(self): + return self.StartingLBA + + def get_size_in_sectors(self): + return self.SizeInSectors + + def __str__(self): + processed_entry = "" + processed_entry = "Boot flag: {0:#x} {1}\n".format(self.is_bootable(), "(Bootable)" if self.is_bootable() else '') + processed_entry += "Partition type: {0:#x} ({1})\n".format(self.get_value(self.PartitionType), self.get_type()) + processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.StartingLBA) + processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.StartingCylinder(), + self.StartingCHS[0], + self.StartingSector()) + processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.EndingCylinder(), + self.EndingCHS[0], + self.EndingSector()) + processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.SizeInSectors) + return processed_entry From e0a512e9ff17a81b7c36c79da01c894eab72bbc5 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 16:13:25 +0900 Subject: [PATCH 101/158] Add EOF of MBR Symbol --- volatility3/framework/symbols/windows/mft.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index e5de8f3fa..6881c92be 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -466,4 +466,4 @@ "size": 1024 } } -} \ No newline at end of file +} From b01333115b06ba106250916e2390888870794b13 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 17:08:27 +0900 Subject: [PATCH 102/158] __str__ Formatting --- .../framework/plugins/windows/mbrscan.py | 26 ++++++------ .../symbols/windows/extensions/mbr.py | 40 +++++++++++++------ .../framework/symbols/windows/mbr.json | 2 +- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 6bfed2bde..beda342fb 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -58,15 +58,18 @@ class MBRScan(interfaces.plugins.PluginInterface): all_zeros = boot_code.count(b"\x00") == len(boot_code) if not all_zeros: - partition_entry_list = ["FirstEntry", "SecondEntry", "ThirdEntry", "FourthEntry"] - #partition_type = getattr(partition_table, "FirstEntry").PartitionType + + first_entry = partition_table.FirstEntry + second_entry = partition_table.SecondEntry + third_entry = partition_table.ThirdEntry + fourth_entry = partition_table.FourthEntry + yield 0, ( format_hints.Hex(offset), - partition_table.FirstEntry.get_bootable_flag(), - partition_table.FirstEntry.get_partition_type(), - format_hints.Hex(partition_table.FirstEntry.get_starting_chs()) - #interfaces.renderers.Disassembly(boot_code, 0, architecture), - #format_hints.HexBytes(boot_code) + partition_table.get_disk_signature(), + str(partition_table.FirstEntry), + interfaces.renderers.Disassembly(boot_code, 0, architecture), + format_hints.HexBytes(boot_code) ) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") @@ -74,9 +77,8 @@ class MBRScan(interfaces.plugins.PluginInterface): def run(self): return renderers.TreeGrid([ ("Offset", format_hints.Hex), - ("Bootable", bool), - ("Partition Type", str), - ("Starting CHS",format_hints.Hex) - #("Disasm", interfaces.renderers.Disassembly), - #("Hexdump", format_hints.HexBytes) + ("Disk Signature", str), + ("First Entry", str), + ("Disasm", interfaces.renderers.Disassembly), + ("Hexdump", format_hints.HexBytes) ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index c02d741df..e4aaefad1 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -7,8 +7,12 @@ from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: - signature = self.DiskSignature.values - return signature + return "{0:02x}-{1:02x}-{2:02x}-{3:02x}".format( + self.DiskSignature[0], + self.DiskSignature[1], + self.DiskSignature[2], + self.DiskSignature[3] + ) class PARTITION_ENTRY(objects.StructType): @@ -46,15 +50,25 @@ class PARTITION_ENTRY(objects.StructType): return self.SizeInSectors def __str__(self): - processed_entry = "" - processed_entry = "Boot flag: {0:#x} {1}\n".format(self.is_bootable(), "(Bootable)" if self.is_bootable() else '') - processed_entry += "Partition type: {0:#x} ({1})\n".format(self.get_value(self.PartitionType), self.get_type()) - processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.StartingLBA) - processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.StartingCylinder(), - self.StartingCHS[0], - self.StartingSector()) - processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format(self.EndingCylinder(), - self.EndingCHS[0], - self.EndingSector()) - processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.SizeInSectors) + processed_entry = "========= Partition Info =========\n" + processed_entry += "Boot Flag: {0:#x} {1}\n".format( + self.is_bootable(), + "(Bootable)" if self.is_bootable() else '' + ) + processed_entry += "Partition Type: {0:#x} ({1})\n".format( + self.PartitionType, + self.get_partition_type() + ) + processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.get_starting_lba()) + processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format( + self.get_starting_cylinder(), + self.get_starting_chs(), + self.get_starting_sector() + ) + processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format( + self.get_ending_cylinder(), + self.get_ending_chs(), + self.get_ending_sector() + ) + processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.get_size_in_sectors()) return processed_entry diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 382af403e..9173633d1 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -251,4 +251,4 @@ "size": 512 } } -} \ No newline at end of file +} From 5de6462fae23f4702d5b7c209e9d151c6589dd91 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 17:09:36 +0900 Subject: [PATCH 103/158] Restore mft.json --- volatility3/framework/symbols/windows/mft.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index 6881c92be..e5de8f3fa 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -466,4 +466,4 @@ "size": 1024 } } -} +} \ No newline at end of file From b6a14e6de4fab2ec2473b1f3492a6a69f063f1b3 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Mar 2022 23:42:21 +0900 Subject: [PATCH 104/158] Add Symbol code comment, hash --- .../framework/plugins/windows/mbrscan.py | 29 +++++++++++++------ .../symbols/windows/extensions/mbr.py | 26 +++++++++++++---- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index beda342fb..fcc9dabb1 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -3,6 +3,7 @@ # import logging +import hashlib from volatility3.framework import constants, interfaces, renderers, symbols from volatility3.framework.configuration import requirements @@ -52,22 +53,30 @@ class MBRScan(interfaces.plugins.PluginInterface): mbr_start_offset = offset - (mbr_length - len(mbr_signature)) partition_table = self.context.object(partition_table_object, offset = mbr_start_offset, layer_name = layer.name) - boot_code = layer.read(mbr_start_offset, boot_code_length, pad = True) + full_mbr = layer.read(mbr_start_offset, mbr_length, pad = True) + boot_code = full_mbr[:boot_code_length] if boot_code: all_zeros = boot_code.count(b"\x00") == len(boot_code) if not all_zeros: - - first_entry = partition_table.FirstEntry - second_entry = partition_table.SecondEntry - third_entry = partition_table.ThirdEntry - fourth_entry = partition_table.FourthEntry + bootcode_hash = hashlib.md5(boot_code).hexdigest() + full_bootcode_hash = hashlib.md5(full_mbr).hexdigest() + partition_entries = [ partition_table.FirstEntry, partition_table.SecondEntry, + partition_table.ThirdEntry, partition_table.FourthEntry ] + partition_info = "" + + for index, partition_entry_object in enumerate(partition_entries): + partition_entry_object.set_index(index) + partition_info += str(partition_entry_object) + yield 0, ( format_hints.Hex(offset), partition_table.get_disk_signature(), - str(partition_table.FirstEntry), + bootcode_hash, + full_bootcode_hash, + partition_info, interfaces.renderers.Disassembly(boot_code, 0, architecture), format_hints.HexBytes(boot_code) ) @@ -76,9 +85,11 @@ class MBRScan(interfaces.plugins.PluginInterface): def run(self): return renderers.TreeGrid([ - ("Offset", format_hints.Hex), + ("Potential MBR at Physical Offset", format_hints.Hex), ("Disk Signature", str), - ("First Entry", str), + ("Bootcode md5", str), + ("Bootcode (FULL) md5", str), + ("Partition Entries Info", str), ("Disasm", interfaces.renderers.Disassembly), ("Hexdump", format_hints.HexBytes) ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index e4aaefad1..8e02db822 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -7,6 +7,7 @@ from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: + """Get Disk Signature (GUID).""" return "{0:02x}-{1:02x}-{2:02x}-{3:02x}".format( self.DiskSignature[0], self.DiskSignature[1], @@ -15,42 +16,57 @@ class PARTITION_TABLE(objects.StructType): ) class PARTITION_ENTRY(objects.StructType): + + def set_index(self, index:int): + self.index = index def get_bootable_flag(self) -> int: + """Get Bootable Flag.""" return self.BootableFlag def is_bootable(self) -> bool: + """Check Bootable Partition.""" return False if not (self.BootableFlag == 0x80) else True def get_partition_type(self) -> str: + """Get Partition Type.""" return self.PartitionType.lookup() if self.PartitionType.is_valid_choice else "Not Defined PartitionType" def get_starting_chs(self): + """Get Starting CHS (Cylinder Header Sector) Address.""" return self.StartingCHS[0] def get_ending_chs(self): + """Get Ending CHS (Cylinder Header Sector) Address.""" return self.EndingCHS[0] def get_starting_sector(self): + """Get Starting Sector.""" return self.StartingCHS[1] % 64 - def get_starting_cylinder(self): - return (self.StartingCHS[1] - self.get_starting_sector()) * 4 + self.StartingCHS[2] - def get_ending_sector(self): + """Get Ending Sector.""" return self.EndingCHS[1] % 64 + def get_starting_cylinder(self): + """Get Starting Cylinder.""" + return (self.StartingCHS[1] - self.get_starting_sector()) * 4 + self.StartingCHS[2] + def get_ending_cylinder(self): + """Get Ending Cylinder.""" return (self.EndingCHS[1] - self.get_ending_sector()) * 4 + self.EndingCHS[2] def get_starting_lba(self): + """Get Starting LBA (Logical Block Addressing).""" return self.StartingLBA def get_size_in_sectors(self): + """Get Size in Sectors.""" return self.SizeInSectors def __str__(self): - processed_entry = "========= Partition Info =========\n" + """Get overall of Partition Entry Info""" + processed_entry = "\n===== Partition Table #{0} =====\n".format(self.index+1) processed_entry += "Boot Flag: {0:#x} {1}\n".format( self.is_bootable(), "(Bootable)" if self.is_bootable() else '' @@ -70,5 +86,5 @@ class PARTITION_ENTRY(objects.StructType): self.get_ending_chs(), self.get_ending_sector() ) - processed_entry += "Size in sectors: {0:#x} ({0})\n\n".format(self.get_size_in_sectors()) + processed_entry += "Size in Sectors: {0:#x} ({0})\n".format(self.get_size_in_sectors()) return processed_entry From 7c00b2f4ea04a9d7fa171ee37301e090d42ae383 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 10 Mar 2022 00:13:46 +0900 Subject: [PATCH 105/158] Add Code Comment, Hash Funtion, Exception --- .../framework/plugins/windows/mbrscan.py | 78 +++++++++++-------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index fcc9dabb1..f0d52f418 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -5,7 +5,7 @@ import logging import hashlib -from volatility3.framework import constants, interfaces, renderers, symbols +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints @@ -27,13 +27,19 @@ class MBRScan(interfaces.plugins.PluginInterface): architectures = ["Intel32", "Intel64"]) ] + @classmethod + def get_hash(cls, data:bytes) -> str: + return hashlib.md5(data).hexdigest() + def _generator(self): kernel = self.context.modules[self.config['kernel']] physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) + # Decide of Memory Dump Architecture layer = self.context.layers[physical_layer_name] architecture = "intel" if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) else "intel64" + # Read in the Symbol File symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, config_path = self.config_path, sub_path = "windows", @@ -45,43 +51,51 @@ class MBRScan(interfaces.plugins.PluginInterface): partition_table_object = symbol_table + constants.BANG + "PARTITION_TABLE" + # Define Signature and Data Length mbr_signature = b"\x55\xAA" mbr_length = 0x200 - boot_code_length = 0x1B8 + bootcode_length = 0x1B8 + # Scan the Layer for Raw Master Boot Record (MBR) and parse the fields for offset, _value in layer.scan(context = self.context, scanner = scanners.MultiStringScanner(patterns = [mbr_signature])): - mbr_start_offset = offset - (mbr_length - len(mbr_signature)) - partition_table = self.context.object(partition_table_object, offset = mbr_start_offset, layer_name = layer.name) + try: + mbr_start_offset = offset - (mbr_length - len(mbr_signature)) + partition_table = self.context.object(partition_table_object, offset = mbr_start_offset, layer_name = layer.name) - full_mbr = layer.read(mbr_start_offset, mbr_length, pad = True) - boot_code = full_mbr[:boot_code_length] - - if boot_code: - all_zeros = boot_code.count(b"\x00") == len(boot_code) - - if not all_zeros: - bootcode_hash = hashlib.md5(boot_code).hexdigest() - full_bootcode_hash = hashlib.md5(full_mbr).hexdigest() - - partition_entries = [ partition_table.FirstEntry, partition_table.SecondEntry, - partition_table.ThirdEntry, partition_table.FourthEntry ] - partition_info = "" - - for index, partition_entry_object in enumerate(partition_entries): - partition_entry_object.set_index(index) - partition_info += str(partition_entry_object) + # Extract only BootCode + full_mbr = layer.read(mbr_start_offset, mbr_length, pad = True) + bootcode = full_mbr[:bootcode_length] - yield 0, ( - format_hints.Hex(offset), - partition_table.get_disk_signature(), - bootcode_hash, - full_bootcode_hash, - partition_info, - interfaces.renderers.Disassembly(boot_code, 0, architecture), - format_hints.HexBytes(boot_code) - ) - else: - vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + if bootcode: + all_zeros = bootcode.count(b"\x00") == len(bootcode) + + if not all_zeros: + partition_entries = [ + partition_table.FirstEntry, + partition_table.SecondEntry, + partition_table.ThirdEntry, + partition_table.FourthEntry + ] + partition_info = "\n" + + for index, partition_entry_object in enumerate(partition_entries): + partition_entry_object.set_index(index) + partition_info += str(partition_entry_object) + + yield 0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_info, + interfaces.renderers.Disassembly(bootcode, 0, architecture), + format_hints.HexBytes(bootcode) + ) + else: + vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + + except exceptions.PagedInvalidAddressException: + pass def run(self): return renderers.TreeGrid([ From a35fa04f00929dc1a4e50c3db5e107bdcf6b49b4 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 10 Mar 2022 01:12:49 +0900 Subject: [PATCH 106/158] Update BootableFlag Symbol --- .../symbols/windows/extensions/mbr.py | 7 ++++-- .../framework/symbols/windows/mbr.json | 22 ++++--------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 8e02db822..7cb4c1463 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -2,6 +2,8 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import struct + from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): @@ -18,6 +20,7 @@ class PARTITION_TABLE(objects.StructType): class PARTITION_ENTRY(objects.StructType): def set_index(self, index:int): + """Set Partition Entry Index.""" self.index = index def get_bootable_flag(self) -> int: @@ -26,7 +29,7 @@ class PARTITION_ENTRY(objects.StructType): def is_bootable(self) -> bool: """Check Bootable Partition.""" - return False if not (self.BootableFlag == 0x80) else True + return False if not (self.get_bootable_flag() == 0x80) else True def get_partition_type(self) -> str: """Get Partition Type.""" @@ -68,7 +71,7 @@ class PARTITION_ENTRY(objects.StructType): """Get overall of Partition Entry Info""" processed_entry = "\n===== Partition Table #{0} =====\n".format(self.index+1) processed_entry += "Boot Flag: {0:#x} {1}\n".format( - self.is_bootable(), + self.get_bootable_flag(), "(Bootable)" if self.is_bootable() else '' ) processed_entry += "Partition Type: {0:#x} ({1})\n".format( diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 9173633d1..122c020c3 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -34,10 +34,10 @@ "endian": "little" }, "int": { - "endian": "little", "kind": "int", + "size": 4, "signed": true, - "size": 4 + "endian": "little" }, "unsigned short": { "kind": "int", @@ -51,12 +51,6 @@ "signed": false, "endian": "little" }, - "char": { - "endian": "little", - "kind": "char", - "signed": true, - "size": 1 - }, "wchar": { "kind": "int", "size": 2, @@ -66,14 +60,6 @@ }, "symbols": {}, "enums": { - "BootableFlag":{ - "base": "unsigned char", - "constants": { - "Bootable": 0, - "Non-Bootable": 128 - }, - "size": 1 - }, "PartitionTypes": { "base": "unsigned char", "constants": { @@ -140,8 +126,8 @@ "BootableFlag": { "offset": 0, "type": { - "kind": "enum", - "name": "BootableFlag" + "kind": "base", + "name": "unsigned char" } }, "StartingCHS": { From 1b71aad3669ea4325aecf564ffb1e1c5999de139 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 10 Mar 2022 01:12:49 +0900 Subject: [PATCH 107/158] Update BootableFlag Symbol --- .../symbols/windows/extensions/mbr.py | 7 ++++-- .../framework/symbols/windows/mbr.json | 22 ++++--------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 8e02db822..7cb4c1463 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -2,6 +2,8 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import struct + from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): @@ -18,6 +20,7 @@ class PARTITION_TABLE(objects.StructType): class PARTITION_ENTRY(objects.StructType): def set_index(self, index:int): + """Set Partition Entry Index.""" self.index = index def get_bootable_flag(self) -> int: @@ -26,7 +29,7 @@ class PARTITION_ENTRY(objects.StructType): def is_bootable(self) -> bool: """Check Bootable Partition.""" - return False if not (self.BootableFlag == 0x80) else True + return False if not (self.get_bootable_flag() == 0x80) else True def get_partition_type(self) -> str: """Get Partition Type.""" @@ -68,7 +71,7 @@ class PARTITION_ENTRY(objects.StructType): """Get overall of Partition Entry Info""" processed_entry = "\n===== Partition Table #{0} =====\n".format(self.index+1) processed_entry += "Boot Flag: {0:#x} {1}\n".format( - self.is_bootable(), + self.get_bootable_flag(), "(Bootable)" if self.is_bootable() else '' ) processed_entry += "Partition Type: {0:#x} ({1})\n".format( diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 9173633d1..122c020c3 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -34,10 +34,10 @@ "endian": "little" }, "int": { - "endian": "little", "kind": "int", + "size": 4, "signed": true, - "size": 4 + "endian": "little" }, "unsigned short": { "kind": "int", @@ -51,12 +51,6 @@ "signed": false, "endian": "little" }, - "char": { - "endian": "little", - "kind": "char", - "signed": true, - "size": 1 - }, "wchar": { "kind": "int", "size": 2, @@ -66,14 +60,6 @@ }, "symbols": {}, "enums": { - "BootableFlag":{ - "base": "unsigned char", - "constants": { - "Bootable": 0, - "Non-Bootable": 128 - }, - "size": 1 - }, "PartitionTypes": { "base": "unsigned char", "constants": { @@ -140,8 +126,8 @@ "BootableFlag": { "offset": 0, "type": { - "kind": "enum", - "name": "BootableFlag" + "kind": "base", + "name": "unsigned char" } }, "StartingCHS": { From eba7ad1c0ddd875c4027a2b2e7c189ff74377820 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Mar 2022 21:13:25 +0000 Subject: [PATCH 108/158] Renderers: Use built-in python CSV support --- volatility3/cli/text_renderer.py | 44 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 8663bb995..eadf10d37 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -1,6 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import csv import datetime import json import logging @@ -8,7 +9,7 @@ import random import string import sys from functools import wraps -from typing import Callable, Any, List, Tuple, Dict +from typing import Any, Callable, Dict, List, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.renderers import format_hints @@ -66,7 +67,6 @@ def multitypedata_as_text(value: format_hints.MultiTypeData) -> str: def optional(func: Callable) -> Callable: - @wraps(func) def wrapped(x: Any) -> str: if isinstance(x, interfaces.renderers.BaseAbsentValue): @@ -80,7 +80,6 @@ def optional(func: Callable) -> Callable: def quoted_optional(func: Callable) -> Callable: - @wraps(func) def wrapped(x: Any) -> str: result = optional(func)(x) @@ -193,16 +192,17 @@ class NoneRenderer(CLIRenderer): if not grid.populated: grid.populate(lambda x, y: True, True) + class CSVRenderer(CLIRenderer): _type_renderers = { - format_hints.Bin: quoted_optional(lambda x: f"0b{x:b}"), - format_hints.Hex: quoted_optional(lambda x: f"0x{x:x}"), - format_hints.HexBytes: quoted_optional(hex_bytes_as_text), - format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), - interfaces.renderers.Disassembly: quoted_optional(display_disassembly), - bytes: quoted_optional(lambda x: " ".join([f"{b:02x}" for b in x])), - datetime.datetime: quoted_optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), - 'default': quoted_optional(lambda x: f"{x}") + format_hints.Bin: optional(lambda x: f"0b{x:b}"), + format_hints.Hex: optional(lambda x: f"0x{x:x}"), + format_hints.HexBytes: optional(hex_bytes_as_text), + format_hints.MultiTypeData: optional(multitypedata_as_text), + interfaces.renderers.Disassembly: optional(display_disassembly), + bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), + 'default': optional(lambda x: f"{x}") } name = "csv" @@ -219,28 +219,27 @@ class CSVRenderer(CLIRenderer): """ outfd = sys.stdout - line = ['"TreeDepth"'] + header_list = ['TreeDepth'] for column in grid.columns: # Ignore the type because namedtuples don't realize they have accessible attributes - line.append("{}".format('"' + column.name + '"')) - outfd.write(f"{','.join(line)}") + header_list.append(f"{column.name}") + + writer = csv.DictWriter(outfd, header_list) def visitor(node: interfaces.renderers.TreeNode, accumulator): - accumulator.write("\n") # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case - accumulator.write(str(max(0, node.path_depth - 1)) + ",") - line = [] + row = {'TreeDepth': str(max(0, node.path_depth - 1))} for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get(column.type, self._type_renderers['default']) - line.append(renderer(node.values[column_index])) - accumulator.write(f"{','.join(line)}") + row[f'{column.name}'] = renderer(node.values[column_index]) + accumulator.writerow(row) return accumulator if not grid.populated: - grid.populate(visitor, outfd) + grid.populate(visitor, writer) else: - grid.visit(node = None, function = visitor, initial_accumulator = outfd) + grid.visit(node = None, function = visitor, initial_accumulator = writer) outfd.write("\n") @@ -274,7 +273,8 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths = dict([(column.name, len(column.name)) for column in grid.columns]) def visitor( - node: interfaces.renderers.TreeNode, accumulator: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]] + node: interfaces.renderers.TreeNode, + accumulator: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]] ) -> List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]]: # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case max_column_widths[tree_indent_column] = max(max_column_widths.get(tree_indent_column, 0), node.path_depth) From 0de8c645a4d2159f36c960a6e2c27981798d8599 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 13 Mar 2022 21:20:04 +0000 Subject: [PATCH 109/158] Renderers: Add column headers for CSV --- volatility3/cli/text_renderer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index eadf10d37..8e07d58d1 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -225,6 +225,7 @@ class CSVRenderer(CLIRenderer): header_list.append(f"{column.name}") writer = csv.DictWriter(outfd, header_list) + writer.writeheader() def visitor(node: interfaces.renderers.TreeNode, accumulator): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case From f97348616f559f0849264fbba2b8e9bb8cdae5b8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 14 Mar 2022 09:47:34 +0900 Subject: [PATCH 110/158] Define 'all_zero' default value, Update output column name --- volatility3/framework/plugins/windows/mbrscan.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index f0d52f418..8f0d7a3a4 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -66,6 +66,8 @@ class MBRScan(interfaces.plugins.PluginInterface): full_mbr = layer.read(mbr_start_offset, mbr_length, pad = True) bootcode = full_mbr[:bootcode_length] + all_zeros = None + if bootcode: all_zeros = bootcode.count(b"\x00") == len(bootcode) @@ -101,8 +103,8 @@ class MBRScan(interfaces.plugins.PluginInterface): return renderers.TreeGrid([ ("Potential MBR at Physical Offset", format_hints.Hex), ("Disk Signature", str), - ("Bootcode md5", str), - ("Bootcode (FULL) md5", str), + ("Bootcode MD5", str), + ("Full MBR MD5", str), ("Partition Entries Info", str), ("Disasm", interfaces.renderers.Disassembly), ("Hexdump", format_hints.HexBytes) From 7c89fc3f070814ba27bd61fdd1141cb1eb0b883c Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Tue, 15 Mar 2022 14:22:36 +0200 Subject: [PATCH 111/158] bug fix :( --- volatility3/framework/plugins/windows/malfind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index bfd29a254..700ced8ee 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -53,7 +53,7 @@ class Malfind(interfaces.plugins.PluginInterface): """ CHUNK_SIZE = 0x1000 - all_zero_page = "\x00" * CHUNK_SIZE + all_zero_page = b"\x00" * CHUNK_SIZE offset = 0 vad_length = vad.get_end() - vad.get_start() From fa723ec134e881cc7c3a4987bb1b6eb176a45ac8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Mar 2022 01:29:20 +0000 Subject: [PATCH 112/158] CLI: Implement specifying a config name to write --- volatility3/cli/__init__.py | 25 +++++++++++++++++++++++-- volatility3/cli/volshell/__init__.py | 15 +++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 4cdbd26e8..8cf9621a3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,6 +19,7 @@ import os import sys import tempfile import traceback +from datetime import datetime from typing import Any, Dict, Type, Union from urllib import parse, request @@ -157,6 +158,10 @@ class CommandLine: help = "Write configuration JSON file out to config.json", default = False, action = 'store_true') + parser.add_argument("--save-config", + help = "Save configuration JSON file to a file", + default = None, + type = str) parser.add_argument("--clear-cache", help = "Clears out all short-term cached items", default = False, @@ -320,8 +325,15 @@ class CommandLine: self.file_handler_class_factory()) if args.write_config: - vollog.debug("Writing out configuration data to config.json") - with open("config.json", "w") as f: + args.save_config = 'config.json' + if args.save_config: + vollog.debug("Writing out configuration data to {args.save_config}") + if os.path.exists(os.path.abspath(args.save_config)): + # Backup existing file + backup_filename = self.find_backup_filename(args.save_config) + vollog.debug(f"Backing up existing file to {backup_filename}") + os.rename(args.save_config, backup_filename) + with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) @@ -334,6 +346,15 @@ class CommandLine: except (exceptions.VolatilityException) as excp: self.process_exceptions(excp) + def find_backup_filename(self, original: str): + suffix = "" + new_name = f"{original}.{datetime.strftime(datetime.today(), '%y%m%d')}.bak" + while os.path.exists(f"{new_name}{suffix}"): + if not suffix: + suffix = 1 + suffix += 1 + return f"{new_name}{suffix}" + @classmethod def location_from_file(cls, filename: str) -> str: """Returns the URL location from a file parameter (which may be a URL) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 812d44337..42e82e5bf 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -85,6 +85,10 @@ class VolShell(cli.CommandLine): help = "Write configuration JSON file out to config.json", default = False, action = 'store_true') + parser.add_argument("--save-config", + help = "Save configuration JSON file to a file", + default = None, + type = str) parser.add_argument("--clear-cache", help = "Clears out all short-term cached items", default = False, @@ -234,8 +238,15 @@ class VolShell(cli.CommandLine): self.file_handler_class_factory()) if args.write_config: - vollog.debug("Writing out configuration data to config.json") - with open("config.json", "w") as f: + args.save_config = 'config.json' + if args.save_config: + vollog.debug("Writing out configuration data to {args.save_config}") + if os.path.exists(os.path.abspath(args.save_config)): + # Backup existing file + backup_filename = self.find_backup_filename(args.save_config) + vollog.debug(f"Backing up existing file to {backup_filename}") + os.rename(args.save_config, backup_filename) + with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) From eb38756dbebbd3a6cae366ab5cc5b045faa00b10 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Mar 2022 01:35:47 +0000 Subject: [PATCH 113/158] CLI: Add deprecation warning to --write-config --- volatility3/cli/__init__.py | 5 ++++- volatility3/cli/volshell/__init__.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 8cf9621a3..d22fa154a 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -324,11 +324,14 @@ class CommandLine: constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) + backup_filename = True if args.write_config: + vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' + backup_filename = False if args.save_config: vollog.debug("Writing out configuration data to {args.save_config}") - if os.path.exists(os.path.abspath(args.save_config)): + if os.path.exists(os.path.abspath(args.save_config)) and backup_filename: # Backup existing file backup_filename = self.find_backup_filename(args.save_config) vollog.debug(f"Backing up existing file to {backup_filename}") diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 42e82e5bf..fbf79b117 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -237,11 +237,14 @@ class VolShell(cli.CommandLine): constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) + backup_filename = True if args.write_config: + vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' + backup_filename = False if args.save_config: vollog.debug("Writing out configuration data to {args.save_config}") - if os.path.exists(os.path.abspath(args.save_config)): + if os.path.exists(os.path.abspath(args.save_config)) and backup_filename: # Backup existing file backup_filename = self.find_backup_filename(args.save_config) vollog.debug(f"Backing up existing file to {backup_filename}") From 0f4f4f2b3ac652c0a2ed1a3eaf0ef464f8f939fa Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Mar 2022 01:41:18 +0000 Subject: [PATCH 114/158] CLI: Add configuration option for blatting over config files --- volatility3/cli/__init__.py | 2 +- volatility3/cli/volshell/__init__.py | 2 +- volatility3/framework/constants/__init__.py | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index d22fa154a..1933607c3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -324,7 +324,7 @@ class CommandLine: constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) - backup_filename = True + backup_filename = constants.BACKUP_EXISTING_CONFIG_OUTPUT if args.write_config: vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index fbf79b117..f3bed1b73 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -237,7 +237,7 @@ class VolShell(cli.CommandLine): constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) - backup_filename = True + backup_filename = constants.BACKUP_EXISTING_CONFIG_OUTPUT if args.write_config: vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 665e62d30..4af4408af 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,7 +9,7 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys -from typing import Optional, Callable +from typing import Callable, Optional import volatility3.framework.constants.linux import volatility3.framework.constants.windows @@ -80,6 +80,7 @@ ProgressCallback = Optional[Callable[[float, str], None]] OS_CATEGORIES = ['windows', 'mac', 'linux'] + class Parallelism(enum.IntEnum): """An enumeration listing the different types of parallelism applied to volatility.""" @@ -100,3 +101,6 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" + +BACKUP_EXISTING_CONFIG_OUTPUT = True +"""Whether existing files are backed up or overwritten when writing configuration output""" From 3ab3fa3ed8bbcd6f5b5e4a8d9f5361c587086249 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 16 Mar 2022 17:39:06 +0900 Subject: [PATCH 115/158] Remove Windows Symbol Initialize unuse import --- volatility3/framework/symbols/windows/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index 3aa607574..f09dadedf 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import volatility3.framework.symbols.windows.extensions.pool + from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import extensions from volatility3.framework.symbols.windows.extensions import registry, pool From 1645443d3bad7e672dec09d22ddc95f5f4d7e272 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 16 Mar 2022 22:30:57 +0900 Subject: [PATCH 116/158] Remove Hexdump Column --- volatility3/framework/plugins/windows/mbrscan.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 8f0d7a3a4..3db107567 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -90,8 +90,7 @@ class MBRScan(interfaces.plugins.PluginInterface): self.get_hash(bootcode), self.get_hash(full_mbr), partition_info, - interfaces.renderers.Disassembly(bootcode, 0, architecture), - format_hints.HexBytes(bootcode) + interfaces.renderers.Disassembly(bootcode, 0, architecture) ) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") @@ -106,6 +105,5 @@ class MBRScan(interfaces.plugins.PluginInterface): ("Bootcode MD5", str), ("Full MBR MD5", str), ("Partition Entries Info", str), - ("Disasm", interfaces.renderers.Disassembly), - ("Hexdump", format_hints.HexBytes) + ("Disasm", interfaces.renderers.Disassembly) ], self._generator()) From b02783baf11861847681fc8a5362c173a7772baf Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 16 Mar 2022 22:37:44 +0900 Subject: [PATCH 117/158] Remove index initialize, __str__ method by partition entry logic update --- .../symbols/windows/extensions/mbr.py | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 7cb4c1463..3fdb67ee3 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -18,10 +18,6 @@ class PARTITION_TABLE(objects.StructType): ) class PARTITION_ENTRY(objects.StructType): - - def set_index(self, index:int): - """Set Partition Entry Index.""" - self.index = index def get_bootable_flag(self) -> int: """Get Bootable Flag.""" @@ -66,28 +62,3 @@ class PARTITION_ENTRY(objects.StructType): def get_size_in_sectors(self): """Get Size in Sectors.""" return self.SizeInSectors - - def __str__(self): - """Get overall of Partition Entry Info""" - processed_entry = "\n===== Partition Table #{0} =====\n".format(self.index+1) - processed_entry += "Boot Flag: {0:#x} {1}\n".format( - self.get_bootable_flag(), - "(Bootable)" if self.is_bootable() else '' - ) - processed_entry += "Partition Type: {0:#x} ({1})\n".format( - self.PartitionType, - self.get_partition_type() - ) - processed_entry += "Starting Sector (LBA): {0:#x} ({0})\n".format(self.get_starting_lba()) - processed_entry += "Starting CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format( - self.get_starting_cylinder(), - self.get_starting_chs(), - self.get_starting_sector() - ) - processed_entry += "Ending CHS: Cylinder: {0} Head: {1} Sector: {2}\n".format( - self.get_ending_cylinder(), - self.get_ending_chs(), - self.get_ending_sector() - ) - processed_entry += "Size in Sectors: {0:#x} ({0})\n".format(self.get_size_in_sectors()) - return processed_entry From 02394f89a8120d9cfc09bdf1e123d3a3cd3984a5 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 17 Mar 2022 02:22:51 +0900 Subject: [PATCH 118/158] Add return type hint, Add full option, Update yield data --- .../framework/plugins/windows/mbrscan.py | 184 +++++++++++++++--- 1 file changed, 157 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 3db107567..60b403ae0 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -5,6 +5,8 @@ import logging import hashlib +from typing import Iterator, List, Tuple + from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners @@ -21,17 +23,21 @@ class MBRScan(interfaces.plugins.PluginInterface): _version = (1, 0, 0) @classmethod - def get_requirements(cls): + def get_requirements(cls)-> List[interfaces.configuration.RequirementInterface]: return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]) + architectures = ["Intel32", "Intel64"]), + requirements.BooleanRequirement(name = 'full', + description ="It analyzes and provides all the information in the partition entry. (It returns a lot of information, so we recommend you render it in CSV.)", + default = False, + optional = True) ] @classmethod def get_hash(cls, data:bytes) -> str: return hashlib.md5(data).hexdigest() - def _generator(self): + def _generator(self) -> Iterator[Tuple]: kernel = self.context.modules[self.config['kernel']] physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) @@ -72,38 +78,162 @@ class MBRScan(interfaces.plugins.PluginInterface): all_zeros = bootcode.count(b"\x00") == len(bootcode) if not all_zeros: - partition_entries = [ - partition_table.FirstEntry, - partition_table.SecondEntry, - partition_table.ThirdEntry, - partition_table.FourthEntry - ] - partition_info = "\n" - - for index, partition_entry_object in enumerate(partition_entries): - partition_entry_object.set_index(index) - partition_info += str(partition_entry_object) - - yield 0, ( + if not self.config.get("full", True): + yield (0, ( format_hints.Hex(offset), partition_table.get_disk_signature(), self.get_hash(bootcode), self.get_hash(full_mbr), - partition_info, + partition_table.FirstEntry.is_bootable(), + partition_table.FirstEntry.get_partition_type(), + format_hints.Hex(partition_table.FirstEntry.get_size_in_sectors()), + partition_table.SecondEntry.is_bootable(), + partition_table.SecondEntry.get_partition_type(), + format_hints.Hex(partition_table.SecondEntry.get_size_in_sectors()), + partition_table.ThirdEntry.is_bootable(), + partition_table.ThirdEntry.get_partition_type(), + format_hints.Hex(partition_table.ThirdEntry.get_size_in_sectors()), + partition_table.FourthEntry.is_bootable(), + partition_table.FourthEntry.get_partition_type(), + format_hints.Hex(partition_table.FourthEntry.get_size_in_sectors()), interfaces.renderers.Disassembly(bootcode, 0, architecture) - ) + )) + else: + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_table.FirstEntry.is_bootable(), + format_hints.Hex(partition_table.FirstEntry.get_bootable_flag()), + partition_table.FirstEntry.get_partition_type(), + format_hints.Hex(partition_table.FirstEntry.PartitionType), + format_hints.Hex(partition_table.FirstEntry.get_starting_lba()), + partition_table.FirstEntry.get_starting_cylinder(), + partition_table.FirstEntry.get_starting_chs(), + partition_table.FirstEntry.get_starting_sector(), + partition_table.FirstEntry.get_ending_cylinder(), + partition_table.FirstEntry.get_ending_chs(), + partition_table.FirstEntry.get_ending_sector(), + format_hints.Hex(partition_table.FirstEntry.get_size_in_sectors()), + partition_table.SecondEntry.is_bootable(), + format_hints.Hex(partition_table.SecondEntry.get_bootable_flag()), + partition_table.SecondEntry.get_partition_type(), + format_hints.Hex(partition_table.SecondEntry.PartitionType), + format_hints.Hex(partition_table.SecondEntry.get_starting_lba()), + partition_table.SecondEntry.get_starting_cylinder(), + partition_table.SecondEntry.get_starting_chs(), + partition_table.SecondEntry.get_starting_sector(), + partition_table.SecondEntry.get_ending_cylinder(), + partition_table.SecondEntry.get_ending_chs(), + partition_table.SecondEntry.get_ending_sector(), + format_hints.Hex(partition_table.SecondEntry.get_size_in_sectors()), + partition_table.ThirdEntry.is_bootable(), + format_hints.Hex(partition_table.ThirdEntry.get_bootable_flag()), + partition_table.ThirdEntry.get_partition_type(), + format_hints.Hex(partition_table.ThirdEntry.PartitionType), + format_hints.Hex(partition_table.ThirdEntry.get_starting_lba()), + partition_table.ThirdEntry.get_starting_cylinder(), + partition_table.ThirdEntry.get_starting_chs(), + partition_table.ThirdEntry.get_starting_sector(), + partition_table.ThirdEntry.get_ending_cylinder(), + partition_table.ThirdEntry.get_ending_chs(), + partition_table.ThirdEntry.get_ending_sector(), + format_hints.Hex(partition_table.ThirdEntry.get_size_in_sectors()), + partition_table.FourthEntry.is_bootable(), + format_hints.Hex(partition_table.FourthEntry.get_bootable_flag()), + partition_table.FourthEntry.get_partition_type(), + format_hints.Hex(partition_table.FourthEntry.PartitionType), + format_hints.Hex(partition_table.FourthEntry.get_starting_lba()), + partition_table.FourthEntry.get_starting_cylinder(), + partition_table.FourthEntry.get_starting_chs(), + partition_table.FourthEntry.get_starting_sector(), + partition_table.FourthEntry.get_ending_cylinder(), + partition_table.FourthEntry.get_ending_chs(), + partition_table.FourthEntry.get_ending_sector(), + format_hints.Hex(partition_table.FourthEntry.get_size_in_sectors()), + interfaces.renderers.Disassembly(bootcode, 0, architecture) + )) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") except exceptions.PagedInvalidAddressException: pass - def run(self): - return renderers.TreeGrid([ - ("Potential MBR at Physical Offset", format_hints.Hex), - ("Disk Signature", str), - ("Bootcode MD5", str), - ("Full MBR MD5", str), - ("Partition Entries Info", str), - ("Disasm", interfaces.renderers.Disassembly) - ], self._generator()) + def run(self)-> renderers.TreeGrid: + if not self.config.get("full", True): + return renderers.TreeGrid([ + ("Potential MBR at Physical Offset", format_hints.Hex), + ("Disk Signature", str), + ("Bootcode MD5", str), + ("Full MBR MD5", str), + ("PartABootable", bool), + ("PartAType", str), + ("PartASectorInSize", format_hints.Hex), + ("PartBBootable", bool), + ("PartBType", str), + ("PartBSectorInSize", format_hints.Hex), + ("PartCBootable", bool), + ("PartCType", str), + ("PartCSectorInSize", format_hints.Hex), + ("PartDBootable", bool), + ("PartDType", str), + ("PartDSectorInSize", format_hints.Hex), + ("Disasm", interfaces.renderers.Disassembly) + ], self._generator()) + else: + return renderers.TreeGrid([ + ("Potential MBR at Physical Offset", format_hints.Hex), + ("Disk Signature", str), + ("Bootcode MD5", str), + ("Full MBR MD5", str), + ("PartABootable", bool), + ("PartABootFlag", format_hints.Hex), + ("PartAType", str), + ("PartATypeRaw", format_hints.Hex), + ("PartAStartingLBA", format_hints.Hex), + ("PartAStartingCylinder", int), + ("PartAStartingCHS", int), + ("PartAStartingSector", int), + ("PartAEndingCylinder", int), + ("PartAEndingCHS", int), + ("PartAEndingSector", int), + ("PartASectorInSize", format_hints.Hex), + ("PartBBootable", bool), + ("PartBBootFlag", format_hints.Hex), + ("PartBType", str), + ("PartBTypeRaw", format_hints.Hex), + ("PartBStartingLBA", format_hints.Hex), + ("PartBStartingCylinder", int), + ("PartBStartingCHS", int), + ("PartBStartingSector", int), + ("PartBEndingCylinder", int), + ("PartBEndingCHS", int), + ("PartBEndingSector", int), + ("PartBSectorInSize", format_hints.Hex), + ("PartCBootable", bool), + ("PartCBootFlag", format_hints.Hex), + ("PartCType", str), + ("PartCTypeRaw", format_hints.Hex), + ("PartCStartingLBA", format_hints.Hex), + ("PartCStartingCylinder", int), + ("PartCStartingCHS", int), + ("PartCStartingSector", int), + ("PartCEndingCylinder", int), + ("PartCEndingCHS", int), + ("PartCEndingSector", int), + ("PartCSectorInSize", format_hints.Hex), + ("PartDBootable", bool), + ("PartDBootFlag", format_hints.Hex), + ("PartDType", str), + ("PartDTypeRaw", format_hints.Hex), + ("PartDStartingLBA", format_hints.Hex), + ("PartDStartingCylinder", int), + ("PartDStartingCHS", int), + ("PartDStartingSector", int), + ("PartDEndingCylinder", int), + ("PartDEndingCHS", int), + ("PartDEndingSector", int), + ("PartDSectorInSize", format_hints.Hex), + ("Disasm", interfaces.renderers.Disassembly) + ], self._generator()) From a6217784cea49698975730ed6782a1c0023ecd46 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Mar 2022 20:12:22 +0000 Subject: [PATCH 119/158] Windows: Tidy up hashdump plugin Shouldn't have merged this with mention of profiles. Also fixes #678. --- .../framework/plugins/windows/hashdump.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index 4d0b25b5c..e9f8047e0 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -4,10 +4,10 @@ import binascii import hashlib import logging -from struct import unpack, pack -from typing import List, Tuple, Optional +from struct import pack, unpack +from typing import List, Optional, Tuple -from Crypto.Cipher import ARC4, DES, AES +from Crypto.Cipher import AES, ARC4, DES from Crypto.Hash import MD5 from volatility3.framework import interfaces, renderers @@ -28,7 +28,7 @@ class Hashdump(interfaces.plugins.PluginInterface): def get_requirements(cls): return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) ] @@ -63,7 +63,8 @@ class Hashdump(interfaces.plugins.PluginInterface): def get_hive_key(cls, hive: registry.RegistryHive, key: str): result = None try: - result = hive.get_key(key) + if hive: + result = hive.get_key(key) except KeyError: vollog.info( f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image") @@ -132,7 +133,7 @@ class Hashdump(interfaces.plugins.PluginInterface): rc4_key = md5.digest() rc4 = ARC4.new(rc4_key) - hbootkey = rc4.encrypt(sam_data[0x80:0xA0]) # lgtm [py/weak-cryptographic-algorithm] + hbootkey = rc4.encrypt(sam_data[0x80:0xA0]) # lgtm [py/weak-cryptographic-algorithm] return hbootkey elif revision == 3: # AES encrypted @@ -151,7 +152,7 @@ class Hashdump(interfaces.plugins.PluginInterface): des2 = DES.new(des_k2, DES.MODE_ECB) cipher = AES.new(hbootkey[:16], AES.MODE_CBC, salt) obfkey = cipher.decrypt(enc_hash) - return des1.decrypt(obfkey[:8]) + des2.decrypt(obfkey[8:16]) # lgtm [py/weak-cryptographic-algorithm] + return des1.decrypt(obfkey[:8]) + des2.decrypt(obfkey[8:16]) # lgtm [py/weak-cryptographic-algorithm] @classmethod def get_user_hashes(cls, user: registry.CM_KEY_NODE, samhive: registry.RegistryHive, @@ -229,9 +230,9 @@ class Hashdump(interfaces.plugins.PluginInterface): md5.update(hbootkey[:0x10] + pack(" Optional[bytes]: @@ -253,13 +254,9 @@ class Hashdump(interfaces.plugins.PluginInterface): # replaces the dump_hashes method in vol2 def _generator(self, syshive: registry.RegistryHive, samhive: registry.RegistryHive): if syshive is None: - vollog.debug("SYSTEM address is None: Did you use the correct profile?") - yield (0, (renderers.NotAvailableValue(), renderers.NotAvailableValue(), renderers.NotAvailableValue(), - renderers.NotAvailableValue())) + vollog.debug("SYSTEM address is None: No system hive found") if samhive is None: - vollog.debug("SAM address is None: Did you use the correct profile?") - yield (0, (renderers.NotAvailableValue(), renderers.NotAvailableValue(), renderers.NotAvailableValue(), - renderers.NotAvailableValue())) + vollog.debug("SAM address is None: No SAM hive found") bootkey = self.get_bootkey(syshive) hbootkey = self.get_hbootkey(samhive, bootkey) if hbootkey: From cb8a1fb90c7e1571b82bc6bc58cd45f5ffcfff0e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 16 Mar 2022 20:36:26 +0000 Subject: [PATCH 120/158] CLI: Fail on overwriting a config file --- volatility3/cli/__init__.py | 9 ++------- volatility3/cli/volshell/__init__.py | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 1933607c3..8d198e57c 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -324,18 +324,13 @@ class CommandLine: constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) - backup_filename = constants.BACKUP_EXISTING_CONFIG_OUTPUT if args.write_config: vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' - backup_filename = False if args.save_config: vollog.debug("Writing out configuration data to {args.save_config}") - if os.path.exists(os.path.abspath(args.save_config)) and backup_filename: - # Backup existing file - backup_filename = self.find_backup_filename(args.save_config) - vollog.debug(f"Backing up existing file to {backup_filename}") - os.rename(args.save_config, backup_filename) + if os.path.exists(os.path.abspath(args.save_config)): + parser.error(f"Cannot write configuration: file {args.save_config} already exists") with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) except exceptions.UnsatisfiedException as excp: diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index f3bed1b73..30fe75e06 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -237,18 +237,13 @@ class VolShell(cli.CommandLine): constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) - backup_filename = constants.BACKUP_EXISTING_CONFIG_OUTPUT if args.write_config: vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') args.save_config = 'config.json' - backup_filename = False if args.save_config: vollog.debug("Writing out configuration data to {args.save_config}") - if os.path.exists(os.path.abspath(args.save_config)) and backup_filename: - # Backup existing file - backup_filename = self.find_backup_filename(args.save_config) - vollog.debug(f"Backing up existing file to {backup_filename}") - os.rename(args.save_config, backup_filename) + if os.path.exists(os.path.abspath(args.save_config)): + parser.error(f"Cannot write configuration: file {args.save_config} already exists") with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) except exceptions.UnsatisfiedException as excp: From ed1a19d1ac2bb6267ce1627f60e80b34c59f1047 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 18 Mar 2022 01:07:59 +0900 Subject: [PATCH 121/158] Add hex dump column if full data option --- volatility3/framework/plugins/windows/mbrscan.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 60b403ae0..3602e5e78 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -28,7 +28,7 @@ class MBRScan(interfaces.plugins.PluginInterface): requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), requirements.BooleanRequirement(name = 'full', - description ="It analyzes and provides all the information in the partition entry. (It returns a lot of information, so we recommend you render it in CSV.)", + description ="It analyzes and provides all the information in the partition entry and bootcode hexdump. (It returns a lot of information, so we recommend you render it in CSV.)", default = False, optional = True) ] @@ -152,7 +152,8 @@ class MBRScan(interfaces.plugins.PluginInterface): partition_table.FourthEntry.get_ending_chs(), partition_table.FourthEntry.get_ending_sector(), format_hints.Hex(partition_table.FourthEntry.get_size_in_sectors()), - interfaces.renderers.Disassembly(bootcode, 0, architecture) + interfaces.renderers.Disassembly(bootcode, 0, architecture), + format_hints.HexBytes(bootcode) )) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") @@ -235,5 +236,6 @@ class MBRScan(interfaces.plugins.PluginInterface): ("PartDEndingCHS", int), ("PartDEndingSector", int), ("PartDSectorInSize", format_hints.Hex), - ("Disasm", interfaces.renderers.Disassembly) + ("Disasm", interfaces.renderers.Disassembly), + ("Bootcode", format_hints.HexBytes) ], self._generator()) From 59102aae2a165a3ea383f1e28b9713257a6e53de Mon Sep 17 00:00:00 2001 From: "Nick L. Petroni, Jr" Date: Sun, 16 Jan 2022 17:32:02 -0500 Subject: [PATCH 122/158] update doc copyright --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index eab5c2c94..3be41db5d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # # @@ -126,7 +126,7 @@ master_doc = 'index' # General information about the project. project = 'Volatility 3' -copyright = '2012-2019, Volatility Foundation' +copyright = '2012-2022, Volatility Foundation' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 174036cc727b98a53e7d83dee9cfc82dcd370382 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 19 Mar 2022 15:26:10 +0900 Subject: [PATCH 123/158] Fix Typo Error for Disassembly rendering code comment --- volatility3/cli/text_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 8e07d58d1..1ddfcca84 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -101,7 +101,7 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str: disasm: Input disassembly objects Returns: - A string as rendererd by capstone where available, otherwise output as if it were just bytes + A string as rendered by capstone where available, otherwise output as if it were just bytes """ if CAPSTONE_PRESENT: From 1aad1c8b1a933f46ae2753bb209dd986e49ce99e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 21 Mar 2022 00:32:24 +0900 Subject: [PATCH 124/158] Initialize devicetree plugin --- volatility3/framework/plugins/windows/devicetree.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 volatility3/framework/plugins/windows/devicetree.py diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py new file mode 100644 index 000000000..e69de29bb From 5af889b0593947854f161cbccd965f5b5036994b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 21 Mar 2022 15:11:51 +0900 Subject: [PATCH 125/158] Fix operating system comparision syntax for create cache path constant. --- volatility3/framework/constants/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 665e62d30..629d5db80 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -11,7 +11,6 @@ import os.path import sys from typing import Optional, Callable -import volatility3.framework.constants.linux import volatility3.framework.constants.windows PLUGINS_PATH = [ @@ -63,7 +62,7 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" -if sys.platform == 'windows': +if sys.platform == 'win32': CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") os.makedirs(CACHE_PATH, exist_ok = True) From 37f6750c92668407e07ec7e8d641a4195490a95f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 23 Mar 2022 07:55:53 +0900 Subject: [PATCH 126/158] ReImport volatility.framework.constants.linux --- volatility3/framework/constants/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 629d5db80..063862be8 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -11,6 +11,7 @@ import os.path import sys from typing import Optional, Callable +import volatility3.framework.constants.linux import volatility3.framework.constants.windows PLUGINS_PATH = [ From cf4ef0fa38eb7e68e51986a35ae91da4f9a04d5a Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 23 Mar 2022 15:12:54 +0900 Subject: [PATCH 127/158] Set __init__ and fix description of mac environment plugins --- volatility3/framework/plugins/mac/__init__.py | 8 ++++++++ volatility3/framework/plugins/mac/ifconfig.py | 2 +- volatility3/framework/plugins/mac/mount.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/__init__.py b/volatility3/framework/plugins/mac/__init__.py index e69de29bb..ef6762bee 100644 --- a/volatility3/framework/plugins/mac/__init__.py +++ b/volatility3/framework/plugins/mac/__init__.py @@ -0,0 +1,8 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +"""All core mac plugins. + +These modules should only be imported from volatility3.plugins NOT +volatility3.framework.plugins +""" diff --git a/volatility3/framework/plugins/mac/ifconfig.py b/volatility3/framework/plugins/mac/ifconfig.py index c366a19f0..99666b763 100644 --- a/volatility3/framework/plugins/mac/ifconfig.py +++ b/volatility3/framework/plugins/mac/ifconfig.py @@ -9,7 +9,7 @@ from volatility3.framework.symbols import mac class Ifconfig(plugins.PluginInterface): - """Lists loaded kernel modules""" + """ Lists network interface information for all devices """ _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index 398559446..6486d00ff 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -12,7 +12,7 @@ from volatility3.framework.symbols import mac class Mount(plugins.PluginInterface): """A module containing a collection of plugins that produce data typically - foundin Mac's mount command""" + founding Mac's mount command""" _required_framework_version = (2, 0, 0) From 95825e99dfe1dafae062e7c84dbd0f6ca96b26d6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 23 Mar 2022 16:09:19 +0900 Subject: [PATCH 128/158] Update Vollog level if all zero mbr data, PagedInvalidAddressException handling --- volatility3/framework/plugins/windows/mbrscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 3602e5e78..1e78c1c25 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -156,10 +156,10 @@ class MBRScan(interfaces.plugins.PluginInterface): format_hints.HexBytes(bootcode) )) else: - vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + vollog.log(constants.LOGLEVEL_VVVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") except exceptions.PagedInvalidAddressException: - pass + continue def run(self)-> renderers.TreeGrid: if not self.config.get("full", True): From 64b8f681f4c778f3ed20350baed69b8ea2e9b2de Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 23 Mar 2022 18:05:42 +0900 Subject: [PATCH 129/158] Update sentence by code review --- volatility3/framework/plugins/mac/ifconfig.py | 2 +- volatility3/framework/plugins/mac/mount.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/ifconfig.py b/volatility3/framework/plugins/mac/ifconfig.py index 99666b763..330c13f07 100644 --- a/volatility3/framework/plugins/mac/ifconfig.py +++ b/volatility3/framework/plugins/mac/ifconfig.py @@ -9,7 +9,7 @@ from volatility3.framework.symbols import mac class Ifconfig(plugins.PluginInterface): - """ Lists network interface information for all devices """ + """Lists network interface information for all devices""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index 6486d00ff..ba3ab83c8 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -12,7 +12,7 @@ from volatility3.framework.symbols import mac class Mount(plugins.PluginInterface): """A module containing a collection of plugins that produce data typically - founding Mac's mount command""" + found in Mac's mount command""" _required_framework_version = (2, 0, 0) From 02d90e9e42974959440fb9a45aa585ad9870d24b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 24 Mar 2022 08:45:29 +0000 Subject: [PATCH 130/158] CLI: Remove unnecessary extra code --- volatility3/cli/__init__.py | 9 --------- volatility3/framework/constants/__init__.py | 3 --- 2 files changed, 12 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 8d198e57c..35ad84011 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -344,15 +344,6 @@ class CommandLine: except (exceptions.VolatilityException) as excp: self.process_exceptions(excp) - def find_backup_filename(self, original: str): - suffix = "" - new_name = f"{original}.{datetime.strftime(datetime.today(), '%y%m%d')}.bak" - while os.path.exists(f"{new_name}{suffix}"): - if not suffix: - suffix = 1 - suffix += 1 - return f"{new_name}{suffix}" - @classmethod def location_from_file(cls, filename: str) -> str: """Returns the URL location from a file parameter (which may be a URL) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 4af4408af..f3d31dd2e 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -101,6 +101,3 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" - -BACKUP_EXISTING_CONFIG_OUTPUT = True -"""Whether existing files are backed up or overwritten when writing configuration output""" From 2cfc24f7d3c4af2b710819c80925963c216fcce9 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 15:50:15 +0900 Subject: [PATCH 131/158] Changes in structure and data return for efficient partition entries data display --- .../framework/plugins/windows/mbrscan.py | 204 ++++++------------ .../symbols/windows/extensions/mbr.py | 2 - 2 files changed, 65 insertions(+), 141 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 1e78c1c25..991d48bf9 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -78,88 +78,57 @@ class MBRScan(interfaces.plugins.PluginInterface): all_zeros = bootcode.count(b"\x00") == len(bootcode) if not all_zeros: - if not self.config.get("full", True): - yield (0, ( - format_hints.Hex(offset), - partition_table.get_disk_signature(), - self.get_hash(bootcode), - self.get_hash(full_mbr), - partition_table.FirstEntry.is_bootable(), - partition_table.FirstEntry.get_partition_type(), - format_hints.Hex(partition_table.FirstEntry.get_size_in_sectors()), - partition_table.SecondEntry.is_bootable(), - partition_table.SecondEntry.get_partition_type(), - format_hints.Hex(partition_table.SecondEntry.get_size_in_sectors()), - partition_table.ThirdEntry.is_bootable(), - partition_table.ThirdEntry.get_partition_type(), - format_hints.Hex(partition_table.ThirdEntry.get_size_in_sectors()), - partition_table.FourthEntry.is_bootable(), - partition_table.FourthEntry.get_partition_type(), - format_hints.Hex(partition_table.FourthEntry.get_size_in_sectors()), - interfaces.renderers.Disassembly(bootcode, 0, architecture) - )) - else: - yield (0, ( - format_hints.Hex(offset), - partition_table.get_disk_signature(), - self.get_hash(bootcode), - self.get_hash(full_mbr), - partition_table.FirstEntry.is_bootable(), - format_hints.Hex(partition_table.FirstEntry.get_bootable_flag()), - partition_table.FirstEntry.get_partition_type(), - format_hints.Hex(partition_table.FirstEntry.PartitionType), - format_hints.Hex(partition_table.FirstEntry.get_starting_lba()), - partition_table.FirstEntry.get_starting_cylinder(), - partition_table.FirstEntry.get_starting_chs(), - partition_table.FirstEntry.get_starting_sector(), - partition_table.FirstEntry.get_ending_cylinder(), - partition_table.FirstEntry.get_ending_chs(), - partition_table.FirstEntry.get_ending_sector(), - format_hints.Hex(partition_table.FirstEntry.get_size_in_sectors()), - partition_table.SecondEntry.is_bootable(), - format_hints.Hex(partition_table.SecondEntry.get_bootable_flag()), - partition_table.SecondEntry.get_partition_type(), - format_hints.Hex(partition_table.SecondEntry.PartitionType), - format_hints.Hex(partition_table.SecondEntry.get_starting_lba()), - partition_table.SecondEntry.get_starting_cylinder(), - partition_table.SecondEntry.get_starting_chs(), - partition_table.SecondEntry.get_starting_sector(), - partition_table.SecondEntry.get_ending_cylinder(), - partition_table.SecondEntry.get_ending_chs(), - partition_table.SecondEntry.get_ending_sector(), - format_hints.Hex(partition_table.SecondEntry.get_size_in_sectors()), - partition_table.ThirdEntry.is_bootable(), - format_hints.Hex(partition_table.ThirdEntry.get_bootable_flag()), - partition_table.ThirdEntry.get_partition_type(), - format_hints.Hex(partition_table.ThirdEntry.PartitionType), - format_hints.Hex(partition_table.ThirdEntry.get_starting_lba()), - partition_table.ThirdEntry.get_starting_cylinder(), - partition_table.ThirdEntry.get_starting_chs(), - partition_table.ThirdEntry.get_starting_sector(), - partition_table.ThirdEntry.get_ending_cylinder(), - partition_table.ThirdEntry.get_ending_chs(), - partition_table.ThirdEntry.get_ending_sector(), - format_hints.Hex(partition_table.ThirdEntry.get_size_in_sectors()), - partition_table.FourthEntry.is_bootable(), - format_hints.Hex(partition_table.FourthEntry.get_bootable_flag()), - partition_table.FourthEntry.get_partition_type(), - format_hints.Hex(partition_table.FourthEntry.PartitionType), - format_hints.Hex(partition_table.FourthEntry.get_starting_lba()), - partition_table.FourthEntry.get_starting_cylinder(), - partition_table.FourthEntry.get_starting_chs(), - partition_table.FourthEntry.get_starting_sector(), - partition_table.FourthEntry.get_ending_cylinder(), - partition_table.FourthEntry.get_ending_chs(), - partition_table.FourthEntry.get_ending_sector(), - format_hints.Hex(partition_table.FourthEntry.get_size_in_sectors()), - interfaces.renderers.Disassembly(bootcode, 0, architecture), - format_hints.HexBytes(bootcode) - )) + + partition_entries = [ + partition_table.FirstEntry, partition_table.SecondEntry, + partition_table.ThirdEntry, partition_table.FourthEntry + ] + + for partition_index, partition_entry_object in enumerate(partition_entries, start=1): + # Output disassembly information and bootcode for each partition entry is inefficient, + # so it can only be processed in the last index. + last_partition_index = 4 + bootcode_buf = bootcode if(partition_index == last_partition_index) else b"" + + if not self.config.get("full", True): + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_index, + partition_entry_object.is_bootable(), + partition_entry_object.get_partition_type(), + format_hints.Hex(partition_entry_object.get_size_in_sectors()), + interfaces.renderers.Disassembly(bootcode_buf, 0, architecture) + )) + else: + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_index, + partition_entry_object.is_bootable(), + format_hints.Hex(partition_entry_object.get_bootable_flag()), + partition_entry_object.get_partition_type(), + format_hints.Hex(partition_entry_object.PartitionType), + format_hints.Hex(partition_entry_object.get_starting_lba()), + partition_entry_object.get_starting_cylinder(), + partition_entry_object.get_starting_chs(), + partition_entry_object.get_starting_sector(), + partition_entry_object.get_ending_cylinder(), + partition_entry_object.get_ending_chs(), + partition_entry_object.get_ending_sector(), + format_hints.Hex(partition_entry_object.get_size_in_sectors()), + interfaces.renderers.Disassembly(bootcode_buf, 0, architecture), + format_hints.HexBytes(bootcode_buf) + )) else: - vollog.log(constants.LOGLEVEL_VVVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") except exceptions.PagedInvalidAddressException: - continue + pass def run(self)-> renderers.TreeGrid: if not self.config.get("full", True): @@ -168,18 +137,10 @@ class MBRScan(interfaces.plugins.PluginInterface): ("Disk Signature", str), ("Bootcode MD5", str), ("Full MBR MD5", str), - ("PartABootable", bool), - ("PartAType", str), - ("PartASectorInSize", format_hints.Hex), - ("PartBBootable", bool), - ("PartBType", str), - ("PartBSectorInSize", format_hints.Hex), - ("PartCBootable", bool), - ("PartCType", str), - ("PartCSectorInSize", format_hints.Hex), - ("PartDBootable", bool), - ("PartDType", str), - ("PartDSectorInSize", format_hints.Hex), + ("PartitionIndex", int), + ("Bootable", bool), + ("PartitionType", str), + ("SectorInSize", format_hints.Hex), ("Disasm", interfaces.renderers.Disassembly) ], self._generator()) else: @@ -188,54 +149,19 @@ class MBRScan(interfaces.plugins.PluginInterface): ("Disk Signature", str), ("Bootcode MD5", str), ("Full MBR MD5", str), - ("PartABootable", bool), - ("PartABootFlag", format_hints.Hex), - ("PartAType", str), - ("PartATypeRaw", format_hints.Hex), - ("PartAStartingLBA", format_hints.Hex), - ("PartAStartingCylinder", int), - ("PartAStartingCHS", int), - ("PartAStartingSector", int), - ("PartAEndingCylinder", int), - ("PartAEndingCHS", int), - ("PartAEndingSector", int), - ("PartASectorInSize", format_hints.Hex), - ("PartBBootable", bool), - ("PartBBootFlag", format_hints.Hex), - ("PartBType", str), - ("PartBTypeRaw", format_hints.Hex), - ("PartBStartingLBA", format_hints.Hex), - ("PartBStartingCylinder", int), - ("PartBStartingCHS", int), - ("PartBStartingSector", int), - ("PartBEndingCylinder", int), - ("PartBEndingCHS", int), - ("PartBEndingSector", int), - ("PartBSectorInSize", format_hints.Hex), - ("PartCBootable", bool), - ("PartCBootFlag", format_hints.Hex), - ("PartCType", str), - ("PartCTypeRaw", format_hints.Hex), - ("PartCStartingLBA", format_hints.Hex), - ("PartCStartingCylinder", int), - ("PartCStartingCHS", int), - ("PartCStartingSector", int), - ("PartCEndingCylinder", int), - ("PartCEndingCHS", int), - ("PartCEndingSector", int), - ("PartCSectorInSize", format_hints.Hex), - ("PartDBootable", bool), - ("PartDBootFlag", format_hints.Hex), - ("PartDType", str), - ("PartDTypeRaw", format_hints.Hex), - ("PartDStartingLBA", format_hints.Hex), - ("PartDStartingCylinder", int), - ("PartDStartingCHS", int), - ("PartDStartingSector", int), - ("PartDEndingCylinder", int), - ("PartDEndingCHS", int), - ("PartDEndingSector", int), - ("PartDSectorInSize", format_hints.Hex), + ("PartitionIndex", int), + ("Bootable", bool), + ("BootFlag", format_hints.Hex), + ("PartitionType", str), + ("PartitionTypeRaw", format_hints.Hex), + ("StartingLBA", format_hints.Hex), + ("StartingCylinder", int), + ("StartingCHS", int), + ("StartingSector", int), + ("EndingCylinder", int), + ("EndingCHS", int), + ("EndingSector", int), + ("SectorInSize", format_hints.Hex), ("Disasm", interfaces.renderers.Disassembly), ("Bootcode", format_hints.HexBytes) ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 3fdb67ee3..8100371fd 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -2,8 +2,6 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import struct - from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): From b07859db7f55bae7d9c9cb15aea978c06968beda Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 17:02:41 +0900 Subject: [PATCH 132/158] Add vollog for PagedInvalidAddressException --- volatility3/framework/plugins/windows/mbrscan.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 991d48bf9..cdc40a7be 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -126,10 +126,12 @@ class MBRScan(interfaces.plugins.PluginInterface): )) else: vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + continue - except exceptions.PagedInvalidAddressException: - pass - + except exceptions.PagedInvalidAddressException as excp: + vollog.debug(f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") + continue + def run(self)-> renderers.TreeGrid: if not self.config.get("full", True): return renderers.TreeGrid([ From a49292ab483426d7db1c5be4c0a31db39859da36 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 17:21:46 +0900 Subject: [PATCH 133/158] Fix type for partition --- volatility3/framework/symbols/windows/mbr.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/mbr.json b/volatility3/framework/symbols/windows/mbr.json index 122c020c3..2a6ec7779 100644 --- a/volatility3/framework/symbols/windows/mbr.json +++ b/volatility3/framework/symbols/windows/mbr.json @@ -170,7 +170,7 @@ "offset": 12, "type": { "kind": "base", - "name": "int" + "name": "unsigned int" } } }, From f8443b994dc4ac1f512e49928d555e6b2abcadd6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 17:29:07 +0900 Subject: [PATCH 134/158] Refactoring for partition index --- volatility3/framework/plugins/windows/mbrscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index cdc40a7be..fc473ae7e 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -87,7 +87,7 @@ class MBRScan(interfaces.plugins.PluginInterface): for partition_index, partition_entry_object in enumerate(partition_entries, start=1): # Output disassembly information and bootcode for each partition entry is inefficient, # so it can only be processed in the last index. - last_partition_index = 4 + last_partition_index = len(partition_entries) bootcode_buf = bootcode if(partition_index == last_partition_index) else b"" if not self.config.get("full", True): From a9dabf90d71543ee335ec1a20252d9f27b68ef93 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 23:22:23 +0900 Subject: [PATCH 135/158] Adjust vollog log level of Exception --- volatility3/framework/plugins/windows/mbrscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index fc473ae7e..2148efc41 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -125,11 +125,11 @@ class MBRScan(interfaces.plugins.PluginInterface): format_hints.HexBytes(bootcode_buf) )) else: - vollog.log(constants.LOGLEVEL_VV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + vollog.log(constants.LOGLEVEL_VVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") continue except exceptions.PagedInvalidAddressException as excp: - vollog.debug(f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") + vollog.log(constants.LOGLEVEL_VVV, f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") continue def run(self)-> renderers.TreeGrid: From 8a99c17d4266f7e869404f37a4e60e61bb6cfe90 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 25 Mar 2022 23:27:25 +0900 Subject: [PATCH 136/158] Adjust vollog log level of Exception --- volatility3/framework/plugins/windows/mbrscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 2148efc41..ba74a7b7e 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -125,11 +125,11 @@ class MBRScan(interfaces.plugins.PluginInterface): format_hints.HexBytes(bootcode_buf) )) else: - vollog.log(constants.LOGLEVEL_VVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + vollog.log(constants.LOGLEVEL_VVVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") continue except exceptions.PagedInvalidAddressException as excp: - vollog.log(constants.LOGLEVEL_VVV, f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") + vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") continue def run(self)-> renderers.TreeGrid: From ff433dd4b8fb0bb3d7125614e7c295358f512709 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 26 Mar 2022 01:22:17 +0900 Subject: [PATCH 137/158] Change the empty byte to NotApplicableValue for efficient partition data output. --- .../framework/plugins/windows/mbrscan.py | 49 +++++++++++++++---- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index ba74a7b7e..39e962c96 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -84,14 +84,45 @@ class MBRScan(interfaces.plugins.PluginInterface): partition_table.ThirdEntry, partition_table.FourthEntry ] + if not self.config.get("full", True): + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + interfaces.renderers.Disassembly(bootcode, 0, architecture) + )) + else: + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + interfaces.renderers.Disassembly(bootcode, 0, architecture), + format_hints.HexBytes(bootcode) + )) + for partition_index, partition_entry_object in enumerate(partition_entries, start=1): - # Output disassembly information and bootcode for each partition entry is inefficient, - # so it can only be processed in the last index. - last_partition_index = len(partition_entries) - bootcode_buf = bootcode if(partition_index == last_partition_index) else b"" if not self.config.get("full", True): - yield (0, ( + yield (1, ( format_hints.Hex(offset), partition_table.get_disk_signature(), self.get_hash(bootcode), @@ -100,10 +131,10 @@ class MBRScan(interfaces.plugins.PluginInterface): partition_entry_object.is_bootable(), partition_entry_object.get_partition_type(), format_hints.Hex(partition_entry_object.get_size_in_sectors()), - interfaces.renderers.Disassembly(bootcode_buf, 0, architecture) + renderers.NotApplicableValue() )) else: - yield (0, ( + yield (1, ( format_hints.Hex(offset), partition_table.get_disk_signature(), self.get_hash(bootcode), @@ -121,8 +152,8 @@ class MBRScan(interfaces.plugins.PluginInterface): partition_entry_object.get_ending_chs(), partition_entry_object.get_ending_sector(), format_hints.Hex(partition_entry_object.get_size_in_sectors()), - interfaces.renderers.Disassembly(bootcode_buf, 0, architecture), - format_hints.HexBytes(bootcode_buf) + renderers.NotApplicableValue(), + renderers.NotApplicableValue() )) else: vollog.log(constants.LOGLEVEL_VVVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") From d17c56af1ce3799485700e2b613c18a2d219ee8e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 26 Mar 2022 01:23:12 +0900 Subject: [PATCH 138/158] Remove space the plugin description --- volatility3/framework/plugins/windows/mbrscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py index 39e962c96..d064e7d29 100644 --- a/volatility3/framework/plugins/windows/mbrscan.py +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -17,7 +17,7 @@ from volatility3.framework.symbols.windows.extensions import mbr vollog = logging.getLogger(__name__) class MBRScan(interfaces.plugins.PluginInterface): - """ Scans for and parses potential Master Boot Records (MBRs) """ + """Scans for and parses potential Master Boot Records (MBRs)""" _required_framework_version = (2, 0, 1) _version = (1, 0, 0) From 5c402f33e9bc967bd12a3fcfa36eb40b7a8b8b87 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 26 Mar 2022 01:28:36 +0900 Subject: [PATCH 139/158] Improvement of MBR extension's incomplete return type --- .../framework/symbols/windows/extensions/mbr.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index 8100371fd..fc7996c52 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -29,34 +29,34 @@ class PARTITION_ENTRY(objects.StructType): """Get Partition Type.""" return self.PartitionType.lookup() if self.PartitionType.is_valid_choice else "Not Defined PartitionType" - def get_starting_chs(self): + def get_starting_chs(self) -> int: """Get Starting CHS (Cylinder Header Sector) Address.""" return self.StartingCHS[0] - def get_ending_chs(self): + def get_ending_chs(self) -> int: """Get Ending CHS (Cylinder Header Sector) Address.""" return self.EndingCHS[0] - def get_starting_sector(self): + def get_starting_sector(self) -> int: """Get Starting Sector.""" return self.StartingCHS[1] % 64 - def get_ending_sector(self): + def get_ending_sector(self) -> int: """Get Ending Sector.""" return self.EndingCHS[1] % 64 - def get_starting_cylinder(self): + def get_starting_cylinder(self) -> int: """Get Starting Cylinder.""" return (self.StartingCHS[1] - self.get_starting_sector()) * 4 + self.StartingCHS[2] - def get_ending_cylinder(self): + def get_ending_cylinder(self) -> int: """Get Ending Cylinder.""" return (self.EndingCHS[1] - self.get_ending_sector()) * 4 + self.EndingCHS[2] - def get_starting_lba(self): + def get_starting_lba(self) -> int: """Get Starting LBA (Logical Block Addressing).""" return self.StartingLBA - def get_size_in_sectors(self): + def get_size_in_sectors(self) -> int: """Get Size in Sectors.""" return self.SizeInSectors From fb081ec233c08548b40b5bc96d182464f68e9795 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 00:36:03 +0900 Subject: [PATCH 140/158] Add Windows DRIVER_OBJECT, DEVICE_OBJECT method --- .../symbols/windows/extensions/__init__.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index dc0de1dda..c1972cb7f 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -351,17 +351,38 @@ class EX_FAST_REF(objects.StructType): class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel device objects.""" - def get_device_name(self) -> str: - header = self.get_object_header() - return header.NameInfo.Name.String # type: ignore + def get_device_name(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: + """Get device's name from the object header.""" + try: + header = self.get_object_header() + return header.NameInfo.Name.String # type: ignore + except(ValueError): + return renderers.UnparsableValue() + def get_attached_devices(self) -> interfaces.objects.ObjectInterface: + """Enumerate the device's attaches""" + device = self.AttachedDevice.dereference() + while device: + yield device + device = device.AttachedDevice.dereference() class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" - def get_driver_name(self) -> str: - header = self.get_object_header() - return header.NameInfo.Name.String # type: ignore + def get_driver_name(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: + """Get driver's name from the object header.""" + try: + header = self.get_object_header() + return header.NameInfo.Name.String # type: ignore + except(ValueError): + return renderers.UnparsableValue() + + def get_devices(self) -> interfaces.objects.ObjectInterface: + """Enumerate the driver's device objects""" + device = self.DeviceObject.dereference() + while device: + yield device + device = device.NextDevice.dereference() def is_valid(self) -> bool: """Determine if the object is valid.""" From 6b91927bb4f389cde1eb6f9bf339706e42cc1e3b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 00:51:35 +0900 Subject: [PATCH 141/158] Initialize DeviceTree plugin --- .../framework/plugins/windows/devicetree.py | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index e69de29bb..67bfcfbbf 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -0,0 +1,145 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterator, List, Tuple + +from volatility3.framework import constants, renderers, exceptions, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import driverscan + +DEVICE_CODES = { + 0x00000027 : "FILE_DEVICE_8042_PORT", + 0x00000032 : "FILE_DEVICE_ACPI", + 0x00000029 : "FILE_DEVICE_BATTERY", + 0x00000001 : "FILE_DEVICE_BEEP", + 0x0000002a : "FILE_DEVICE_BUS_EXTENDER", + 0x00000002 : "FILE_DEVICE_CD_ROM", + 0x00000003 : "FILE_DEVICE_CD_ROM_FILE_SYSTEM", + 0x00000030 : "FILE_DEVICE_CHANGER", + 0x00000004 : "FILE_DEVICE_CONTROLLER", + 0x00000005 : "FILE_DEVICE_DATALINK", + 0x00000006 : "FILE_DEVICE_DFS", + 0x00000035 : "FILE_DEVICE_DFS_FILE_SYSTEM", + 0x00000036 : "FILE_DEVICE_DFS_VOLUME", + 0x00000007 : "FILE_DEVICE_DISK", + 0x00000008 : "FILE_DEVICE_DISK_FILE_SYSTEM", + 0x00000033 : "FILE_DEVICE_DVD", + 0x00000009 : "FILE_DEVICE_FILE_SYSTEM", + 0x0000003a : "FILE_DEVICE_FIPS", + 0x00000034 : "FILE_DEVICE_FULLSCREEN_VIDEO", + 0x0000000a : "FILE_DEVICE_INPORT_PORT", + 0x0000000b : "FILE_DEVICE_KEYBOARD", + 0x0000002f : "FILE_DEVICE_KS", + 0x00000039 : "FILE_DEVICE_KSEC", + 0x0000000c : "FILE_DEVICE_MAILSLOT", + 0x0000002d : "FILE_DEVICE_MASS_STORAGE", + 0x0000000d : "FILE_DEVICE_MIDI_IN", + 0x0000000e : "FILE_DEVICE_MIDI_OUT", + 0x0000002b : "FILE_DEVICE_MODEM", + 0x0000000f : "FILE_DEVICE_MOUSE", + 0x00000010 : "FILE_DEVICE_MULTI_UNC_PROVIDER", + 0x00000011 : "FILE_DEVICE_NAMED_PIPE", + 0x00000012 : "FILE_DEVICE_NETWORK", + 0x00000013 : "FILE_DEVICE_NETWORK_BROWSER", + 0x00000014 : "FILE_DEVICE_NETWORK_FILE_SYSTEM", + 0x00000028 : "FILE_DEVICE_NETWORK_REDIRECTOR", + 0x00000015 : "FILE_DEVICE_NULL", + 0x00000016 : "FILE_DEVICE_PARALLEL_PORT", + 0x00000017 : "FILE_DEVICE_PHYSICAL_NETCARD", + 0x00000018 : "FILE_DEVICE_PRINTER", + 0x00000019 : "FILE_DEVICE_SCANNER", + 0x0000001c : "FILE_DEVICE_SCREEN", + 0x00000037 : "FILE_DEVICE_SERENUM", + 0x0000001a : "FILE_DEVICE_SERIAL_MOUSE_PORT", + 0x0000001b : "FILE_DEVICE_SERIAL_PORT", + 0x00000031 : "FILE_DEVICE_SMARTCARD", + 0x0000002e : "FILE_DEVICE_SMB", + 0x0000001d : "FILE_DEVICE_SOUND", + 0x0000001e : "FILE_DEVICE_STREAMS", + 0x0000001f : "FILE_DEVICE_TAPE", + 0x00000020 : "FILE_DEVICE_TAPE_FILE_SYSTEM", + 0x00000038 : "FILE_DEVICE_TERMSRV", + 0x00000021 : "FILE_DEVICE_TRANSPORT", + 0x00000022 : "FILE_DEVICE_UNKNOWN", + 0x0000002c : "FILE_DEVICE_VDM", + 0x00000023 : "FILE_DEVICE_VIDEO", + 0x00000024 : "FILE_DEVICE_VIRTUAL_DISK", + 0x00000025 : "FILE_DEVICE_WAVE_IN", + 0x00000026 : "FILE_DEVICE_WAVE_OUT", +} + +vollog = logging.getLogger(__name__) + +class DeviceTree(interfaces.plugins.PluginInterface): + """Listing tree based on drivers and attached devices in a particular windows memory image.""" + + _required_framework_version = (2, 0, 1) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name = "kernel", description = "Windows kernel", + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = "driverscan", plugin = driverscan.DriverScan, version = (1, 0, 0)), + ] + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config["kernel"]] + + # Scan the Layer for drivers + for driver in driverscan.DriverScan.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): + try: + driver_name = driver.get_driver_name() + + yield (0, ( + format_hints.Hex(driver.vol.offset), + "DRV", + driver_name, + renderers.NotApplicableValue(), + renderers.NotApplicableValue() + )) + + # Scan to get the device information of driver. + for device in driver.get_devices(): + device_name = device.get_device_name() + device_type = DEVICE_CODES.get(device.DeviceType, "UNKNOWN") + + yield (1, ( + format_hints.Hex(driver.vol.offset), + "DEV", + driver_name, + device_name, + device_type + )) + + # Scan to get the attached devices information of device. + for level, attached_device in enumerate(device.get_attached_devices(), start=2): + device_name = attached_device.get_device_name() + + attached_device_name = "Unparsable Value" if isinstance(device_name, renderers.UnparsableValue) else device_name + name = "{} - {}".format(attached_device_name, attached_device.DriverObject.DriverName.get_string()) + + attached_device_type = DEVICE_CODES.get(attached_device.DeviceType, "UNKNOWN") + + yield (level, ( + format_hints.Hex(driver.vol.offset), + "ATT", + driver_name, + name, + attached_device_type + )) + + except(exceptions.PagedInvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in drivers and devices: {format_hints.Hex(driver.vol.offset)}") + continue + + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid([ + ("Offset", format_hints.Hex), ("Type", str), ("DriverName", str), ("DeviceName", str), ("DeviceType", str), + ], self._generator()) From 95a11c366965b03c4f69b6b75687e600d2dd231f Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 03:25:16 +0900 Subject: [PATCH 142/158] Core: Bump the development version to 2.0.3 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index badec946b..46d5be577 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_PATCH = 3 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From ff9c5cea4f021485bee51c8ae7fd5b5f3b5b93d8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 03:31:29 +0900 Subject: [PATCH 143/158] Modify return type hint of method for get driver's and device's --- .../framework/symbols/windows/extensions/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index c1972cb7f..2266305fa 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -7,9 +7,10 @@ import datetime import functools import logging import math -from typing import Iterable, Iterator, List, Optional, Tuple, Union +from typing import Generator, Iterable, Iterator, List, Optional, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols +from volatility3.framework.interfaces.objects import ObjectInterface from volatility3.framework.layers import intel from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic @@ -359,7 +360,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): except(ValueError): return renderers.UnparsableValue() - def get_attached_devices(self) -> interfaces.objects.ObjectInterface: + def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the device's attaches""" device = self.AttachedDevice.dereference() while device: @@ -377,7 +378,7 @@ class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): except(ValueError): return renderers.UnparsableValue() - def get_devices(self) -> interfaces.objects.ObjectInterface: + def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" device = self.DeviceObject.dereference() while device: From 2f3bc8cf0d58263c64357ca06ffe0d494648f668 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 03:34:24 +0900 Subject: [PATCH 144/158] Revert method for get driver's and device's name --- .../framework/plugins/windows/devicetree.py | 5 ++--- .../symbols/windows/extensions/__init__.py | 18 ++++++------------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index 67bfcfbbf..add654795 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -77,7 +77,7 @@ vollog = logging.getLogger(__name__) class DeviceTree(interfaces.plugins.PluginInterface): """Listing tree based on drivers and attached devices in a particular windows memory image.""" - _required_framework_version = (2, 0, 1) + _required_framework_version = (2, 0, 3) _version = (1, 0, 0) @classmethod @@ -121,8 +121,7 @@ class DeviceTree(interfaces.plugins.PluginInterface): for level, attached_device in enumerate(device.get_attached_devices(), start=2): device_name = attached_device.get_device_name() - attached_device_name = "Unparsable Value" if isinstance(device_name, renderers.UnparsableValue) else device_name - name = "{} - {}".format(attached_device_name, attached_device.DriverObject.DriverName.get_string()) + name = "{} - {}".format(device_name, attached_device.DriverObject.DriverName.get_string()) attached_device_type = DEVICE_CODES.get(attached_device.DeviceType, "UNKNOWN") diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 2266305fa..b7b53f6e9 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -352,13 +352,10 @@ class EX_FAST_REF(objects.StructType): class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel device objects.""" - def get_device_name(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: + def get_device_name(self) -> str: """Get device's name from the object header.""" - try: - header = self.get_object_header() - return header.NameInfo.Name.String # type: ignore - except(ValueError): - return renderers.UnparsableValue() + header = self.get_object_header() + return header.NameInfo.Name.String # type: ignore def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the device's attaches""" @@ -370,13 +367,10 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" - def get_driver_name(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: + def get_driver_name(self) -> str: """Get driver's name from the object header.""" - try: - header = self.get_object_header() - return header.NameInfo.Name.String # type: ignore - except(ValueError): - return renderers.UnparsableValue() + header = self.get_object_header() + return header.NameInfo.Name.String # type: ignore def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" From 8c29ba9fa5ccb733780378c28ba9c3d5fa856b6c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 03:36:41 +0900 Subject: [PATCH 145/158] Modify code comment of get_attached_devices method --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b7b53f6e9..7d083fbba 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -358,7 +358,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): return header.NameInfo.Name.String # type: ignore def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: - """Enumerate the device's attaches""" + """Enumerate the attached device's objects""" device = self.AttachedDevice.dereference() while device: yield device From 8104ee5fcb6393f6fdfc663a44d164dada15f77d Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 03:43:36 +0900 Subject: [PATCH 146/158] Prettier of TreeGrid column --- volatility3/framework/plugins/windows/devicetree.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index add654795..bd1a1ce36 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -140,5 +140,9 @@ class DeviceTree(interfaces.plugins.PluginInterface): def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([ - ("Offset", format_hints.Hex), ("Type", str), ("DriverName", str), ("DeviceName", str), ("DeviceType", str), + ("Offset", format_hints.Hex), + ("Type", str), + ("DriverName", str), + ("DeviceName", str), + ("DeviceType", str), ], self._generator()) From 7b3fa278e058f296940bd1713b908beab8177664 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 16:03:25 +0900 Subject: [PATCH 147/158] Move handling of ValueError, PagedInvalidAddressException to _generator --- .../framework/plugins/windows/devicetree.py | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py index bd1a1ce36..8e92de0cc 100644 --- a/volatility3/framework/plugins/windows/devicetree.py +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -94,19 +94,31 @@ class DeviceTree(interfaces.plugins.PluginInterface): # Scan the Layer for drivers for driver in driverscan.DriverScan.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): try: - driver_name = driver.get_driver_name() + try: + driver_name = driver.get_driver_name() + except (ValueError, exceptions.PagedInvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, + f"Failed to get Driver name : {driver.vol.offset:x}") + driver_name = renderers.UnparsableValue() yield (0, ( format_hints.Hex(driver.vol.offset), "DRV", driver_name, renderers.NotApplicableValue(), + renderers.NotApplicableValue(), renderers.NotApplicableValue() )) # Scan to get the device information of driver. for device in driver.get_devices(): - device_name = device.get_device_name() + try: + device_name = device.get_device_name() + except (ValueError, exceptions.PagedInvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, + f"Failed to get Device name : {device.vol.offset:x}") + device_name = renderers.UnparsableValue() + device_type = DEVICE_CODES.get(device.DeviceType, "UNKNOWN") yield (1, ( @@ -114,35 +126,42 @@ class DeviceTree(interfaces.plugins.PluginInterface): "DEV", driver_name, device_name, + renderers.NotApplicableValue(), device_type )) # Scan to get the attached devices information of device. for level, attached_device in enumerate(device.get_attached_devices(), start=2): - device_name = attached_device.get_device_name() - - name = "{} - {}".format(device_name, attached_device.DriverObject.DriverName.get_string()) + try: + device_name = attached_device.get_device_name() + except (ValueError, exceptions.PagedInvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, + f"Failed to get Attached Device Name: {attached_device.vol.offset:x}") + device_name = renderers.UnparsableValue() + attached_device_driver_name = attached_device.DriverObject.DriverName.get_string() attached_device_type = DEVICE_CODES.get(attached_device.DeviceType, "UNKNOWN") yield (level, ( format_hints.Hex(driver.vol.offset), "ATT", driver_name, - name, + device_name, + attached_device_driver_name, attached_device_type )) except(exceptions.PagedInvalidAddressException): - vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in drivers and devices: {format_hints.Hex(driver.vol.offset)}") + vollog.log(constants.LOGLEVEL_VVVV, + f"Invalid address identified in drivers and devices: {driver.vol.offset:x}") continue - def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([ ("Offset", format_hints.Hex), ("Type", str), ("DriverName", str), ("DeviceName", str), + ("DriverNameOfAttDevice", str), ("DeviceType", str), ], self._generator()) From 655ab9305d86bbb39ddacfdcad721685df092beb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 20:39:11 +0900 Subject: [PATCH 148/158] Fix typo error for licenses, docs --- doc/source/simple-plugin.rst | 2 +- volatility3/framework/plugins/mac/kauth_scopes.py | 2 +- volatility3/framework/plugins/mac/kevents.py | 2 +- volatility3/framework/plugins/mac/vfsevents.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 9360ccf40..ffc7b263f 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -196,7 +196,7 @@ The plugin then defaults the ``BaseDllName`` and ``FullDllName`` variables to an which is a way of indicating to the user interface that the value couldn't be read for some reason (but that it isn't fatal). There are currently four different reasons a value may be unreadable: -* **Unreadble**: values which are empty because the data cannot be read +* **Unredble**: values which are empty because the data cannot be read * **Unparsable**: values which are empty because the data cannot be interpreted correctly * **NotApplicable**: values which are empty because they don't make sense for this particular entry * **NotAvailable**: values which cannot be provided now (but might in a future run, via new symbols or an updated plugin) diff --git a/volatility3/framework/plugins/mac/kauth_scopes.py b/volatility3/framework/plugins/mac/kauth_scopes.py index 910de35fd..f1a2ad345 100644 --- a/volatility3/framework/plugins/mac/kauth_scopes.py +++ b/volatility3/framework/plugins/mac/kauth_scopes.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 16fa51fa4..6f82c75cd 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # diff --git a/volatility3/framework/plugins/mac/vfsevents.py b/volatility3/framework/plugins/mac/vfsevents.py index 38f4172ce..bc5668495 100644 --- a/volatility3/framework/plugins/mac/vfsevents.py +++ b/volatility3/framework/plugins/mac/vfsevents.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # From 0fcc0d8b36bd20a1814f0cac6d29c37b1629de95 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 28 Mar 2022 20:42:13 +0900 Subject: [PATCH 149/158] Fix typo error of docs (Unreadable) --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index ffc7b263f..8446b0ef5 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -196,7 +196,7 @@ The plugin then defaults the ``BaseDllName`` and ``FullDllName`` variables to an which is a way of indicating to the user interface that the value couldn't be read for some reason (but that it isn't fatal). There are currently four different reasons a value may be unreadable: -* **Unredble**: values which are empty because the data cannot be read +* **Unreadable**: values which are empty because the data cannot be read * **Unparsable**: values which are empty because the data cannot be interpreted correctly * **NotApplicable**: values which are empty because they don't make sense for this particular entry * **NotAvailable**: values which cannot be provided now (but might in a future run, via new symbols or an updated plugin) From f51914746428c3ce61b68838389ba0667c08064d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 28 Mar 2022 18:00:19 +0100 Subject: [PATCH 150/158] Windows: Fix the location of a netstat data file Very kindly pointed out by @Digitalisx. Closes #691 --- .../symbols/windows/{ => netscan}/netscan-win81-19935-x64.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename volatility3/framework/symbols/windows/{ => netscan}/netscan-win81-19935-x64.json (100%) diff --git a/volatility3/framework/symbols/windows/netscan-win81-19935-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win81-19935-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-win81-19935-x64.json rename to volatility3/framework/symbols/windows/netscan/netscan-win81-19935-x64.json From 02569f4e0658a11142eb248bccd0bb8c356a7342 Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Tue, 29 Mar 2022 11:57:24 -0500 Subject: [PATCH 151/158] refs #668 handle freed windows big pools more accurately. add --show_free option to the bigpools plugin --- .../framework/plugins/windows/bigpools.py | 24 ++++++++++++++----- .../symbols/windows/extensions/pool.py | 5 +++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/bigpools.py b/volatility3/framework/plugins/windows/bigpools.py index c81125f07..329ccdb4e 100644 --- a/volatility3/framework/plugins/windows/bigpools.py +++ b/volatility3/framework/plugins/windows/bigpools.py @@ -21,7 +21,7 @@ class BigPools(interfaces.plugins.PluginInterface): """List big page pools.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -32,7 +32,11 @@ class BigPools(interfaces.plugins.PluginInterface): requirements.StringRequirement(name = 'tags', description = "Comma separated list of pool tags to filter pools returned", optional = True, - default = None) + default = None), + requirements.BooleanRequirement(name = 'show_free', + description = 'Show freed regions (otherwise only show allocations in use)', + default = False, + optional = True) ] @classmethod @@ -40,7 +44,8 @@ class BigPools(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - tags: Optional[list] = None): + tags: Optional[list] = None, + show_free: bool = False): """Returns the big page pool objects from the kernel PoolBigPageTable array. Args: @@ -97,7 +102,7 @@ class BigPools(interfaces.plugins.PluginInterface): for big_pool in big_pools: if big_pool.is_valid(): - if tags is None or big_pool.get_key() in tags: + if (tags is None or big_pool.get_key() in tags) and (show_free or not big_pool.is_free()): yield big_pool def _generator(self) -> Iterator[Tuple[int, Tuple[int, str]]]: # , str, int]]]: @@ -110,13 +115,19 @@ class BigPools(interfaces.plugins.PluginInterface): for big_pool in self.list_big_pools(context = self.context, layer_name = kernel.layer_name, symbol_table = kernel.symbol_table_name, - tags = tags): + tags = tags, + show_free = self.config.get("show_free")): num_bytes = big_pool.get_number_of_bytes() if not isinstance(num_bytes, interfaces.renderers.BaseAbsentValue): num_bytes = format_hints.Hex(num_bytes) - yield (0, (format_hints.Hex(big_pool.Va), big_pool.get_key(), big_pool.get_pool_type(), num_bytes)) + if big_pool.is_free(): + status = "Free" + else: + status = "Allocated" + + yield (0, (format_hints.Hex(big_pool.Va), big_pool.get_key(), big_pool.get_pool_type(), num_bytes, status)) def run(self): return renderers.TreeGrid([ @@ -124,4 +135,5 @@ class BigPools(interfaces.plugins.PluginInterface): ('Tag', str), ('PoolType', str), ('NumberOfBytes', format_hints.Hex), + ('Status', str), ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index d50d6e47a..368765497 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -233,7 +233,10 @@ class POOL_TRACKER_BIG_PAGES(objects.StructType): def is_valid(self) -> bool: return self.Key > 0 - # return self.Va > 0x1 + + def is_free(self) -> bool: + """Returns if the allocation is freed (True) or in-use (False)""" + return self.Va & 1 == 1 def get_key(self) -> str: """Returns the Key value as a 4 character string""" From 0673282539bd706e44de2fbfbf1ca0f244302a63 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 10 Apr 2022 23:36:34 +0100 Subject: [PATCH 152/158] Configuration: Change unsatisfied response for Modules --- volatility3/cli/__init__.py | 14 ++++++------- .../framework/configuration/requirements.py | 20 +++++++++++++++---- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 35ad84011..488892d37 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,6 @@ import os import sys import tempfile import traceback -from datetime import datetime from typing import Any, Dict, Type, Union from urllib import parse, request @@ -453,16 +452,17 @@ class CommandLine: print(f"Unsatisfied requirement {config_path}: {excp.unsatisfied[config_path].description}") - if symbols_failed: - print("\nA symbol table requirement was not fulfilled. Please verify that:\n" - "\tYou have the correct symbol file for the requirement\n" - "\tThe symbol file is under the correct directory or zip file\n" - "\tThe symbol file is named appropriately or contains the correct banner\n") if translation_failed: print("\nA translation layer requirement was not fulfilled. Please verify that:\n" "\tA file was provided to create this layer (by -f, --single-location or by config)\n" "\tThe file exists and is readable\n" - "\tThe necessary symbols are present and identified by volatility3") + "\tThe file is a valid memory image and was acquired cleanly") + if symbols_failed: + print("\nA symbol table requirement was not fulfilled. Please verify that:\n" + "\tThe associated translation layer requirement was fulfilled\n" + "\tYou have the correct symbol file for the requirement\n" + "\tThe symbol file is under the correct directory or zip file\n" + "\tThe symbol file is named appropriately or contains the correct banner\n") def populate_config(self, context: interfaces.context.ContextInterface, configurables_list: Dict[str, Type[interfaces.configuration.ConfigurableInterface]], diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index a0fb186ae..746b72226 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -10,7 +10,7 @@ expect to be in the context (such as particular layers or symboltables). """ import abc import logging -from typing import Any, ClassVar, List, Optional, Type, Dict, Tuple +from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type from volatility3.framework import constants, interfaces @@ -303,7 +303,8 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if not subreq.optional]): + [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if + not subreq.optional]): return None obj = self._construct_class(context, config_path, args) @@ -358,7 +359,8 @@ class SymbolTableRequirement(interfaces.configuration.ConstructableRequirementIn args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if not subreq.optional]): + [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if + not subreq.optional]): return None # Fill out the parameter for class creation @@ -462,6 +464,15 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa "TypeError - Module Requirement only accepts string labels: {}".format(repr(value))) return {config_path: self} + result = {} + for subreq in self._requirements: + req_unsatisfied = self._requirements[subreq].unsatisfied(context, config_path) + if req_unsatisfied: + result.update(req_unsatisfied) + if not result: + result = {config_path: self} + return result + ### NOTE: This validate method has side effects (the dependencies can change)!!! self._validate_class(context, interfaces.configuration.parent_path(config_path)) @@ -482,7 +493,8 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if not subreq.optional]): + [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if + not subreq.optional]): return None obj = self._construct_class(context, config_path, args) From fd81dba42064414f93e20c724580502be2412528 Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Wed, 20 Apr 2022 13:21:43 -0500 Subject: [PATCH 153/158] refs #668 bump the minor version and not the revision for an additive change --- volatility3/framework/plugins/windows/bigpools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/bigpools.py b/volatility3/framework/plugins/windows/bigpools.py index 329ccdb4e..f8bb332df 100644 --- a/volatility3/framework/plugins/windows/bigpools.py +++ b/volatility3/framework/plugins/windows/bigpools.py @@ -21,7 +21,7 @@ class BigPools(interfaces.plugins.PluginInterface): """List big page pools.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 3e70030d1743ee7b45d33fb1dbb65dee1675ef60 Mon Sep 17 00:00:00 2001 From: iMHLv2 Date: Wed, 20 Apr 2022 13:22:18 -0500 Subject: [PATCH 154/158] refs #668 by convention, use show-free instead of show_free --- volatility3/framework/plugins/windows/bigpools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/bigpools.py b/volatility3/framework/plugins/windows/bigpools.py index f8bb332df..9e120446f 100644 --- a/volatility3/framework/plugins/windows/bigpools.py +++ b/volatility3/framework/plugins/windows/bigpools.py @@ -33,7 +33,7 @@ class BigPools(interfaces.plugins.PluginInterface): description = "Comma separated list of pool tags to filter pools returned", optional = True, default = None), - requirements.BooleanRequirement(name = 'show_free', + requirements.BooleanRequirement(name = 'show-free', description = 'Show freed regions (otherwise only show allocations in use)', default = False, optional = True) @@ -116,7 +116,7 @@ class BigPools(interfaces.plugins.PluginInterface): layer_name = kernel.layer_name, symbol_table = kernel.symbol_table_name, tags = tags, - show_free = self.config.get("show_free")): + show_free = self.config.get("show-free")): num_bytes = big_pool.get_number_of_bytes() if not isinstance(num_bytes, interfaces.renderers.BaseAbsentValue): From 1154a2a9f59da058fd9beb19713446c1b84b8218 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Apr 2022 22:20:00 +0100 Subject: [PATCH 155/158] Symbols: Catch bad JSON files without metadata Fixes #719 --- volatility3/framework/symbols/intermed.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index c48fcc8a7..a6a7a0fae 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -11,13 +11,13 @@ import os import pathlib import zipfile from abc import ABCMeta -from typing import Any, Dict, Generator, Iterable, List, Optional, Type, Tuple, Mapping +from typing import Any, Dict, Generator, Iterable, List, Mapping, Optional, Tuple, Type from volatility3 import schemas, symbols from volatility3.framework import class_subclasses, constants, exceptions, interfaces, objects from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources -from volatility3.framework.symbols import native, metadata +from volatility3.framework.symbols import metadata, native vollog = logging.getLogger(__name__) @@ -113,6 +113,9 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): metadata = json_object.get('metadata', None) + if not metadata: + raise exceptions.SymbolSpaceError(f"Invalid ISF file attempted to be parsed: {isf_url}") + # Determine the delegate or throw an exception self._delegate = self._closest_version(metadata.get('format', "0.0.0"), self._versions)(context, config_path, name, json_object, native_types, @@ -540,7 +543,8 @@ class Version3Format(Version2Format): if 'type' in symbol: symbol_type = self._interdict_to_template(symbol['type']) - self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address, type = symbol_type) + self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address, + type = symbol_type) return self._symbol_cache[name] From 4e2dd7b24fb1c2452cbfdb685d0c21614da3edba Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Apr 2022 22:49:05 +0100 Subject: [PATCH 156/158] Core: Bump patch number for additive change to API --- API_CHANGES.md | 19 +++++++++++++++++++ volatility3/framework/constants/__init__.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index 4a65de04b..b3962f072 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,25 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.0.4 +===== +Add in the linux `task.get_threads` method added to rhe API. + +2.0.3 +===== +`DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` added to the API. + +2.0.2 +===== +Fix the behaviour of the offsets returned by the PDB scanner. + +2.0.0 +===== +Remove the `symbol_shift` mechanism, where symbol tables could alter their own symbols. +Symbols from a symbol table are now always the offset values. They can be added to a Module +and when symbols are requested from a Module they are shifted by the module's offset to get +an absolute offset. This can be done with `Module.get_absolute_symbol_address` or as part of +`Module.object_from_symbol(absolute = False, ...)`. 1.2.0 ===== diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 46d5be577..ddc96bf31 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -40,7 +40,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 3 # Number of changes that do not change the interface +VERSION_PATCH = 4 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 0dc0b8ca4019b8401b5478e25949ee2647caebb1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 28 Apr 2022 00:12:41 +0100 Subject: [PATCH 157/158] Core: Bump the API correctly for an addition (to 2.1.0) --- API_CHANGES.md | 2 +- volatility3/framework/constants/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index b3962f072..03fcc010c 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,7 +4,7 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. -2.0.4 +2.1.0 ===== Add in the linux `task.get_threads` method added to rhe API. diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index ddc96bf31..5060906d5 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -39,8 +39,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 4 # Number of changes that do not change the interface +VERSION_MINOR = 1 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From a236bdd60047702dea18aed0170dcc2efc4a0cc2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 28 Apr 2022 08:26:52 +0900 Subject: [PATCH 158/158] Fix typo for API_CHANGES.md --- API_CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index 03fcc010c..4e1820eff 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -6,7 +6,7 @@ When an API feature or function is removed or changed, the major version is bump 2.1.0 ===== -Add in the linux `task.get_threads` method added to rhe API. +Add in the linux `task.get_threads` method added to the API. 2.0.3 =====