From 3c3b2b3bbd12576cb1a489fbe273a34de29160e6 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 2 Apr 2025 14:14:52 -0500 Subject: [PATCH 01/19] MFT Extensions: Fix type hints These type hints are a bit misleading, and have been updated to reflect their real return type. --- volatility3/framework/symbols/windows/extensions/mft.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index ebba882c0..86580be16 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -10,7 +10,7 @@ from volatility3.framework import objects, constants, exceptions class MFTEntry(objects.StructType): """This represents the base MFT Record""" - def get_signature(self) -> str: + def get_signature(self) -> "objects.String": signature = self.Signature.cast("string", max_length=4, encoding="latin-1") return signature @@ -18,7 +18,7 @@ class MFTEntry(objects.StructType): class MFTFileName(objects.StructType): """This represents an MFT $FILE_NAME Attribute""" - def get_full_name(self) -> str: + def get_full_name(self) -> "objects.String": output = self.Name.cast( "string", encoding="utf16", max_length=self.NameLength * 2, errors="replace" ) @@ -28,7 +28,7 @@ class MFTFileName(objects.StructType): class MFTAttribute(objects.StructType): """This represents an MFT ATTRIBUTE""" - def get_resident_filename(self) -> Optional[str]: + def get_resident_filename(self) -> Optional["objects.String"]: # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous if ( @@ -51,7 +51,7 @@ class MFTAttribute(objects.StructType): except exceptions.InvalidAddressException: return None - def get_resident_filecontent(self) -> Optional[bytes]: + def get_resident_filecontent(self) -> Optional["objects.Bytes"]: # smear observed in mass testing of samples # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems if ( From 86c5c16ed6f9729913a6ba797013a2ad03d6faa0 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 2 Apr 2025 14:40:44 -0500 Subject: [PATCH 02/19] Windows MFTScan Plugins: Performance fixes There was a subtle issue that was causing substantial performance issues in the MFTScan plugins. The `record_map` was purportedly of type `Dict[str, Tuple[int, str, int]]`, but in reality, the second member was a list, and its `str` item was actually being populated with unprocessed values from method calls on the MFT extension classes, which actually return `object.String`. These objects are substantially larger than basic `str` types: ``` [ins] In [5]: pympler.asizeof.asizeof(rec_name) Out[5]: 312648 [ins] In [6]: pympler.asizeof.asizeof(str(rec_name)) Out[6]: 64 ``` This caused this dictionary to grow in size to several gigabytes on larger samples, resulting in thrashing and OOM errors. --- .../framework/plugins/windows/mftscan.py | 130 +++++++++++------- 1 file changed, 79 insertions(+), 51 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 7f9095c6a..9281c964e 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -4,8 +4,14 @@ import contextlib import datetime import logging +from typing import ( + Callable, + Iterator, + Optional, + Tuple, + DefaultDict, +) -from typing import Generator, Iterable, Dict, Tuple, Callable from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -17,6 +23,22 @@ from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) +class MFTRecord: + # TODO: Change to dataclass with (slots=True) if/when we move minimum + # Python version up to 3.10 + __slots__ = ["record_name", "data_count", "offset"] + + def __init__( + self, + record_name: Optional[str] = None, + data_count: int = 0, + offset: Optional[int] = None, + ): + self.record_name = record_name + self.data_count = data_count + self.offset = offset + + class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" @@ -53,14 +75,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): primary_layer_name: str, attr_callback: Callable[ [ - Dict[int, Tuple[str, int, int]], - interfaces.objects.ObjectInterface, - interfaces.objects.ObjectInterface, + DefaultDict[str, MFTRecord], + mft.MFTEntry, + mft.MFTAttribute, str, ], - Generator, + Iterator[Tuple], ], - ) -> interfaces.objects.ObjectInterface: + ) -> Iterator[Tuple]: try: primary = context.layers[primary_layer_name] except KeyError: @@ -70,14 +92,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): return try: - phys_layer = primary.config["memory_layer"] + memory_layer_name = primary.config["memory_layer"] except KeyError: vollog.error( "Unable to obtain memory layer from primary layer. Please file a bug on GitHub about this issue." ) return - layer = context.layers[phys_layer] + layer = context.layers[memory_layer_name] # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( @@ -98,23 +120,24 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # get each of the individual Field Sets - mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" + mft_object_typ_name = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object_typ_name = symbol_table + constants.BANG + "ATTRIBUTE" - record_map = {} + record_map: DefaultDict[str, MFTRecord] = DefaultDict(MFTRecord) # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan( context=context, scanner=yarascan.YaraScanner(rules=rules) ): with contextlib.suppress(exceptions.InvalidAddressException): - mft_record = context.object( - mft_object, offset=offset, layer_name=layer.name + mft_record: mft.MFTEntry = context.object( + mft_object_typ_name, offset=offset, layer_name=layer.name ) + # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - attr = context.object( - attribute_object, + attr: mft.MFTAttribute = context.object( + attribute_object_typ_name, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -131,8 +154,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Update the base offset to point to the next attribute attr_base_offset += attr.Attr_Header.Length # Get the next attribute - attr = context.object( - attribute_object, + attr: mft.MFTAttribute = context.object( + attribute_object_typ_name, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -140,9 +163,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def parse_mft_records( cls, - record_map: Dict[int, Tuple[str, int, int]], - mft_record: interfaces.objects.ObjectInterface, - attr: interfaces.objects.ObjectInterface, + record_map: DefaultDict[str, MFTRecord], + mft_record: mft.MFTEntry, + attr: mft.MFTAttribute, symbol_table_name: str, ): # MFT Flags determine the file type or dir @@ -160,7 +183,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): attr_data = attr.Attr_Data.cast(si_object) yield 0, ( format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), + str(mft_record.get_signature()), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, @@ -178,7 +201,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" attr_data = attr.Attr_Data.cast(fn_object) - file_name = attr_data.get_full_name() + file_name = str(attr_data.get_full_name()) # If we don't have a valid enum, coerce to hex so we can keep the record try: @@ -188,7 +211,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): yield 1, ( format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), + str(mft_record.get_signature()), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, @@ -204,11 +227,11 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def parse_data_record( cls, - mft_record: interfaces.objects.ObjectInterface, - attr: interfaces.objects.ObjectInterface, - record_map: Dict[int, Tuple[str, int, int]], + mft_record: mft.MFTEntry, + attr: mft.MFTAttribute, + record_map: DefaultDict[str, MFTRecord], return_first_record: bool, - ) -> Generator[Iterable, None, None]: + ) -> Iterator[Tuple]: """ Returns the parsed data from a MFT record """ @@ -227,7 +250,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): else: # past the first $DATA record, attempt to get the ADS name # NotAvailableValue = > 1st Data, but name was not parsable - ads_name = attr.get_resident_filename() or renderers.NotAvailableValue() + ads_name_obj = attr.get_resident_filename() + ads_name = ( + str(ads_name_obj) + if ads_name_obj is not None + else renderers.NotAvailableValue() + ) content = attr.get_resident_filecontent() if content: @@ -236,11 +264,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): content = renderers.NotAvailableValue() yield ( - format_hints.Hex(record_map[mft_record.vol.offset][2]), - mft_record.get_signature(), + format_hints.Hex(record_map[mft_record.vol.offset].offset), + str(mft_record.get_signature()), mft_record.RecordNumber, attr.Attr_Header.AttrType.lookup(), - record_map[mft_record.vol.offset][0], + record_map[mft_record.vol.offset].record_name + or renderers.NotAvailableValue(), ads_name, content, ) @@ -248,41 +277,40 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def parse_data_records( cls, - record_map: Dict[int, Tuple[str, int, int]], - mft_record: interfaces.objects.ObjectInterface, - attr: interfaces.objects.ObjectInterface, + record_map: DefaultDict[str, MFTRecord], + mft_record: mft.MFTEntry, + attr: mft.MFTAttribute, symbol_table_name: str, return_first_record: bool, - ) -> Generator[Iterable, None, None]: + ) -> Iterator[Tuple]: """ Parses DATA records while maintaining the FILE_NAME association from previous parsing of the record Suports returning the first/main $DATA as well as however many ADS records a file might have """ - if mft_record.vol.offset not in record_map: - # file name, DATA count, offset - record_map[mft_record.vol.offset] = [renderers.NotAvailableValue(), 0, None] + rec = record_map[mft_record.vol.offset] + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" - attr_data = attr.Attr_Data.cast(fn_object) - rec_name = attr_data.get_full_name() - record_map[mft_record.vol.offset][0] = rec_name + fn_object_typename = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" + attr_data = attr.Attr_Data.cast(fn_object_typename) + name_obj = attr_data.get_full_name() + rec.record_name = str(name_obj) if name_obj is not None else None elif attr.Attr_Header.AttrType.lookup() == "DATA": # first data - record_map[mft_record.vol.offset][2] = attr.Attr_Data.vol.offset + rec.offset = attr.Attr_Data.vol.offset display_data = False # first DATA attribute of this record - if record_map[mft_record.vol.offset][1] == 0: + if rec.data_count == 0: if return_first_record: display_data = True - record_map[mft_record.vol.offset][1] = 1 + rec.data_count = 1 # at the second DATA attribute of this record - elif record_map[mft_record.vol.offset][1] == 1 and not return_first_record: + elif rec.data_count == 1 and not return_first_record: display_data = True if display_data: @@ -357,7 +385,7 @@ class ADS(interfaces.plugins.PluginInterface): @classmethod def parse_ads_data_records( cls, - record_map: Dict[int, Tuple[str, int, int]], + record_map: DefaultDict[str, MFTRecord], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, symbol_table_name: str, @@ -427,11 +455,11 @@ class ResidentData(interfaces.plugins.PluginInterface): @classmethod def parse_first_data_records( cls, - record_map: Dict[int, Tuple[str, int, int]], - mft_record: interfaces.objects.ObjectInterface, - attr: interfaces.objects.ObjectInterface, + record_map: DefaultDict[str, MFTRecord], + mft_record: mft.MFTEntry, + attr: mft.MFTAttribute, symbol_table_name: str, - ): + ) -> Iterator[Tuple]: return MFTScan.parse_data_records( record_map, mft_record, attr, symbol_table_name, True ) From 57524edb874a2dbb30ad2adba3d092f39757c4c4 Mon Sep 17 00:00:00 2001 From: David McDonald <49174690+dgmcdona@users.noreply.github.com> Date: Thu, 3 Apr 2025 09:58:35 -0500 Subject: [PATCH 03/19] Remove unnecessary quotes from type hints Co-authored-by: ikelos --- volatility3/framework/symbols/windows/extensions/mft.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 86580be16..9dd7f1a6a 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -10,7 +10,7 @@ from volatility3.framework import objects, constants, exceptions class MFTEntry(objects.StructType): """This represents the base MFT Record""" - def get_signature(self) -> "objects.String": + def get_signature(self) -> objects.String: signature = self.Signature.cast("string", max_length=4, encoding="latin-1") return signature @@ -18,7 +18,7 @@ class MFTEntry(objects.StructType): class MFTFileName(objects.StructType): """This represents an MFT $FILE_NAME Attribute""" - def get_full_name(self) -> "objects.String": + def get_full_name(self) -> objects.String: output = self.Name.cast( "string", encoding="utf16", max_length=self.NameLength * 2, errors="replace" ) @@ -28,7 +28,7 @@ class MFTFileName(objects.StructType): class MFTAttribute(objects.StructType): """This represents an MFT ATTRIBUTE""" - def get_resident_filename(self) -> Optional["objects.String"]: + def get_resident_filename(self) -> Optional[objects.String]: # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous if ( @@ -51,7 +51,7 @@ class MFTAttribute(objects.StructType): except exceptions.InvalidAddressException: return None - def get_resident_filecontent(self) -> Optional["objects.Bytes"]: + def get_resident_filecontent(self) -> Optional[objects.Bytes]: # smear observed in mass testing of samples # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems if ( From e84036c5a6b56f3899c339b04b445c4d81cbf7bc Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 10:02:08 -0500 Subject: [PATCH 04/19] Add missing 'e' to variable names --- .../framework/plugins/windows/mftscan.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 9281c964e..0c204ac1f 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -4,14 +4,7 @@ import contextlib import datetime import logging -from typing import ( - Callable, - Iterator, - Optional, - Tuple, - DefaultDict, -) - +from typing import Callable, DefaultDict, Iterator, Optional, Tuple from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -120,8 +113,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # get each of the individual Field Sets - mft_object_typ_name = symbol_table + constants.BANG + "MFT_ENTRY" - attribute_object_typ_name = symbol_table + constants.BANG + "ATTRIBUTE" + mft_object_type_name = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object_type_name = symbol_table + constants.BANG + "ATTRIBUTE" record_map: DefaultDict[str, MFTRecord] = DefaultDict(MFTRecord) @@ -131,13 +124,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ): with contextlib.suppress(exceptions.InvalidAddressException): mft_record: mft.MFTEntry = context.object( - mft_object_typ_name, offset=offset, layer_name=layer.name + mft_object_type_name, offset=offset, layer_name=layer.name ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset attr: mft.MFTAttribute = context.object( - attribute_object_typ_name, + attribute_object_type_name, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -155,7 +148,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): attr_base_offset += attr.Attr_Header.Length # Get the next attribute attr: mft.MFTAttribute = context.object( - attribute_object_typ_name, + attribute_object_type_name, offset=offset + attr_base_offset, layer_name=layer.name, ) From 4492da0263d866ed36d0e6f0b197789896d347b2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 10:38:13 -0500 Subject: [PATCH 05/19] Create attribute iterator method Moves logic for iterating through `MFTEntry` attributes into a new `attributes()` method on the extension class. --- .../framework/plugins/windows/mftscan.py | 28 ++--------------- .../symbols/windows/extensions/mft.py | 31 ++++++++++++++++++- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 0c204ac1f..78312f284 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -114,7 +114,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object_type_name = symbol_table + constants.BANG + "MFT_ENTRY" - attribute_object_type_name = symbol_table + constants.BANG + "ATTRIBUTE" record_map: DefaultDict[str, MFTRecord] = DefaultDict(MFTRecord) @@ -127,30 +126,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_object_type_name, offset=offset, layer_name=layer.name ) - # We will update this on each pass in the next loop and use it as the new offset. - attr_base_offset = mft_record.FirstAttrOffset - attr: mft.MFTAttribute = context.object( - attribute_object_type_name, - offset=offset + attr_base_offset, - layer_name=layer.name, - ) - - # There is no field that has a count of Attributes - # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr.Attr_Header.AttrType.is_valid_choice: - yield from attr_callback(record_map, mft_record, attr, symbol_table) - - # If there's no advancement the loop will never end, so break it now - if attr.Attr_Header.Length == 0: - break - - # Update the base offset to point to the next attribute - attr_base_offset += attr.Attr_Header.Length - # Get the next attribute - attr: mft.MFTAttribute = context.object( - attribute_object_type_name, - offset=offset + attr_base_offset, - layer_name=layer.name, + for attribute in mft_record.attributes(symbol_table): + yield from attr_callback( + record_map, mft_record, attribute, symbol_table ) @classmethod diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 9dd7f1a6a..4e1140e25 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Optional +from typing import Optional, Iterator from volatility3.framework import objects, constants, exceptions @@ -14,6 +14,35 @@ class MFTEntry(objects.StructType): signature = self.Signature.cast("string", max_length=4, encoding="latin-1") return signature + def attributes(self, symbol_table_name: str) -> Iterator["MFTAttribute"]: + # We will update this on each pass in the next loop and use it as the new offset. + attr_base_offset = self.FirstAttrOffset + attribute_object_type_name = symbol_table_name + constants.BANG + "ATTRIBUTE" + + attr: MFTAttribute = self._context.object( + attribute_object_type_name, + offset=self.vol.offset + attr_base_offset, + layer_name=self.vol.layer_name, + ) + + # There is no field that has a count of Attributes + # Keep Attempting to read attributes until we get an invalid attr_header.AttrType + while attr.Attr_Header.AttrType.is_valid_choice: + yield attr + + # If there's no advancement the loop will never end, so break it now + if attr.Attr_Header.Length == 0: + break + + # Update the base offset to point to the next attribute + attr_base_offset += attr.Attr_Header.Length + # Get the next attribute + attr: MFTAttribute = self._context.object( + attribute_object_type_name, + offset=self.vol.offset + attr_base_offset, + layer_name=self.vol.layer_name, + ) + class MFTFileName(objects.StructType): """This represents an MFT $FILE_NAME Attribute""" From 43e6fefe395f901ec1ae01faf325b8a00c33e7b9 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 11:33:56 -0500 Subject: [PATCH 06/19] Add attribute iterator to MFTEntry extension class --- .../framework/plugins/windows/mftscan.py | 13 +++-- .../symbols/windows/extensions/mft.py | 56 +++++++++++++++++-- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 78312f284..0ad2747c3 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -100,7 +100,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # Read in the Symbol File - symbol_table = intermed.IntermediateSymbolTable.create( + symbol_table_name = intermed.IntermediateSymbolTable.create( context=context, config_path=config_path, sub_path="windows", @@ -113,9 +113,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # get each of the individual Field Sets - mft_object_type_name = symbol_table + constants.BANG + "MFT_ENTRY" record_map: DefaultDict[str, MFTRecord] = DefaultDict(MFTRecord) + mft_object_type_name = symbol_table_name + constants.BANG + "MFT_ENTRY" # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan( @@ -123,12 +123,15 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ): with contextlib.suppress(exceptions.InvalidAddressException): mft_record: mft.MFTEntry = context.object( - mft_object_type_name, offset=offset, layer_name=layer.name + mft_object_type_name, + offset=offset, + layer_name=layer.name, + symbol_table_name=symbol_table_name, ) - for attribute in mft_record.attributes(symbol_table): + for attribute in mft_record.attributes(): yield from attr_callback( - record_map, mft_record, attribute, symbol_table + record_map, mft_record, attribute, symbol_table_name ) @classmethod diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 4e1140e25..e004aa590 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,22 +2,70 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Optional, Iterator +from typing import Dict, Iterator, List, Optional, Tuple -from volatility3.framework import objects, constants, exceptions +from volatility3.framework import constants, exceptions, interfaces, objects class MFTEntry(objects.StructType): """This represents the base MFT Record""" + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + size: int, + members: Dict[str, Tuple[int, interfaces.objects.Template]], + **kwargs, + ) -> None: + super().__init__(context, type_name, object_info, size, members) + + self._symbol_table_name = kwargs.get("symbol_table_name") + self._attr_generator = self._attributes() + self._attrs: List[MFTAttribute] = [] + + @property + def symbol_table_name(self) -> str: + if self._symbol_table_name is None: + raise ValueError( + "MFTEntry was instantiated without an MFT symbol table name" + ) + return self._symbol_table_name + def get_signature(self) -> objects.String: signature = self.Signature.cast("string", max_length=4, encoding="latin-1") return signature - def attributes(self, symbol_table_name: str) -> Iterator["MFTAttribute"]: + def filename(self, symbol_table_name: str) -> Optional[objects.String]: + try: + fname_attr = next( + attr + for attr in self.attributes() + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME" + ) + except StopIteration: + return None + + fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" + attr_data = fname_attr.Attr_Data.cast(fn_object) + + return attr_data.get_full_name() + + def attributes(self) -> Iterator["MFTAttribute"]: + yield from self._attrs + + for attr in self._attr_generator: + self._attrs.append(attr) + yield attr + + def _attributes(self) -> Iterator["MFTAttribute"]: + # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = self.FirstAttrOffset - attribute_object_type_name = symbol_table_name + constants.BANG + "ATTRIBUTE" + attribute_object_type_name = ( + self.symbol_table_name + constants.BANG + "ATTRIBUTE" + ) attr: MFTAttribute = self._context.object( attribute_object_type_name, From 41cf17ed653eb5794c4fc7ba7146b296014882b5 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 16:43:57 -0500 Subject: [PATCH 07/19] Refactor: Ditch dictionary usage, eliminate callbacks This simplifies the design of these plugins by moving as much MFTEntry specific data into the extension class (caching attributes, since they'll need to be accessed repeatedly) and moving away from the callback-based implementation to one where classmethods consume `mft.MFTEntry` objects in order to produce their values. These changes do two important things: - They allow us to preserve `object.String` objects until the generator function, which makes the public interface much better since people can navigate back the the source of the data within their context - Completely eliminates the `record_map` that was causing so much memory consumption. --- .../framework/plugins/windows/mftscan.py | 437 +++++++++--------- .../symbols/windows/extensions/mft.py | 105 +++-- 2 files changed, 282 insertions(+), 260 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 0ad2747c3..a4cdad582 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -1,12 +1,11 @@ # This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import contextlib import datetime import logging -from typing import Callable, DefaultDict, Iterator, Optional, Tuple +from typing import Iterator, NamedTuple, Optional, Tuple, Union -from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework import constants, exceptions, interfaces, objects, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed @@ -16,22 +15,6 @@ from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) -class MFTRecord: - # TODO: Change to dataclass with (slots=True) if/when we move minimum - # Python version up to 3.10 - __slots__ = ["record_name", "data_count", "offset"] - - def __init__( - self, - record_name: Optional[str] = None, - data_count: int = 0, - offset: Optional[int] = None, - ): - self.record_name = record_name - self.data_count = data_count - self.offset = offset - - class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" @@ -39,6 +22,20 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _version = (2, 0, 1) + class MFTScanResult(NamedTuple): + offset: format_hints.Hex + record_type: str + record_number: int + link_count: int + mft_type: str + permissions: Union[str, interfaces.renderers.BaseAbsentValue] + attribute_type: str + created: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] + modified: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] + updated: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] + accessed: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] + filename: Union[interfaces.renderers.BaseAbsentValue, objects.String] + @classmethod def get_requirements(cls): return [ @@ -66,16 +63,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): context: interfaces.context.ContextInterface, config_path: str, primary_layer_name: str, - attr_callback: Callable[ - [ - DefaultDict[str, MFTRecord], - mft.MFTEntry, - mft.MFTAttribute, - str, - ], - Iterator[Tuple], - ], - ) -> Iterator[Tuple]: + ) -> Iterator[mft.MFTEntry]: try: primary = context.layers[primary_layer_name] except KeyError: @@ -114,34 +102,25 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets - record_map: DefaultDict[str, MFTRecord] = DefaultDict(MFTRecord) mft_object_type_name = symbol_table_name + constants.BANG + "MFT_ENTRY" # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan( context=context, scanner=yarascan.YaraScanner(rules=rules) ): - with contextlib.suppress(exceptions.InvalidAddressException): - mft_record: mft.MFTEntry = context.object( - mft_object_type_name, - offset=offset, - layer_name=layer.name, - symbol_table_name=symbol_table_name, - ) + mft_record: mft.MFTEntry = context.object( + mft_object_type_name, + offset=offset, + layer_name=layer.name, + symbol_table_name=symbol_table_name, + ) - for attribute in mft_record.attributes(): - yield from attr_callback( - record_map, mft_record, attribute, symbol_table_name - ) + yield mft_record @classmethod - def parse_mft_records( - cls, - record_map: DefaultDict[str, MFTRecord], - mft_record: mft.MFTEntry, - attr: mft.MFTAttribute, - symbol_table_name: str, - ): + def parse_standard_information_records( + cls, mft_record: mft.MFTEntry + ) -> Iterator[Tuple[int, MFTScanResult]]: # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record try: @@ -150,155 +129,104 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": - si_object = ( - symbol_table_name + constants.BANG + "STANDARD_INFORMATION_ENTRY" - ) - attr_data = attr.Attr_Data.cast(si_object) - yield 0, ( - format_hints.Hex(attr_data.vol.offset), - str(mft_record.get_signature()), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - renderers.NotApplicableValue(), - attr.Attr_Header.AttrType.lookup(), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - renderers.NotApplicableValue(), - ) + try: + # There should only be one STANDARD_INFORMATION attribute, but we + # do this just in case. + for std_information in mft_record.standard_information_attributes(): + yield 0, cls.MFTScanResult( + format_hints.Hex(std_information.vol.offset), + str(mft_record.get_signature()), + mft_record.RecordNumber, + mft_record.LinkCount, + mft_flag, + renderers.NotApplicableValue(), + "STANDARD_INFORMATION", + conversion.wintime_to_datetime(std_information.CreationTime), + conversion.wintime_to_datetime(std_information.ModifiedTime), + conversion.wintime_to_datetime(std_information.UpdatedTime), + conversion.wintime_to_datetime(std_information.AccessedTime), + renderers.NotApplicableValue(), + ) + except exceptions.InvalidAddressException: + pass + + @classmethod + def parse_filename_records( + cls, mft_record: mft.MFTEntry + ) -> Iterator[Tuple[int, MFTScanResult]]: + # MFT Flags determine the file type or dir + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + mft_flag = mft_record.Flags.lookup() + except ValueError: + mft_flag = hex(mft_record.Flags) # File Name Attribute - elif attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" + try: + for filename_info in mft_record.filename_attributes(): - attr_data = attr.Attr_Data.cast(fn_object) - file_name = str(attr_data.get_full_name()) + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + permissions = filename_info.Flags.lookup() + except ValueError: + permissions = hex(filename_info.Flags) - # If we don't have a valid enum, coerce to hex so we can keep the record - try: - permissions = attr_data.Flags.lookup() - except ValueError: - permissions = hex(attr_data.Flags) - - yield 1, ( - format_hints.Hex(attr_data.vol.offset), - str(mft_record.get_signature()), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - permissions, - attr.Attr_Header.AttrType.lookup(), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - file_name, - ) - - @classmethod - def parse_data_record( - cls, - mft_record: mft.MFTEntry, - attr: mft.MFTAttribute, - record_map: DefaultDict[str, MFTRecord], - return_first_record: bool, - ) -> Iterator[Tuple]: - """ - Returns the parsed data from a MFT record - """ - # we only care about resident data - if attr.Attr_Header.NonResidentFlag: - return - - # we aren't looking ADS when we want the first data record - if return_first_record: - ads_name = renderers.NotApplicableValue() - - # skip records without a name if we want ADS entries - elif attr.Attr_Header.NameLength == 0: - return - - else: - # past the first $DATA record, attempt to get the ADS name - # NotAvailableValue = > 1st Data, but name was not parsable - ads_name_obj = attr.get_resident_filename() - ads_name = ( - str(ads_name_obj) - if ads_name_obj is not None - else renderers.NotAvailableValue() - ) - - content = attr.get_resident_filecontent() - if content: - content = renderers.LayerData.from_object(content) - else: - content = renderers.NotAvailableValue() - - yield ( - format_hints.Hex(record_map[mft_record.vol.offset].offset), - str(mft_record.get_signature()), - mft_record.RecordNumber, - attr.Attr_Header.AttrType.lookup(), - record_map[mft_record.vol.offset].record_name - or renderers.NotAvailableValue(), - ads_name, - content, - ) - - @classmethod - def parse_data_records( - cls, - record_map: DefaultDict[str, MFTRecord], - mft_record: mft.MFTEntry, - attr: mft.MFTAttribute, - symbol_table_name: str, - return_first_record: bool, - ) -> Iterator[Tuple]: - """ - Parses DATA records while maintaining the FILE_NAME association - from previous parsing of the record - Suports returning the first/main $DATA as well as however many - ADS records a file might have - """ - rec = record_map[mft_record.vol.offset] - - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - fn_object_typename = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" - attr_data = attr.Attr_Data.cast(fn_object_typename) - name_obj = attr_data.get_full_name() - rec.record_name = str(name_obj) if name_obj is not None else None - elif attr.Attr_Header.AttrType.lookup() == "DATA": - # first data - rec.offset = attr.Attr_Data.vol.offset - - display_data = False - - # first DATA attribute of this record - if rec.data_count == 0: - if return_first_record: - display_data = True - - rec.data_count = 1 - - # at the second DATA attribute of this record - elif rec.data_count == 1 and not return_first_record: - display_data = True - - if display_data: - yield from cls.parse_data_record( - mft_record, attr, record_map, return_first_record + yield 1, cls.MFTScanResult( + format_hints.Hex(filename_info.vol.offset), + str(mft_record.get_signature()), + mft_record.RecordNumber, + mft_record.LinkCount, + mft_flag, + permissions, + "FILE_NAME", + conversion.wintime_to_datetime(filename_info.CreationTime), + conversion.wintime_to_datetime(filename_info.ModifiedTime), + conversion.wintime_to_datetime(filename_info.UpdatedTime), + conversion.wintime_to_datetime(filename_info.AccessedTime), + filename_info.get_full_name(), ) + except exceptions.InvalidAddressException: + return + + @classmethod + def parse_mft_records( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + primary_layer_name: str, + ) -> Iterator[Tuple[int, MFTScanResult]]: + for mft_record in cls.enumerate_mft_records( + context=context, + config_path=config_path, + primary_layer_name=primary_layer_name, + ): + yield from cls.parse_standard_information_records(mft_record) + yield from cls.parse_filename_records(mft_record) def _generator(self): - yield from self.enumerate_mft_records( + for level, record in self.parse_mft_records( self.context, self.config_path, self.config["primary"], - self.parse_mft_records, - ) + ): + yield level, ( + record.offset, + record.record_type, + record.record_number, + record.link_count, + record.mft_type, + record.permissions, + record.attribute_type, + record.created, + record.modified, + record.updated, + record.accessed, + ( + str(record.filename) + if isinstance(record.filename, objects.String) + else record.filename + ), + ) def generate_timeline(self): for row in self._generator(): @@ -340,6 +268,15 @@ class ADS(interfaces.plugins.PluginInterface): _version = (1, 0, 2) + class ADSResult(NamedTuple): + offset: format_hints.Hex + signature: str + record_number: int + attribute_type: str + filename: Union[objects.String, interfaces.renderers.BaseAbsentValue] + stream_name: Union[objects.String, interfaces.renderers.BaseAbsentValue] + content: Union[renderers.LayerData, interfaces.renderers.BaseAbsentValue] + @classmethod def get_requirements(cls): return [ @@ -357,36 +294,57 @@ class ADS(interfaces.plugins.PluginInterface): ] @classmethod - def parse_ads_data_records( - cls, - record_map: DefaultDict[str, MFTRecord], - mft_record: interfaces.objects.ObjectInterface, - attr: interfaces.objects.ObjectInterface, - symbol_table_name: str, - ): - return MFTScan.parse_data_records( - record_map, mft_record, attr, symbol_table_name, False - ) + def parse_ads_data_records(cls, mft_record: mft.MFTEntry) -> Iterator[ADSResult]: + for data_attr in mft_record.alternate_data_streams(): + record_filename = ( + mft_record.longest_filename() or renderers.NotAvailableValue() + ) + content_obj = data_attr.get_resident_filecontent() + content = ( + renderers.LayerData.from_object(content_obj) + if content_obj + else renderers.NotAvailableValue() + ) + ads_filename = ( + data_attr.get_resident_filename() or renderers.NotAvailableValue() + ) + + yield cls.ADSResult( + format_hints.Hex(data_attr.Attr_Data.vol.offset), + mft_record.get_signature(), + mft_record.RecordNumber, + data_attr.Attr_Header.AttrType.lookup(), + record_filename, + ads_filename, + content, + ) def _generator(self): - for ( - offset, - rec_type, - rec_num, - attr_type, - file_name, - ads_name, - content, - ) in MFTScan.enumerate_mft_records( + for mft_entry in MFTScan.enumerate_mft_records( self.context, self.config_path, self.config["primary"], - self.parse_ads_data_records, ): - yield ( - 0, - (offset, rec_type, rec_num, attr_type, file_name, ads_name, content), - ) + for record in self.parse_ads_data_records(mft_entry): + # Convert to basic strings here __only__ because they'll use so + # much memory in the tree otherwise. + yield 0, ( + record.offset, + record.signature, + record.record_number, + record.attribute_type, + ( + str(record.filename) + if isinstance(record.filename, objects.String) + else record.filename + ), + ( + str(record.stream_name) + if isinstance(record.stream_name, objects.String) + else record.stream_name + ), + record.content, + ) def run(self): return renderers.TreeGrid( @@ -410,6 +368,14 @@ class ResidentData(interfaces.plugins.PluginInterface): _version = (1, 0, 2) + class ResidentDataResult(NamedTuple): + offset: format_hints.Hex + signature: str + record_number: int + attribute_type: str + filename: Union[objects.String, interfaces.renderers.BaseAbsentValue] + content: Union[renderers.LayerData, interfaces.renderers.BaseAbsentValue] + @classmethod def get_requirements(cls): return [ @@ -427,33 +393,46 @@ class ResidentData(interfaces.plugins.PluginInterface): ] @classmethod - def parse_first_data_records( + def parse_resident_data( cls, - record_map: DefaultDict[str, MFTRecord], mft_record: mft.MFTEntry, - attr: mft.MFTAttribute, - symbol_table_name: str, - ) -> Iterator[Tuple]: - return MFTScan.parse_data_records( - record_map, mft_record, attr, symbol_table_name, True + ) -> Optional[ResidentDataResult]: + """ + Returns the parsed data from a MFT record + """ + + try: + attr = next(mft_record.resident_data_attributes()) + except StopIteration: + return None + + content = attr.get_resident_filecontent() + if content: + content = renderers.LayerData.from_object(content) + else: + content = renderers.NotAvailableValue() + + # Choose the longest of the two, since it often includes a DOS 8.3 name + filename = mft_record.longest_filename() or renderers.NotAvailableValue() + + return cls.ResidentDataResult( + format_hints.Hex(attr.Attr_Data.vol.offset), + str(mft_record.get_signature()), + mft_record.RecordNumber, + attr.Attr_Header.AttrType.lookup(), + filename, + content, ) def _generator(self): - for ( - offset, - rec_type, - rec_num, - attr_type, - file_name, - _, - content, - ) in MFTScan.enumerate_mft_records( + for mft_record in MFTScan.enumerate_mft_records( self.context, self.config_path, self.config["primary"], - self.parse_first_data_records, ): - yield (0, (offset, rec_type, rec_num, attr_type, file_name, content)) + resident_data_entry = self.parse_resident_data(mft_record) + if resident_data_entry: + yield 0, resident_data_entry def run(self): return renderers.TreeGrid( diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index e004aa590..90ce6b6f2 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -22,7 +22,7 @@ class MFTEntry(objects.StructType): super().__init__(context, type_name, object_info, size, members) self._symbol_table_name = kwargs.get("symbol_table_name") - self._attr_generator = self._attributes() + self._attrs_loaded = False self._attrs: List[MFTAttribute] = [] @property @@ -37,27 +37,24 @@ class MFTEntry(objects.StructType): signature = self.Signature.cast("string", max_length=4, encoding="latin-1") return signature - def filename(self, symbol_table_name: str) -> Optional[objects.String]: - try: - fname_attr = next( - attr - for attr in self.attributes() - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME" - ) - except StopIteration: - return None - - fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" - attr_data = fname_attr.Attr_Data.cast(fn_object) - - return attr_data.get_full_name() - + @property def attributes(self) -> Iterator["MFTAttribute"]: + """ + Lazily evaluate and yield attributes, caching them in an internal list + for re-retrieval. + """ + if not self._attrs_loaded: + self._attrs = list(self._attributes()) + self._attrs_loaded = True + yield from self._attrs - for attr in self._attr_generator: - self._attrs.append(attr) - yield attr + def longest_filename(self) -> Optional[objects.String]: + names = [name.get_full_name() for name in self.filename_attributes()] + if not names: + return None + + return max(names, key=lambda x: len(str(x))) def _attributes(self) -> Iterator["MFTAttribute"]: @@ -75,21 +72,67 @@ class MFTEntry(objects.StructType): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr.Attr_Header.AttrType.is_valid_choice: + try: + while attr.Attr_Header.AttrType.is_valid_choice: + yield attr + + # If there's no advancement the loop will never end, so break it now + if attr.Attr_Header.Length == 0: + break + + # Update the base offset to point to the next attribute + attr_base_offset += attr.Attr_Header.Length + # Get the next attribute + attr: MFTAttribute = self._context.object( + attribute_object_type_name, + offset=self.vol.offset + attr_base_offset, + layer_name=self.vol.layer_name, + ) + except exceptions.InvalidAddressException: + return + + def standard_information_attributes(self) -> Iterator[objects.StructType]: + for attr in self.attributes: + if attr.Attr_Header.AttrType.lookup() != "STANDARD_INFORMATION": + continue + + si_object = ( + self.symbol_table_name + constants.BANG + "STANDARD_INFORMATION_ENTRY" + ) + + yield attr.Attr_Data.cast(si_object) + + def filename_attributes(self) -> Iterator["MFTFileName"]: + for attr in self.attributes: + try: + if attr.Attr_Header.AttrType.lookup() != "FILE_NAME": + continue + + fn_object = self.symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" + attr_data = attr.Attr_Data.cast(fn_object) + except exceptions.InvalidAddressException: + continue + yield attr_data + + def _data_attributes(self): + for attr in self.attributes: + if not ( + attr.Attr_Header.AttrType.lookup() == "DATA" + and attr.Attr_Header.NonResidentFlag == 0 + ): + continue + yield attr - # If there's no advancement the loop will never end, so break it now - if attr.Attr_Header.Length == 0: - break + def resident_data_attributes(self) -> Iterator["MFTAttribute"]: + for attr in self._data_attributes(): + if attr.Attr_Header.NameLength == 0: + yield attr - # Update the base offset to point to the next attribute - attr_base_offset += attr.Attr_Header.Length - # Get the next attribute - attr: MFTAttribute = self._context.object( - attribute_object_type_name, - offset=self.vol.offset + attr_base_offset, - layer_name=self.vol.layer_name, - ) + def alternate_data_streams(self) -> Iterator["MFTAttribute"]: + for attr in self._data_attributes(): + if attr.Attr_Header.NameLength != 0: + yield attr class MFTFileName(objects.StructType): From 5fda7409eb4aabaeff611ea7a6b31eaff6cfd5a0 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 17:09:02 -0500 Subject: [PATCH 08/19] Log `InvalidAddressException` instances --- .../symbols/windows/extensions/mft.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 90ce6b6f2..4c5be81ee 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,10 +2,13 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import logging from typing import Dict, Iterator, List, Optional, Tuple from volatility3.framework import constants, exceptions, interfaces, objects +vollog = logging.getLogger(__name__) + class MFTEntry(objects.StructType): """This represents the base MFT Record""" @@ -88,7 +91,10 @@ class MFTEntry(objects.StructType): offset=self.vol.offset + attr_base_offset, layer_name=self.vol.layer_name, ) - except exceptions.InvalidAddressException: + except exceptions.InvalidAddressException as e: + vollog.debug( + f"Failed to read attribute at {attr.vol.offset:#x}: {e.__class__.__name__}" + ) return def standard_information_attributes(self) -> Iterator[objects.StructType]: @@ -110,7 +116,10 @@ class MFTEntry(objects.StructType): fn_object = self.symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" attr_data = attr.Attr_Data.cast(fn_object) - except exceptions.InvalidAddressException: + except exceptions.InvalidAddressException as e: + vollog.debug( + f"Failed to read attr at {attr.vol.offset:#x}: {e.__class__.__name__}" + ) continue yield attr_data @@ -168,7 +177,10 @@ class MFTAttribute(objects.StructType): encoding="utf16", ) return name - except exceptions.InvalidAddressException: + except exceptions.InvalidAddressException as e: + vollog.debug( + f"Failed to get resident file content due to {e.__class__.__name__}" + ) return None def get_resident_filecontent(self) -> Optional[objects.Bytes]: @@ -190,5 +202,8 @@ class MFTAttribute(objects.StructType): length=self.Attr_Header.ContentLength, ) return bytesobj - except exceptions.InvalidAddressException: + except exceptions.InvalidAddressException as e: + vollog.debug( + f"Failed to get resident file content due to {e.__class__.__name__}" + ) return None From 9f85e1465e2378f8dbbe5084ee436f5f978f9906 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 17:19:17 -0500 Subject: [PATCH 09/19] Major version bumps for all three plugins --- volatility3/framework/plugins/windows/mftscan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index a4cdad582..5c8b419c3 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -20,7 +20,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (3, 0, 0) class MFTScanResult(NamedTuple): offset: format_hints.Hex @@ -266,7 +266,7 @@ class ADS(interfaces.plugins.PluginInterface): _required_framework_version = (2, 22, 0) - _version = (1, 0, 2) + _version = (2, 0, 0) class ADSResult(NamedTuple): offset: format_hints.Hex @@ -366,7 +366,7 @@ class ResidentData(interfaces.plugins.PluginInterface): _required_framework_version = (2, 22, 0) - _version = (1, 0, 2) + _version = (2, 0, 0) class ResidentDataResult(NamedTuple): offset: format_hints.Hex From 8b308133f532e389d2bf2d967f29e3f2f42adcd2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 17:32:25 -0500 Subject: [PATCH 10/19] Bump required version numbers --- volatility3/framework/plugins/windows/mftscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 5c8b419c3..eacd43b8a 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -281,7 +281,7 @@ class ADS(interfaces.plugins.PluginInterface): def get_requirements(cls): return [ requirements.VersionRequirement( - name="MFTScan", component=MFTScan, version=(2, 0, 0) + name="MFTScan", component=MFTScan, version=(3, 0, 0) ), requirements.TranslationLayerRequirement( name="primary", @@ -380,7 +380,7 @@ class ResidentData(interfaces.plugins.PluginInterface): def get_requirements(cls): return [ requirements.VersionRequirement( - name="MFTScan", component=MFTScan, version=(2, 0, 0) + name="MFTScan", component=MFTScan, version=(3, 0, 0) ), requirements.TranslationLayerRequirement( name="primary", From 1d20e6575908e118ad71746ff9d64b6cd5d23d9f Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 17:39:14 -0500 Subject: [PATCH 11/19] Add versioning to MFT extension classes --- .../framework/plugins/windows/mftscan.py | 15 ++++++++++++ .../symbols/windows/extensions/mft.py | 24 ++++++++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index eacd43b8a..7df895f15 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -49,6 +49,21 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): component=timeliner.TimeLinerInterface, version=(1, 0, 0), ), + requirements.VersionRequirement( + name="mft_entry", + component=mft.MFTEntry, + version=(1, 0, 0), + ), + requirements.VersionRequirement( + name="mft_filename", + component=mft.MFTFileName, + version=(1, 0, 0), + ), + requirements.VersionRequirement( + name="mft_attribute", + component=mft.MFTAttribute, + version=(1, 0, 0), + ), requirements.VersionRequirement( name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) ), diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 4c5be81ee..32261c4ff 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -5,14 +5,20 @@ import logging from typing import Dict, Iterator, List, Optional, Tuple +from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects vollog = logging.getLogger(__name__) -class MFTEntry(objects.StructType): +class MFTEntry(objects.StructType, interfaces.configuration.VersionableInterface): """This represents the base MFT Record""" + _version = (1, 0, 0) + _required_framework_version = (2, 26, 0) + + framework.require_interface_version(*_required_framework_version) + def __init__( self, context: interfaces.context.ContextInterface, @@ -144,9 +150,15 @@ class MFTEntry(objects.StructType): yield attr -class MFTFileName(objects.StructType): +class MFTFileName(objects.StructType, interfaces.configuration.VersionableInterface): """This represents an MFT $FILE_NAME Attribute""" + _version = (1, 0, 0) + + _required_framework_version = (2, 26, 0) + + framework.require_interface_version(*_required_framework_version) + def get_full_name(self) -> objects.String: output = self.Name.cast( "string", encoding="utf16", max_length=self.NameLength * 2, errors="replace" @@ -154,9 +166,15 @@ class MFTFileName(objects.StructType): return output -class MFTAttribute(objects.StructType): +class MFTAttribute(objects.StructType, interfaces.configuration.VersionableInterface): """This represents an MFT ATTRIBUTE""" + _version = (1, 0, 0) + + _required_framework_version = (2, 26, 0) + + framework.require_interface_version(*_required_framework_version) + def get_resident_filename(self) -> Optional[objects.String]: # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous From ebbbe913dc92d609b8315a9d1761ee8eab8791d5 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 17:55:28 -0500 Subject: [PATCH 12/19] Convert remaining values to Python primitives --- .../framework/plugins/windows/mftscan.py | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 7df895f15..cb4b880d5 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -224,11 +224,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self.config_path, self.config["primary"], ): + # Convert all `objects.PrimitiveObject` to their simpler Python + # types. This is normally not something we would do, since it's + # lossy and prevents users from getting back to the data source, + # but in this case memory usage is so extreme due to the number of + # records that it becomes necessary. The rich types are still + # exposed through classmethods. yield level, ( record.offset, record.record_type, - record.record_number, - record.link_count, + int(record.record_number), + int(record.link_count), record.mft_type, record.permissions, record.attribute_type, @@ -341,12 +347,16 @@ class ADS(interfaces.plugins.PluginInterface): self.config["primary"], ): for record in self.parse_ads_data_records(mft_entry): - # Convert to basic strings here __only__ because they'll use so - # much memory in the tree otherwise. + # Convert all `objects.PrimitiveObject` to their simpler Python + # types. This is normally not something we would do, since it's + # lossy and prevents users from getting back to the data source, + # but in this case memory usage is so extreme due to the number of + # records that it becomes necessary. The rich types are still + # exposed through classmethods. yield 0, ( record.offset, - record.signature, - record.record_number, + str(record.signature), + int(record.record_number), record.attribute_type, ( str(record.filename) @@ -447,7 +457,20 @@ class ResidentData(interfaces.plugins.PluginInterface): ): resident_data_entry = self.parse_resident_data(mft_record) if resident_data_entry: - yield 0, resident_data_entry + # Convert all `objects.PrimitiveObject` to their simpler Python + # types. This is normally not something we would do, since it's + # lossy and prevents users from getting back to the data source, + # but in this case memory usage is so extreme due to the number of + # records that it becomes necessary. The rich types are still + # exposed through classmethods. + yield 0, ( + resident_data_entry.offset, + resident_data_entry.signature, + int(resident_data_entry.record_number), + resident_data_entry.attribute_type, + str(resident_data_entry.filename), + resident_data_entry.content, + ) def run(self): return renderers.TreeGrid( From 41de562e550424a43fecbebf5f6560be3ec66571 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 17:56:16 -0500 Subject: [PATCH 13/19] Revert "Add versioning to MFT extension classes" This reverts commit 1d20e6575908e118ad71746ff9d64b6cd5d23d9f. --- .../framework/plugins/windows/mftscan.py | 15 ------------ .../symbols/windows/extensions/mft.py | 24 +++---------------- 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index cb4b880d5..f50801c70 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -49,21 +49,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): component=timeliner.TimeLinerInterface, version=(1, 0, 0), ), - requirements.VersionRequirement( - name="mft_entry", - component=mft.MFTEntry, - version=(1, 0, 0), - ), - requirements.VersionRequirement( - name="mft_filename", - component=mft.MFTFileName, - version=(1, 0, 0), - ), - requirements.VersionRequirement( - name="mft_attribute", - component=mft.MFTAttribute, - version=(1, 0, 0), - ), requirements.VersionRequirement( name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) ), diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 32261c4ff..4c5be81ee 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -5,20 +5,14 @@ import logging from typing import Dict, Iterator, List, Optional, Tuple -from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects vollog = logging.getLogger(__name__) -class MFTEntry(objects.StructType, interfaces.configuration.VersionableInterface): +class MFTEntry(objects.StructType): """This represents the base MFT Record""" - _version = (1, 0, 0) - _required_framework_version = (2, 26, 0) - - framework.require_interface_version(*_required_framework_version) - def __init__( self, context: interfaces.context.ContextInterface, @@ -150,15 +144,9 @@ class MFTEntry(objects.StructType, interfaces.configuration.VersionableInterface yield attr -class MFTFileName(objects.StructType, interfaces.configuration.VersionableInterface): +class MFTFileName(objects.StructType): """This represents an MFT $FILE_NAME Attribute""" - _version = (1, 0, 0) - - _required_framework_version = (2, 26, 0) - - framework.require_interface_version(*_required_framework_version) - def get_full_name(self) -> objects.String: output = self.Name.cast( "string", encoding="utf16", max_length=self.NameLength * 2, errors="replace" @@ -166,15 +154,9 @@ class MFTFileName(objects.StructType, interfaces.configuration.VersionableInterf return output -class MFTAttribute(objects.StructType, interfaces.configuration.VersionableInterface): +class MFTAttribute(objects.StructType): """This represents an MFT ATTRIBUTE""" - _version = (1, 0, 0) - - _required_framework_version = (2, 26, 0) - - framework.require_interface_version(*_required_framework_version) - def get_resident_filename(self) -> Optional[objects.String]: # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous From de117042fe748e855ce48378d4cbf83ecd1f922b Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 18:04:43 -0500 Subject: [PATCH 14/19] Remove symbol_table_name from object constructor Get from `self.vol.type_name` instead --- volatility3/framework/plugins/windows/mftscan.py | 1 - volatility3/framework/symbols/windows/extensions/mft.py | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index f50801c70..6aa0142c0 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -112,7 +112,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_object_type_name, offset=offset, layer_name=layer.name, - symbol_table_name=symbol_table_name, ) yield mft_record diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 4c5be81ee..8f994752a 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -20,21 +20,15 @@ class MFTEntry(objects.StructType): object_info: interfaces.objects.ObjectInformation, size: int, members: Dict[str, Tuple[int, interfaces.objects.Template]], - **kwargs, ) -> None: super().__init__(context, type_name, object_info, size, members) - self._symbol_table_name = kwargs.get("symbol_table_name") self._attrs_loaded = False self._attrs: List[MFTAttribute] = [] @property def symbol_table_name(self) -> str: - if self._symbol_table_name is None: - raise ValueError( - "MFTEntry was instantiated without an MFT symbol table name" - ) - return self._symbol_table_name + return self.vol.type_name.split(constants.BANG)[0] def get_signature(self) -> objects.String: signature = self.Signature.cast("string", max_length=4, encoding="latin-1") From a5dfc6acb3264e4c13b9d20f1fe66917eb3e848b Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Apr 2025 18:06:23 -0500 Subject: [PATCH 15/19] Bump required framework version on all three MFTScan plugins --- volatility3/framework/plugins/windows/mftscan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 6aa0142c0..aab32593a 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -18,7 +18,7 @@ vollog = logging.getLogger(__name__) class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 26, 0) _version = (3, 0, 0) @@ -269,7 +269,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): class ADS(interfaces.plugins.PluginInterface): """Scans for Alternate Data Stream""" - _required_framework_version = (2, 22, 0) + _required_framework_version = (2, 26, 0) _version = (2, 0, 0) @@ -373,7 +373,7 @@ class ADS(interfaces.plugins.PluginInterface): class ResidentData(interfaces.plugins.PluginInterface): """Scans for MFT Records with Resident Data""" - _required_framework_version = (2, 22, 0) + _required_framework_version = (2, 26, 0) _version = (2, 0, 0) From 8428e81f0318cc00ede336075ba7a8c7bb7416cb Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 4 Apr 2025 10:30:46 -0500 Subject: [PATCH 16/19] Fix up remaining type-hints Updates type hints on some fields of the result namedtuples to be their `objects.Primitive` types instead of Python primitives, and does any conversion to Python primitives in the generator methods. --- volatility3/framework/plugins/windows/mftscan.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index aab32593a..3f59f1aa6 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -25,8 +25,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): class MFTScanResult(NamedTuple): offset: format_hints.Hex record_type: str - record_number: int - link_count: int + record_number: objects.Integer + link_count: objects.Integer mft_type: str permissions: Union[str, interfaces.renderers.BaseAbsentValue] attribute_type: str @@ -275,8 +275,8 @@ class ADS(interfaces.plugins.PluginInterface): class ADSResult(NamedTuple): offset: format_hints.Hex - signature: str - record_number: int + signature: objects.String + record_number: objects.Integer attribute_type: str filename: Union[objects.String, interfaces.renderers.BaseAbsentValue] stream_name: Union[objects.String, interfaces.renderers.BaseAbsentValue] @@ -379,7 +379,7 @@ class ResidentData(interfaces.plugins.PluginInterface): class ResidentDataResult(NamedTuple): offset: format_hints.Hex - signature: str + signature: objects.String record_number: int attribute_type: str filename: Union[objects.String, interfaces.renderers.BaseAbsentValue] @@ -426,7 +426,7 @@ class ResidentData(interfaces.plugins.PluginInterface): return cls.ResidentDataResult( format_hints.Hex(attr.Attr_Data.vol.offset), - str(mft_record.get_signature()), + mft_record.get_signature(), mft_record.RecordNumber, attr.Attr_Header.AttrType.lookup(), filename, @@ -449,7 +449,7 @@ class ResidentData(interfaces.plugins.PluginInterface): # exposed through classmethods. yield 0, ( resident_data_entry.offset, - resident_data_entry.signature, + str(resident_data_entry.signature), int(resident_data_entry.record_number), resident_data_entry.attribute_type, str(resident_data_entry.filename), From b15e9104e8b3e581e904b1b58e2031f233044b21 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 4 Apr 2025 10:32:13 -0500 Subject: [PATCH 17/19] Rename methods, add docstrings Improves the naming of a couple of the new extension class methods to more accurately reflect the return type, and adds docstrings to extensions class methods. --- .../framework/plugins/windows/mftscan.py | 4 +- .../symbols/windows/extensions/mft.py | 41 ++++++++++++++++--- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 3f59f1aa6..e7390d699 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -131,7 +131,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): try: # There should only be one STANDARD_INFORMATION attribute, but we # do this just in case. - for std_information in mft_record.standard_information_attributes(): + for std_information in mft_record.standard_information_entries(): yield 0, cls.MFTScanResult( format_hints.Hex(std_information.vol.offset), str(mft_record.get_signature()), @@ -162,7 +162,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # File Name Attribute try: - for filename_info in mft_record.filename_attributes(): + for filename_info in mft_record.filename_entries(): # If we don't have a valid enum, coerce to hex so we can keep the record try: diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 8f994752a..bf20c1ffc 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -47,7 +47,7 @@ class MFTEntry(objects.StructType): yield from self._attrs def longest_filename(self) -> Optional[objects.String]: - names = [name.get_full_name() for name in self.filename_attributes()] + names = [name.get_full_name() for name in self.filename_entries()] if not names: return None @@ -91,9 +91,17 @@ class MFTEntry(objects.StructType): ) return - def standard_information_attributes(self) -> Iterator[objects.StructType]: + def standard_information_entries( + self, + ) -> Iterator[objects.StructType]: + """ + Yields a STANDARD_INFORMATION struct for each of the + STANDARD_INFORMATION attributes in this MFT record (although there + should only be one per record). + """ for attr in self.attributes: - if attr.Attr_Header.AttrType.lookup() != "STANDARD_INFORMATION": + attr_type = attr.Attr_Header.AttrType.lookup() + if attr_type != "STANDARD_INFORMATION": continue si_object = ( @@ -102,10 +110,16 @@ class MFTEntry(objects.StructType): yield attr.Attr_Data.cast(si_object) - def filename_attributes(self) -> Iterator["MFTFileName"]: + def filename_entries(self) -> Iterator["MFTFileName"]: + """ + Yields an MFT Filename for each of the FILE_NAME attributes contained + in this MFT record. There are often two - one for the long filename, + and the other with the DOS 8.3 short name. + """ for attr in self.attributes: try: - if attr.Attr_Header.AttrType.lookup() != "FILE_NAME": + attr_type = attr.Attr_Header.AttrType.lookup() + if attr_type != "FILE_NAME": continue fn_object = self.symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" @@ -128,11 +142,18 @@ class MFTEntry(objects.StructType): yield attr def resident_data_attributes(self) -> Iterator["MFTAttribute"]: + """ + Yields all MFT attributes that contain resident data for the primary + stream. + """ for attr in self._data_attributes(): if attr.Attr_Header.NameLength == 0: yield attr def alternate_data_streams(self) -> Iterator["MFTAttribute"]: + """ + Yields all MFT attributes that contain alternate data streams (ADS). + """ for attr in self._data_attributes(): if attr.Attr_Header.NameLength != 0: yield attr @@ -142,6 +163,9 @@ class MFTFileName(objects.StructType): """This represents an MFT $FILE_NAME Attribute""" def get_full_name(self) -> objects.String: + """ + Returns the UTF-16 decoded filename. + """ output = self.Name.cast( "string", encoding="utf16", max_length=self.NameLength * 2, errors="replace" ) @@ -152,6 +176,9 @@ class MFTAttribute(objects.StructType): """This represents an MFT ATTRIBUTE""" def get_resident_filename(self) -> Optional[objects.String]: + """ + Returns the resident filename (typically for an Alternate Data Stream (ADS)). + """ # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous if ( @@ -178,6 +205,10 @@ class MFTAttribute(objects.StructType): return None def get_resident_filecontent(self) -> Optional[objects.Bytes]: + """ + Returns the file content that is resident within this MFT attribute, + for either the primary or an alternate data stream. + """ # smear observed in mass testing of samples # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems if ( From a0ca33b284e41471b56dc6c0dc4e15f39f30383e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 4 Apr 2025 11:29:48 -0500 Subject: [PATCH 18/19] Also yield STANDARD_INFORMATION timestamps in timeliner --- volatility3/framework/plugins/windows/mftscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index e7390d699..bce832d5e 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -238,9 +238,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _depth, row_data = row # Only Output FN Records - if row_data[6] == "FILE_NAME": + if row_data[6] in ("FILE_NAME", "STANDARD_INFORMATION"): filename = row_data[-1] - description = f"MFT FILE_NAME entry for {filename}" + description = f"MFT {row_data[6]} entry for {filename}" yield (description, timeliner.TimeLinerType.CREATED, row_data[7]) yield (description, timeliner.TimeLinerType.MODIFIED, row_data[8]) yield (description, timeliner.TimeLinerType.CHANGED, row_data[9]) From 39b3e76efc521bbdd314d8ed6db3d6cfd0fca6d5 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 4 Apr 2025 17:46:46 -0500 Subject: [PATCH 19/19] Add a filename to STANDARD_INFORMATION timeline entries --- .../framework/plugins/windows/mftscan.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index bce832d5e..f0be3cf7f 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -234,17 +234,24 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) def generate_timeline(self): - for row in self._generator(): - _depth, row_data = row + for record in self.enumerate_mft_records( + self.context, self.config_path, self.config["primary"] + ): + fname = record.longest_filename() - # Only Output FN Records - if row_data[6] in ("FILE_NAME", "STANDARD_INFORMATION"): - filename = row_data[-1] - description = f"MFT {row_data[6]} entry for {filename}" - yield (description, timeliner.TimeLinerType.CREATED, row_data[7]) - yield (description, timeliner.TimeLinerType.MODIFIED, row_data[8]) - yield (description, timeliner.TimeLinerType.CHANGED, row_data[9]) - yield (description, timeliner.TimeLinerType.ACCESSED, row_data[10]) + for _, item in self.parse_standard_information_records(record): + description = f"MFT {item.attribute_type} entry for {fname}" + yield (description, timeliner.TimeLinerType.CREATED, item.created) + yield (description, timeliner.TimeLinerType.MODIFIED, item.modified) + yield (description, timeliner.TimeLinerType.CHANGED, item.updated) + yield (description, timeliner.TimeLinerType.ACCESSED, item.accessed) + + for _, item in self.parse_filename_records(record): + description = f"MFT {item.attribute_type} entry for {item.filename}" + yield (description, timeliner.TimeLinerType.CREATED, item.created) + yield (description, timeliner.TimeLinerType.MODIFIED, item.modified) + yield (description, timeliner.TimeLinerType.CHANGED, item.updated) + yield (description, timeliner.TimeLinerType.ACCESSED, item.accessed) def run(self): return renderers.TreeGrid(