From a46c9d9d8ecf0a36352c672c2193227c59cd33c1 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 29 Sep 2023 10:16:20 +0100 Subject: [PATCH 01/11] Linux: add padded read when getting magic for elf extension to help with smear and missing pages --- .../framework/symbols/linux/extensions/elf.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 416a7e4d2..a05885a7b 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -33,14 +33,18 @@ class elf(objects.StructType): layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() # We read the MAGIC: (0x0 to 0x4) 0x7f 0x45 0x4c 0x46 - magic = self._context.object( - symbol_table_name + constants.BANG + "unsigned long", - layer_name=layer_name, - offset=object_info.offset, - ) + magic = self._context.layers[layer_name].read(object_info.offset, 4, True) # Check validity - if magic != 0x464C457F: + if ( + magic[0] == 0x7F + and magic[1] == 0x45 # E + and magic[2] == 0x4C # L + and magic[3] == 0x46 # F + ): + self._valid_magic = True + else: + self._valid_magic = False return None # We need to read the EI_CLASS (0x4 offset) @@ -72,7 +76,10 @@ class elf(objects.StructType): """ Determine whether it is a valid object """ - return self._type_prefix is not None and self._hdr is not None + if self._valid_magic: + return self._type_prefix is not None and self._hdr is not None + else: + return False def __getattr__(self, name): # Just redirect to the corresponding header From 41a02fbf5b5bd860f35d53c7ed34a97bfa68f3cd Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 3 Oct 2023 07:13:28 +0100 Subject: [PATCH 02/11] Linux: use try/except in linux elf extension to catch paged and invalid addresses rather than crashing --- .../framework/symbols/linux/extensions/elf.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index a05885a7b..8b42b3075 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -3,9 +3,12 @@ # from typing import Dict, Tuple +import logging from volatility3.framework import constants -from volatility3.framework import objects, interfaces +from volatility3.framework import objects, interfaces, exceptions + +vollog = logging.getLogger(__name__) class elf(objects.StructType): @@ -33,20 +36,29 @@ class elf(objects.StructType): layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() # We read the MAGIC: (0x0 to 0x4) 0x7f 0x45 0x4c 0x46 - magic = self._context.layers[layer_name].read(object_info.offset, 4, True) - - # Check validity - if ( - magic[0] == 0x7F - and magic[1] == 0x45 # E - and magic[2] == 0x4C # L - and magic[3] == 0x46 # F - ): - self._valid_magic = True - else: + try: + magic = self._context.object( + symbol_table_name + constants.BANG + "unsigned long", + layer_name=layer_name, + offset=object_info.offset, + ) + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ) as excp: + vollog.debug( + f"Unable to check magic bytes for ELF file at offset {hex(object_info.offset)} in layer {layer_name}: {excp}" + ) self._valid_magic = False return None + # Check validity + if magic != 0x464C457F: # e.g. ELF + self._valid_magic = False + return None + else: + self._valid_magic = True + # We need to read the EI_CLASS (0x4 offset) ei_class = self._context.object( symbol_table_name + constants.BANG + "unsigned char", From 5ab0e4f83f7feb5444ce8d691e58bf56e495db1a Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 14 Nov 2023 07:22:00 +0000 Subject: [PATCH 03/11] Linux: remove _valid_magic from linux elf extension, check for _type_prefix and _hdr attrs instead --- volatility3/framework/symbols/linux/extensions/elf.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 8b42b3075..629a05da5 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -49,15 +49,11 @@ class elf(objects.StructType): vollog.debug( f"Unable to check magic bytes for ELF file at offset {hex(object_info.offset)} in layer {layer_name}: {excp}" ) - self._valid_magic = False return None # Check validity if magic != 0x464C457F: # e.g. ELF - self._valid_magic = False return None - else: - self._valid_magic = True # We need to read the EI_CLASS (0x4 offset) ei_class = self._context.object( @@ -88,7 +84,7 @@ class elf(objects.StructType): """ Determine whether it is a valid object """ - if self._valid_magic: + if hasattr(self, "_type_prefix") and hasattr(self, "_hdr"): return self._type_prefix is not None and self._hdr is not None else: return False From 6c6d036cc0b2606406af418fbb7a15d6db325cfc Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 19 Nov 2023 00:54:56 +0100 Subject: [PATCH 04/11] Adding Alternate Data Stream Scanner --- .../framework/plugins/windows/mftscan.py | 153 +++++++++++++++++- .../framework/symbols/windows/mft.json | 16 +- 2 files changed, 167 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 87416d274..f5c3ebc95 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,7 +5,7 @@ import contextlib import datetime import logging -from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed @@ -31,6 +31,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) ), + ] def _generator(self): @@ -189,3 +190,153 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ], self._generator(), ) + + +class ADS(interfaces.plugins.PluginInterface): + + """Scans for Alternate Data Stream""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + ] + + def _generator(self): + layer = self.context.layers[self.config["primary"]] + + # Yara Rule to scan for MFT Header Signatures + rules = yarascan.YaraScan.process_yara_options( + {"yara_rules": "/FILE0|FILE\*|BAAD/"} + ) + + # Read in the Symbol File + symbol_table = intermed.IntermediateSymbolTable.create( + context=self.context, + config_path=self.config_path, + sub_path="windows", + filename="mft", + class_types={"MFT_ENTRY": mft.MFTEntry,"FILE_NAME_ENTRY": mft.MFTFileName}, + ) + + # get each of the individual Field Sets + mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + header_object = symbol_table + constants.BANG + "ATTR_HEADER" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Scan the layer for Raw MFT records and parse the fields + for offset, _rule_name, _name, _value in layer.scan( + context=self.context, scanner=yarascan.YaraScanner(rules=rules) + ): + with contextlib.suppress(exceptions.PagedInvalidAddressException): + mft_record = self.context.object( + mft_object, offset=offset, layer_name=layer.name + ) + # We will update this on each pass in the next loop and use it as the new offset. + attr_base_offset = mft_record.FirstAttrOffset + + attr_header = self.context.object( + header_object, + offset=offset + attr_base_offset, + layer_name=layer.name, + ) + + # There is no field that has a count of Attributes + # Keep Attempting to read attributes until we get an invalid attr_header.AttrType + file_name = "" + while attr_header.AttrType.is_valid_choice: + + # Offset past the headers to the attribute data + attr_data_offset = ( + offset + + attr_base_offset + + self.context.symbol_space.get_type( + attribute_object + ).relative_child_offset("Attr_Data") + ) + + if attr_header.AttrType.lookup() == "FILE_NAME": + attr_data = self.context.object( + fn_object, offset=attr_data_offset, layer_name=layer.name + ) + file_name = attr_data.get_full_name() + + + # DATA Attribute (can be ADS or not) + if attr_header.AttrType.lookup() == "DATA": + if not attr_header.NonResidentFlag: + # It is a resident file + if attr_header.NameLength > 0: + attr_name_offset = ( + offset + + attr_base_offset + + attr_header.NameOffset + ) + ads_name = self._context.layers[layer.name].read( + attr_name_offset, attr_header.NameLength*2 , pad=True + ).decode('utf-16') + attr_content_offset = ( + offset + + attr_base_offset + + attr_header.ContentOffset + ) + content = self._context.layers[layer.name].read( + attr_content_offset, attr_header.ContentLength , pad=True + ) + + + # Preparing for Disassembly + architecture = layer.metadata.get("architecture", None) + disasm = interfaces.renderers.Disassembly( + content, 0, architecture.lower() + ) + + yield 0, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + attr_header.AttrType.lookup(), + file_name, + ads_name, + format_hints.HexBytes(content), + disasm, + ) + + # If there's no advancement the loop will never end, so break it now + if attr_header.Length == 0: + break + + # Update the base offset to point to the next attribute + attr_base_offset += attr_header.Length + # Get the next attribute + attr_header = self.context.object( + header_object, + offset=offset + attr_base_offset, + layer_name=layer.name, + ) + + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Record Type", str), + ("Record Number", int), + ("MFT Type", str), + ("Filename", str), + ("ADS Filename", str), + ("Hexdump", format_hints.HexBytes), + ("Disasm", interfaces.renderers.Disassembly), + ], + self._generator(), + ) \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index e5de8f3fa..616e8990d 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -300,10 +300,24 @@ "kind": "base", "name": "unsigned short" } + }, + "ContentLength": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ContentOffset": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned short" + } } }, "kind": "struct", - "size": 16 + "size": 22 },"RESIDENT_HEADER": { "fields": { "AttrSize": { From 46a6ab1721e241939af0b4c29bafdf2fcb09c405 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 19 Nov 2023 13:09:47 +0100 Subject: [PATCH 05/11] Making sure it is ADS --- .../framework/plugins/windows/mftscan.py | 77 ++++++++++--------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index f5c3ebc95..b4af2a867 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -254,8 +254,8 @@ class ADS(interfaces.plugins.PluginInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType file_name = "" + is_ads = 0 while attr_header.AttrType.is_valid_choice: - # Offset past the headers to the attribute data attr_data_offset = ( offset @@ -274,43 +274,48 @@ class ADS(interfaces.plugins.PluginInterface): # DATA Attribute (can be ADS or not) if attr_header.AttrType.lookup() == "DATA": - if not attr_header.NonResidentFlag: - # It is a resident file - if attr_header.NameLength > 0: - attr_name_offset = ( - offset - + attr_base_offset - + attr_header.NameOffset - ) - ads_name = self._context.layers[layer.name].read( - attr_name_offset, attr_header.NameLength*2 , pad=True - ).decode('utf-16') - attr_content_offset = ( - offset - + attr_base_offset - + attr_header.ContentOffset - ) - content = self._context.layers[layer.name].read( - attr_content_offset, attr_header.ContentLength , pad=True - ) + if is_ads > 0: + if not attr_header.NonResidentFlag: + # Resident files are the most interesting. + if attr_header.NameLength > 0: + attr_name_offset = ( + offset + + attr_base_offset + + attr_header.NameOffset + ) + ads_name = self._context.layers[layer.name].read( + attr_name_offset, attr_header.NameLength*2 , pad=True + ).decode('utf-16') + attr_content_offset = ( + offset + + attr_base_offset + + attr_header.ContentOffset + ) + content = self._context.layers[layer.name].read( + attr_content_offset, attr_header.ContentLength , pad=True + ) - - # Preparing for Disassembly - architecture = layer.metadata.get("architecture", None) - disasm = interfaces.renderers.Disassembly( - content, 0, architecture.lower() - ) + + # Preparing for Disassembly + architecture = layer.metadata.get("architecture", None) + disasm = interfaces.renderers.Disassembly( + content, 0, architecture.lower() + ) - yield 0, ( - format_hints.Hex(attr_data_offset), - mft_record.get_signature(), - mft_record.RecordNumber, - attr_header.AttrType.lookup(), - file_name, - ads_name, - format_hints.HexBytes(content), - disasm, - ) + yield 0, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + attr_header.AttrType.lookup(), + file_name, + ads_name, + format_hints.HexBytes(content), + disasm, + ) + else: + # The First Data Attr is the file itself not the ADS + is_ads+= 1 + # If there's no advancement the loop will never end, so break it now if attr_header.Length == 0: From c3dbf9714d9158b01385f78fb7434ecee64ac5b8 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 20 Nov 2023 19:12:07 +0100 Subject: [PATCH 06/11] Better variable init --- volatility3/framework/plugins/windows/mftscan.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index b4af2a867..c40f7ef73 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -59,7 +59,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields - for offset, _rule_name, _name, _value in layer.scan( + for offset, _, _, _ in layer.scan( context=self.context, scanner=yarascan.YaraScanner(rules=rules) ): with contextlib.suppress(exceptions.PagedInvalidAddressException): @@ -253,8 +253,9 @@ class ADS(interfaces.plugins.PluginInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - file_name = "" - is_ads = 0 + file_name = "N/A" + is_ads = False + # The First $DATA Attr is the 'principal' file itself not the ADS while attr_header.AttrType.is_valid_choice: # Offset past the headers to the attribute data attr_data_offset = ( @@ -274,7 +275,7 @@ class ADS(interfaces.plugins.PluginInterface): # DATA Attribute (can be ADS or not) if attr_header.AttrType.lookup() == "DATA": - if is_ads > 0: + if is_ads: if not attr_header.NonResidentFlag: # Resident files are the most interesting. if attr_header.NameLength > 0: @@ -313,8 +314,7 @@ class ADS(interfaces.plugins.PluginInterface): disasm, ) else: - # The First Data Attr is the file itself not the ADS - is_ads+= 1 + is_ads = True # If there's no advancement the loop will never end, so break it now From 24609856de629ccb7adbab46977de3c8492846e8 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 25 Nov 2023 17:04:47 +0100 Subject: [PATCH 07/11] Fixing: unused import, typo, ISF enhancement --- .../framework/plugins/windows/mftscan.py | 24 +++++++++---------- .../framework/symbols/windows/mft.json | 8 +++---- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index c40f7ef73..397ce75bc 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,7 +5,7 @@ import contextlib import datetime import logging -from volatility3.framework import constants, exceptions, interfaces, renderers, symbols +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed @@ -53,13 +53,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields - for offset, _, _, _ in layer.scan( + for offset, _rule_name, _name, _value in layer.scan( context=self.context, scanner=yarascan.YaraScanner(rules=rules) ): with contextlib.suppress(exceptions.PagedInvalidAddressException): @@ -86,8 +85,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset + attr_base_offset + self.context.symbol_space.get_type( - attribute_object - ).relative_child_offset("Attr_Data") + header_object + ).size ) # MFT Flags determine the file type or dir @@ -231,7 +230,6 @@ class ADS(interfaces.plugins.PluginInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" header_object = symbol_table + constants.BANG + "ATTR_HEADER" - attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields @@ -253,7 +251,7 @@ class ADS(interfaces.plugins.PluginInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - file_name = "N/A" + file_name = renderers.NotAvailableValue is_ads = False # The First $DATA Attr is the 'principal' file itself not the ADS while attr_header.AttrType.is_valid_choice: @@ -262,8 +260,8 @@ class ADS(interfaces.plugins.PluginInterface): offset + attr_base_offset + self.context.symbol_space.get_type( - attribute_object - ).relative_child_offset("Attr_Data") + header_object + ).size ) if attr_header.AttrType.lookup() == "FILE_NAME": @@ -272,7 +270,6 @@ class ADS(interfaces.plugins.PluginInterface): ) file_name = attr_data.get_full_name() - # DATA Attribute (can be ADS or not) if attr_header.AttrType.lookup() == "DATA": if is_ads: @@ -284,19 +281,21 @@ class ADS(interfaces.plugins.PluginInterface): + attr_base_offset + attr_header.NameOffset ) + ads_name = self._context.layers[layer.name].read( attr_name_offset, attr_header.NameLength*2 , pad=True ).decode('utf-16') + attr_content_offset = ( offset + attr_base_offset + attr_header.ContentOffset - ) + ) + content = self._context.layers[layer.name].read( attr_content_offset, attr_header.ContentLength , pad=True ) - # Preparing for Disassembly architecture = layer.metadata.get("architecture", None) disasm = interfaces.renderers.Disassembly( @@ -330,7 +329,6 @@ class ADS(interfaces.plugins.PluginInterface): layer_name=layer.name, ) - def run(self): return renderers.TreeGrid( [ diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index 616e8990d..d4f2aef7a 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -230,21 +230,21 @@ "offset": 0, "type": { "kind": "struct", - "name": "mft!ATTR_HEADER" + "name": "ATTR_HEADER" } }, "Resident_Header": { "offset": 16, "type": { "kind": "struct", - "name": "mft!RESIDENT_HEADER" + "name": "RESIDENT_HEADER" } }, "Attr_Data": { "offset": 24, "type": { "kind": "struct", - "name": "mft!ATTR_HEADER" + "name": "ATTR_HEADER" } } }, @@ -317,7 +317,7 @@ } }, "kind": "struct", - "size": 22 + "size": 24 },"RESIDENT_HEADER": { "fields": { "AttrSize": { From 7624c494e81fddfe1f4ae1b754fae8fbecb76ee2 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 28 Nov 2023 13:57:02 +0100 Subject: [PATCH 08/11] Simplify attribute object accesses like #1049 + custom class (MFTAttribute) --- .../framework/plugins/windows/mftscan.py | 71 ++++++------------- .../symbols/windows/extensions/mft.py | 22 ++++++ 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 397ce75bc..623497638 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -224,12 +224,12 @@ class ADS(interfaces.plugins.PluginInterface): config_path=self.config_path, sub_path="windows", filename="mft", - class_types={"MFT_ENTRY": mft.MFTEntry,"FILE_NAME_ENTRY": mft.MFTFileName}, + class_types={"MFT_ENTRY": mft.MFTEntry,"FILE_NAME_ENTRY": mft.MFTFileName, "ATTRIBUTE": mft.MFTAttribute}, ) # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - header_object = symbol_table + constants.BANG + "ATTR_HEADER" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields @@ -243,58 +243,32 @@ class ADS(interfaces.plugins.PluginInterface): # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - attr_header = self.context.object( - header_object, + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) # There is no field that has a count of Attributes - # Keep Attempting to read attributes until we get an invalid attr_header.AttrType + # Keep Attempting to read attributes until we get an invalid attr.AttrType file_name = renderers.NotAvailableValue is_ads = False - # The First $DATA Attr is the 'principal' file itself not the ADS - while attr_header.AttrType.is_valid_choice: - # Offset past the headers to the attribute data - attr_data_offset = ( - offset - + attr_base_offset - + self.context.symbol_space.get_type( - header_object - ).size - ) + + # The First $DATA Attr is the 'principal' file itself not the ADS + while attr.Attr_Header.AttrType.is_valid_choice: - if attr_header.AttrType.lookup() == "FILE_NAME": - attr_data = self.context.object( - fn_object, offset=attr_data_offset, layer_name=layer.name - ) + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() - - # DATA Attribute (can be ADS or not) - if attr_header.AttrType.lookup() == "DATA": + + if attr.Attr_Header.AttrType.lookup() == "DATA": if is_ads: - if not attr_header.NonResidentFlag: + if not attr.Attr_Header.NonResidentFlag: # Resident files are the most interesting. - if attr_header.NameLength > 0: - attr_name_offset = ( - offset - + attr_base_offset - + attr_header.NameOffset - ) + if attr.Attr_Header.NameLength > 0: - ads_name = self._context.layers[layer.name].read( - attr_name_offset, attr_header.NameLength*2 , pad=True - ).decode('utf-16') - - attr_content_offset = ( - offset - + attr_base_offset - + attr_header.ContentOffset - ) - - content = self._context.layers[layer.name].read( - attr_content_offset, attr_header.ContentLength , pad=True - ) + ads_name = attr.get_resident_filename() + content = attr.get_resident_filecontent() # Preparing for Disassembly architecture = layer.metadata.get("architecture", None) @@ -303,10 +277,10 @@ class ADS(interfaces.plugins.PluginInterface): ) yield 0, ( - format_hints.Hex(attr_data_offset), + format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), mft_record.RecordNumber, - attr_header.AttrType.lookup(), + attr.Attr_Header.AttrType.lookup(), file_name, ads_name, format_hints.HexBytes(content), @@ -317,18 +291,17 @@ class ADS(interfaces.plugins.PluginInterface): # If there's no advancement the loop will never end, so break it now - if attr_header.Length == 0: + if attr.Attr_Header.Length == 0: break # Update the base offset to point to the next attribute - attr_base_offset += attr_header.Length + attr_base_offset += attr.Attr_Header.Length # Get the next attribute - attr_header = self.context.object( - header_object, + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) - def run(self): return renderers.TreeGrid( [ diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 17b6c8325..1b5d5fce4 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -21,3 +21,25 @@ class MFTFileName(objects.StructType): "string", encoding="utf16", max_length=self.NameLength * 2, errors="replace" ) return output + + +class MFTAttribute(objects.StructType): + """This represents an MFT ATTRIBUTE""" + + def get_resident_filename(self) -> str: + # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data + layer = self._context.layers[self.vol.layer_name] + attr_name_offset = self.vol.offset + self.Attr_Header.NameOffset + + return self._context.layers[layer.name].read( + attr_name_offset, self.Attr_Header.NameLength*2 , pad=True + ).decode('utf-16') + + def get_resident_filecontent(self) -> bytes: + # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data + layer = self._context.layers[self.vol.layer_name] + attr_content_offset = self.vol.offset + self.Attr_Header.ContentOffset + + return self._context.layers[layer.name].read( + attr_content_offset, self.Attr_Header.ContentLength , pad=True + ) From c610497fa04de41042104fdefbfa243c0bcf76a9 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 29 Nov 2023 09:28:34 +0000 Subject: [PATCH 09/11] Windows: update vadyarascan to use generic yarascan requirements --- .../framework/plugins/windows/vadyarascan.py | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 4b30a9d8b..d795818e9 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -18,47 +18,26 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [ + # create a list of requirements for vadyarascan + vadyarascan_requirements = [ requirements.ModuleRequirement( name="kernel", description="Windows kernel", architectures=["Intel32", "Intel64"], ), - requirements.BooleanRequirement( - name="wide", - description="Match wide (unicode) strings", - default=False, - optional=True, - ), - requirements.StringRequirement( - name="yara_rules", description="Yara rules (as a string)", optional=True - ), - requirements.URIRequirement( - name="yara_file", description="Yara rules (as a file)", optional=True - ), - # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code - # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later - requirements.URIRequirement( - name="yara_compiled_file", - description="Yara compiled rules (as a file)", - optional=True, - ), - requirements.IntRequirement( - name="max_size", - default=0x40000000, - description="Set the maximum size (default is 1GB)", - optional=True, - ), requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.VersionRequirement( name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) + ), requirements.ListRequirement( name="pid", element_type=int, @@ -67,6 +46,12 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ), ] + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() + + # return the combined requirements + return yarascan_requirements + vadyarascan_requirements + def _generator(self): kernel = self.context.modules[self.config["kernel"]] From ed2db939d6b36d18dd44bad13d6a603b760b62a2 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 3 Dec 2023 12:24:11 +0100 Subject: [PATCH 10/11] Better exception handling. Fetching data using objects --- .../framework/plugins/windows/mftscan.py | 93 +++++++++---------- .../symbols/windows/extensions/mft.py | 39 +++++--- 2 files changed, 70 insertions(+), 62 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 623497638..4d58eb1e2 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -31,7 +31,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) ), - ] def _generator(self): @@ -53,6 +52,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" @@ -67,9 +67,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - - attr_header = self.context.object( - header_object, + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -77,17 +76,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr_header.AttrType.is_valid_choice: - vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") - - # Offset past the headers to the attribute data - attr_data_offset = ( - offset - + attr_base_offset - + self.context.symbol_space.get_type( - header_object - ).size - ) + while attr.Attr_Header.AttrType.is_valid_choice: + vollog.debug(f"Attr Type: {attr.Attr_Header.AttrType.lookup()}") # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record @@ -97,19 +87,16 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType.lookup() == "STANDARD_INFORMATION": - attr_data = self.context.object( - si_object, offset=attr_data_offset, layer_name=layer.name - ) - + if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": + attr_data = attr.Attr_Data.cast(si_object) yield 0, ( - format_hints.Hex(attr_data_offset), + format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, renderers.NotApplicableValue(), - attr_header.AttrType.lookup(), + attr.Attr_Header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -118,10 +105,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType.lookup() == "FILE_NAME": - attr_data = self.context.object( - fn_object, offset=attr_data_offset, layer_name=layer.name - ) + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() # If we don't have a valid enum, coerce to hex so we can keep the record @@ -131,13 +116,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): permissions = hex(attr_data.Flags) yield 1, ( - format_hints.Hex(attr_data_offset), + format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, - attr_header.AttrType.lookup(), + attr.Attr_Header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -146,14 +131,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # If there's no advancement the loop will never end, so break it now - if attr_header.Length == 0: + if attr.Attr_Header.Length == 0: break # Update the base offset to point to the next attribute - attr_base_offset += attr_header.Length - # Get the next attribute - attr_header = self.context.object( - header_object, + attr_base_offset += attr.Attr_Header.Length + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -224,7 +208,11 @@ class ADS(interfaces.plugins.PluginInterface): config_path=self.config_path, sub_path="windows", filename="mft", - class_types={"MFT_ENTRY": mft.MFTEntry,"FILE_NAME_ENTRY": mft.MFTFileName, "ATTRIBUTE": mft.MFTAttribute}, + class_types={ + "MFT_ENTRY": mft.MFTEntry, + "FILE_NAME_ENTRY": mft.MFTFileName, + "ATTRIBUTE": mft.MFTAttribute, + }, ) # get each of the individual Field Sets @@ -251,30 +239,39 @@ class ADS(interfaces.plugins.PluginInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr.AttrType - file_name = renderers.NotAvailableValue is_ads = False - + file_name = renderers.NotAvailableValue # The First $DATA Attr is the 'principal' file itself not the ADS while attr.Attr_Header.AttrType.is_valid_choice: - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() - if attr.Attr_Header.AttrType.lookup() == "DATA": if is_ads: if not attr.Attr_Header.NonResidentFlag: # Resident files are the most interesting. if attr.Attr_Header.NameLength > 0: - ads_name = attr.get_resident_filename() - content = attr.get_resident_filecontent() + if not ads_name: + ads_name = renderers.NotAvailableValue - # Preparing for Disassembly - architecture = layer.metadata.get("architecture", None) - disasm = interfaces.renderers.Disassembly( - content, 0, architecture.lower() - ) + content = attr.get_resident_filecontent() + if content: + # Preparing for Disassembly + architecture = layer.metadata.get( + "architecture", None + ) + + disasm = ( + interfaces.renderers.Disassembly( + content, 0, architecture.lower() + ) + if architecture + else interfaces.renderers.BaseAbsentValue + ) + else: + content = renderers.NotAvailableValue + disasm = interfaces.renderers.BaseAbsentValue yield 0, ( format_hints.Hex(attr_data.vol.offset), @@ -288,8 +285,7 @@ class ADS(interfaces.plugins.PluginInterface): ) else: is_ads = True - - + # If there's no advancement the loop will never end, so break it now if attr.Attr_Header.Length == 0: break @@ -302,6 +298,7 @@ class ADS(interfaces.plugins.PluginInterface): offset=offset + attr_base_offset, layer_name=layer.name, ) + def run(self): return renderers.TreeGrid( [ @@ -315,4 +312,4 @@ class ADS(interfaces.plugins.PluginInterface): ("Disasm", interfaces.renderers.Disassembly), ], self._generator(), - ) \ No newline at end of file + ) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 1b5d5fce4..14c1f08d6 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import objects +from volatility3.framework import objects, constants, exceptions class MFTEntry(objects.StructType): @@ -28,18 +28,29 @@ class MFTAttribute(objects.StructType): def get_resident_filename(self) -> str: # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data - layer = self._context.layers[self.vol.layer_name] - attr_name_offset = self.vol.offset + self.Attr_Header.NameOffset - - return self._context.layers[layer.name].read( - attr_name_offset, self.Attr_Header.NameLength*2 , pad=True - ).decode('utf-16') - + try: + name = self._context.object( + self.vol.type_name.split(constants.BANG)[0] + constants.BANG + "string", + layer_name=self.vol.layer_name, + offset=self.vol.offset + self.Attr_Header.NameOffset, + max_length=self.Attr_Header.NameLength * 2, + errors="replace", + encoding="utf16", + ) + return name + except exceptions.InvalidAddressException: + return None + def get_resident_filecontent(self) -> bytes: # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data - layer = self._context.layers[self.vol.layer_name] - attr_content_offset = self.vol.offset + self.Attr_Header.ContentOffset - - return self._context.layers[layer.name].read( - attr_content_offset, self.Attr_Header.ContentLength , pad=True - ) + try: + bytesobj = self._context.object( + self.vol.type_name.split(constants.BANG)[0] + constants.BANG + "bytes", + layer_name=self.vol.layer_name, + offset=self.vol.offset + self.Attr_Header.ContentOffset, + native_layer_name=self.vol.native_layer_name, + length=self.Attr_Header.ContentLength, + ) + return bytesobj + except exceptions.InvalidAddressException: + return None From acb088dbcdb8532e643a72fb4571ae1cba24786c Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 3 Dec 2023 13:05:45 +0100 Subject: [PATCH 11/11] Better code reading --- volatility3/framework/plugins/windows/mftscan.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 4d58eb1e2..7e4e1ca18 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -258,17 +258,14 @@ class ADS(interfaces.plugins.PluginInterface): content = attr.get_resident_filecontent() if content: # Preparing for Disassembly + disasm = interfaces.renderers.BaseAbsentValue architecture = layer.metadata.get( "architecture", None ) - - disasm = ( - interfaces.renderers.Disassembly( + if architecture: + disasm = interfaces.renderers.Disassembly( content, 0, architecture.lower() ) - if architecture - else interfaces.renderers.BaseAbsentValue - ) else: content = renderers.NotAvailableValue disasm = interfaces.renderers.BaseAbsentValue