From 2c949dfa50e2b530a3df9cc480933e583a1ca4e7 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Thu, 6 Jan 2022 22:53:24 +0000 Subject: [PATCH 01/13] Create MFTScanner plugin --- .../framework/plugins/windows/mftscan.py | 305 ++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 volatility3/framework/plugins/windows/mftscan.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py new file mode 100644 index 000000000..1d4ca954e --- /dev/null +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -0,0 +1,305 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from struct import unpack +from typing import Iterable + +from volatility3.framework import constants, renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework.renderers import conversion, format_hints +from volatility3.framework.symbols import intermed +from volatility3.plugins import yarascan + +vollog = logging.getLogger(__name__) + +try: + import yara +except ImportError: + vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") + raise + +signatures = { + 'mft_objects': """rule mft_headers + { + strings: + $header1 = "FILE0" + $header2 = "FILE*" + $header3 = "BAAD" + condition: + any of them + }""" +} + +# https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee9338b881c0fa25937/volatility/plugins/mftparser.py#L60 +ATTRIBUTE_TYPE_ID = { + 0x10:"STANDARD_INFORMATION", + 0x20:"ATTRIBUTE_LIST", + 0x30:"FILE_NAME", + 0x40:"OBJECT_ID", + 0x50:"SECURITY_DESCRIPTOR", + 0x60:"VOLUME_NAME", + 0x70:"VOLUME_INFORMATION", + 0x80:"DATA", + 0x90:"INDEX_ROOT", + 0xa0:"INDEX_ALLOCATION", + 0xb0:"BITMAP", + 0xc0:"REPARSE_POINT", + 0xd0:"EA_INFORMATION", #Extended Attribute + 0xe0:"EA", + 0xf0:"PROPERTY_SET", + 0x100:"LOGGED_UTILITY_STREAM", +} + +VERBOSE_STANDARD_INFO_FLAGS = { + 0x1:"Read Only", + 0x2:"Hidden", + 0x4:"System", + 0x20:"Archive", + 0x40:"Device", + 0x80:"Normal", + 0x100:"Temporary", + 0x200:"Sparse File", + 0x400:"Reparse Point", + 0x800:"Compressed", + 0x1000:"Offline", + 0x2000:"Content not indexed", + 0x4000:"Encrypted", + 0x10000000:"Directory", + 0x20000000:"Index view", +} + +FILE_NAME_NAMESPACE = { + 0x0:"POSIX", # Case sensitive, allows all Unicode chars except '/' and NULL + 0x1:"Win32", # Case insensitive, allows most Unicide except specials ('/', '\', ';', '>', '<', '?') + 0x2:"DOS", # Case insensitive, upper case, no special chars, name is 8 or fewer chars in name and 3 or less extension + 0x3:"Win32 & DOS", # Used when original name fits in DOS namespace and 2 names are not needed +} + +MFT_FLAGS = { + 0x0: "Removed", + 0x1: "File", # "In Use", + 0x2: "Directory", # if flag & 0x0002 == 0 this is a regular file + 0x3: "Directory" +} + +INDEX_ENTRY_FLAGS = { + 0x1:"Child Node Exists", + 0x2:"Last entry in list", +} + + +class MFTScan(interfaces.plugins.PluginInterface): + """Scans for MFT FILE objects present in a particular windows memory image.""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, + version = (2, 0, 0)), + ] + + # https://docs.python.org/3/library/struct.html + @classmethod + def unpack_data(self, mft_record, offset, data_type): + """Helper to unpack values from the raw mft_record""" + + if data_type == 'unsigned long': + return unpack(' 1000: + continue + + # attr_header + attr_type = self.unpack_data(mft_record, attr_offset, 'int') + attr_len = self.unpack_data(mft_record, attr_offset+4, 'int') + + # As we look for strucutres of header + 1K we can not unpack non resident structures + nr_flag = self.unpack_data(mft_record, attr_offset+8, 'unsigned char') + + # Skip headers + attr_data = attr_offset+24 # Len of Common and Resident Headers + + if attr_type in ATTRIBUTE_TYPE_ID: + vollog.debug(f'Found Attribute {ATTRIBUTE_TYPE_ID[attr_type]}') + + if ATTRIBUTE_TYPE_ID[attr_type] == 'STANDARD_INFORMATION': + creation_time_win = self.unpack_data(mft_record, attr_data, 'unsigned long long') + modified_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') + altered_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') + access_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') + flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') + permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') + + + mft_entry['attributes']['SI'] = { + "creation_time": self.human_date(creation_time_win), + "modified_time": self.human_date(modified_time_win), + "updated_time": self.human_date(altered_time_win), + "accessed_time": self.human_date(access_time_win), + "flags": permissions + } + + if ATTRIBUTE_TYPE_ID[attr_type] == 'FILE_NAME': + parent_record = self.unpack_data(mft_record, attr_data, 'unsigned long long') + creation_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') + modified_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') + altered_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') + access_time_win = self.unpack_data(mft_record, attr_data+32, 'unsigned long long') + + name_len = self.unpack_data(mft_record, attr_data+64, 'unsigned char') + name_space = self.unpack_data(mft_record, attr_data+65, 'unsigned char') + + # Unicode and partially corruprted records can break us here. + file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] + try: + file_name = file_name.replace(b'\x00', b'').decode() + except: + file_name = str(file_name.replace(b'\x00', b'')) + + flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') + permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') + + mft_entry['attributes']['FN'].append( + { + "creation_time": self.human_date(creation_time_win), + "modified_time": self.human_date(modified_time_win), + "updated_time": self.human_date(altered_time_win), + "accessed_time": self.human_date(access_time_win), + "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), + "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), + "flags": permissions, + "file_name": file_name, + "name_space": name_space + }) + + # Update Offset for next Attribute + attr_offset += attr_len + + return mft_entry + + def _generator(self): + rules = yara.compile(sources = signatures) + + layer = self.context.layers[self.config['primary']] + for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + + try: + mft_record = layer.read(offset, 1024, False) + mft_entry = self.parse_mft_record(mft_record) + except PagedInvalidAddressException: + mft_entry = None + except Exception as err: + vollog.error(err) + mft_entry = None + + if mft_entry: + vollog.debug(mft_entry) + + # Tree Grid is large and variable + si = mft_entry['attributes']['SI'] + fn = mft_entry['attributes']['FN'] + + signature = mft_entry.get('signature', 0) + record_number = mft_entry.get('record_number', 0) + link_count = mft_entry.get('link_count', 0) + permissions = mft_entry.get('flags', '') + + si_creation_time = si.get('creation_time', '') + si_modified_time = si.get('modified_time', '') + si_updated_time = si.get('updated_time', '') + si_accessed_time = si.get('accessed_time', '') + + yield 0, ( + format_hints.Hex(offset), + signature, + record_number, + link_count, + permissions, + 'Standard Information', + 'N/A', + si_creation_time, + si_modified_time, + si_updated_time, + si_accessed_time) + + for entry in fn: + # As this is variable and may or may not exist + # And could have 0-6 entries lets do it per row. + yield 0, ( + format_hints.Hex(offset), + signature, + record_number, + link_count, + permissions, + 'FileName', + entry.get('file_name', ''), + entry.get('creation_time', ''), + entry.get('modified_time', ''), + entry.get('updated_time', ''), + entry.get('accessed_time', '')) + + def run(self): + return renderers.TreeGrid([ + ('Offset', format_hints.Hex), + ('Record Type', str), + ('Record Number', int), + ('Link Count', int), + ('Permissions', str), + ('Attribute Type', str), + ('Filename', str), + ('Created', str), + ('Modified', str), + ('Updated', str), + ('Accessed', str) + ],self._generator()) From 899ec09ce3b620f75c9ecb74e117d8a3a918be3b Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Thu, 6 Jan 2022 23:09:50 +0000 Subject: [PATCH 02/13] Doc Strings --- .../framework/plugins/windows/mftscan.py | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 1d4ca954e..b75756a1d 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,13 @@ import logging from struct import unpack -from typing import Iterable +from typing import Dict -from volatility3.framework import constants, renderers, interfaces, exceptions +from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols import intermed from volatility3.plugins import yarascan vollog = logging.getLogger(__name__) @@ -109,8 +109,17 @@ class MFTScan(interfaces.plugins.PluginInterface): # https://docs.python.org/3/library/struct.html @classmethod - def unpack_data(self, mft_record, offset, data_type): - """Helper to unpack values from the raw mft_record""" + def unpack_data(self, mft_record: bytes, offset: int, data_type: str) -> bytes: + """Helper to unpack values from the raw mft_record + + Args: + mft_record: 1024 bytes starting from header value as returned by layer read + offset: how far in to the record to read + data_type: what is the data type to unpack + + Returns: + bytes: the unpacked data + """ if data_type == 'unsigned long': return unpack(' str: + """Converts a windows epoch to a date time string with a fixed format + + Args: + datetime_object: windows epoch time + + Returns: + str: strftime of the windows epoch in UTC + + """ dtg = conversion.wintime_to_datetime(datetime_object) return dtg.strftime('%Y-%m-%d %H:%M:%S %z') @classmethod - def parse_mft_record(self, mft_record): - """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes""" + def parse_mft_record(self, mft_record: bytes) -> Dict: + """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes + + Args: + mft_record: 1024 bytes starting from header value as returned by layer read + + Returns: + Dict: a Dictionary that contains the Parse MFT Record + """ # https://github.com/Invoke-IR/ForensicPosters flags = self.unpack_data(mft_record, 22, 'unsigned short') @@ -202,10 +226,11 @@ class MFTScan(interfaces.plugins.PluginInterface): # Unicode and partially corruprted records can break us here. file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - try: - file_name = file_name.replace(b'\x00', b'').decode() - except: - file_name = str(file_name.replace(b'\x00', b'')) + file_name = utility.array_to_string(file_name) + #try: + # # file_name = file_name.replace(b'\x00', b'').decode() + #except: + # file_name = str(file_name.replace(b'\x00', b'')) flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') @@ -234,6 +259,7 @@ class MFTScan(interfaces.plugins.PluginInterface): layer = self.context.layers[self.config['primary']] for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + # For each matching rule try to read 1024 bytes (size of an MFT record) at the offset. try: mft_record = layer.read(offset, 1024, False) mft_entry = self.parse_mft_record(mft_record) From 20a5868ff3df16710695b497069aba9ff0387842 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 00:23:31 +0000 Subject: [PATCH 03/13] Change return types for MFT Records DTGs --- .../framework/plugins/windows/mftscan.py | 101 ++++++++---------- 1 file changed, 44 insertions(+), 57 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index b75756a1d..ddc179fe1 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -2,14 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime import logging +import struct -from struct import unpack from typing import Dict from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.exceptions import PagedInvalidAddressException +from volatility3.framework import exceptions from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints from volatility3.plugins import yarascan @@ -122,29 +123,15 @@ class MFTScan(interfaces.plugins.PluginInterface): """ if data_type == 'unsigned long': - return unpack(' str: - """Converts a windows epoch to a date time string with a fixed format - - Args: - datetime_object: windows epoch time - - Returns: - str: strftime of the windows epoch in UTC - - """ - dtg = conversion.wintime_to_datetime(datetime_object) - return dtg.strftime('%Y-%m-%d %H:%M:%S %z') + return struct.unpack(' Dict: @@ -207,10 +194,10 @@ class MFTScan(interfaces.plugins.PluginInterface): mft_entry['attributes']['SI'] = { - "creation_time": self.human_date(creation_time_win), - "modified_time": self.human_date(modified_time_win), - "updated_time": self.human_date(altered_time_win), - "accessed_time": self.human_date(access_time_win), + "creation_time": conversion.wintime_to_datetime(creation_time_win), + "modified_time": conversion.wintime_to_datetime(modified_time_win), + "updated_time": conversion.wintime_to_datetime(altered_time_win), + "accessed_time": conversion.wintime_to_datetime(access_time_win), "flags": permissions } @@ -226,21 +213,21 @@ class MFTScan(interfaces.plugins.PluginInterface): # Unicode and partially corruprted records can break us here. file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - file_name = utility.array_to_string(file_name) - #try: - # # file_name = file_name.replace(b'\x00', b'').decode() - #except: - # file_name = str(file_name.replace(b'\x00', b'')) + #file_name = utility.array_to_string(file_name) + try: + file_name = file_name.replace(b'\x00', b'').decode() + except: + file_name = str(file_name.replace(b'\x00', b'')) flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') mft_entry['attributes']['FN'].append( { - "creation_time": self.human_date(creation_time_win), - "modified_time": self.human_date(modified_time_win), - "updated_time": self.human_date(altered_time_win), - "accessed_time": self.human_date(access_time_win), + "creation_time": conversion.wintime_to_datetime(creation_time_win), + "modified_time": conversion.wintime_to_datetime(modified_time_win), + "updated_time": conversion.wintime_to_datetime(altered_time_win), + "accessed_time": conversion.wintime_to_datetime(access_time_win), "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), "flags": permissions, @@ -263,11 +250,11 @@ class MFTScan(interfaces.plugins.PluginInterface): try: mft_record = layer.read(offset, 1024, False) mft_entry = self.parse_mft_record(mft_record) - except PagedInvalidAddressException: - mft_entry = None - except Exception as err: - vollog.error(err) + except exceptions.PagedInvalidAddressException: mft_entry = None + #except Exception as err: + # vollog.error(err) + # mft_entry = None if mft_entry: vollog.debug(mft_entry) @@ -276,15 +263,15 @@ class MFTScan(interfaces.plugins.PluginInterface): si = mft_entry['attributes']['SI'] fn = mft_entry['attributes']['FN'] - signature = mft_entry.get('signature', 0) - record_number = mft_entry.get('record_number', 0) - link_count = mft_entry.get('link_count', 0) - permissions = mft_entry.get('flags', '') + signature = mft_entry.get('signature', renderers.NotAvailableValue()) + record_number = mft_entry.get('record_number', renderers.NotAvailableValue()) + link_count = mft_entry.get('link_count', renderers.NotAvailableValue()) + permissions = mft_entry.get('flags', renderers.NotAvailableValue()) - si_creation_time = si.get('creation_time', '') - si_modified_time = si.get('modified_time', '') - si_updated_time = si.get('updated_time', '') - si_accessed_time = si.get('accessed_time', '') + si_creation_time = si.get('creation_time', renderers.NotAvailableValue()) + si_modified_time = si.get('modified_time', renderers.NotAvailableValue()) + si_updated_time = si.get('updated_time', renderers.NotAvailableValue()) + si_accessed_time = si.get('accessed_time', renderers.NotAvailableValue()) yield 0, ( format_hints.Hex(offset), @@ -293,7 +280,7 @@ class MFTScan(interfaces.plugins.PluginInterface): link_count, permissions, 'Standard Information', - 'N/A', + renderers.NotApplicableValue(), si_creation_time, si_modified_time, si_updated_time, @@ -302,18 +289,18 @@ class MFTScan(interfaces.plugins.PluginInterface): for entry in fn: # As this is variable and may or may not exist # And could have 0-6 entries lets do it per row. - yield 0, ( + yield 1, ( format_hints.Hex(offset), signature, record_number, link_count, permissions, 'FileName', - entry.get('file_name', ''), - entry.get('creation_time', ''), - entry.get('modified_time', ''), - entry.get('updated_time', ''), - entry.get('accessed_time', '')) + entry.get('file_name',''), + entry.get('creation_time', renderers.NotAvailableValue()), + entry.get('modified_time', renderers.NotAvailableValue()), + entry.get('updated_time', renderers.NotAvailableValue()), + entry.get('accessed_time', renderers.NotAvailableValue())) def run(self): return renderers.TreeGrid([ @@ -324,8 +311,8 @@ class MFTScan(interfaces.plugins.PluginInterface): ('Permissions', str), ('Attribute Type', str), ('Filename', str), - ('Created', str), - ('Modified', str), - ('Updated', str), - ('Accessed', str) + ('Created', datetime.datetime), + ('Modified', datetime.datetime), + ('Updated', datetime.datetime), + ('Accessed', datetime.datetime) ],self._generator()) From 295fb453f5e73f65d39912336ef6517268e26d17 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 00:55:34 +0000 Subject: [PATCH 04/13] Add MFT Filename N/A type --- volatility3/framework/plugins/windows/mftscan.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index ddc179fe1..10db6f112 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -192,7 +192,6 @@ class MFTScan(interfaces.plugins.PluginInterface): flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - mft_entry['attributes']['SI'] = { "creation_time": conversion.wintime_to_datetime(creation_time_win), "modified_time": conversion.wintime_to_datetime(modified_time_win), @@ -296,7 +295,7 @@ class MFTScan(interfaces.plugins.PluginInterface): link_count, permissions, 'FileName', - entry.get('file_name',''), + entry.get('file_name',renderers.NotAvailableValue()), entry.get('creation_time', renderers.NotAvailableValue()), entry.get('modified_time', renderers.NotAvailableValue()), entry.get('updated_time', renderers.NotAvailableValue()), From 9d86599e7f391c41f2fb7e06f6f0f811bd97a7b7 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 22:04:13 +0000 Subject: [PATCH 05/13] MFT Plugin use ISF instead of Struct --- .../framework/plugins/windows/mftscan.py | 352 ++++------------- .../symbols/windows/extensions/mft.py | 104 +++++ .../framework/symbols/windows/mft.json | 371 ++++++++++++++++++ volatility3/framework/symbols/windows/mft.py | 15 + 4 files changed, 577 insertions(+), 265 deletions(-) create mode 100644 volatility3/framework/symbols/windows/extensions/mft.py create mode 100644 volatility3/framework/symbols/windows/mft.json create mode 100644 volatility3/framework/symbols/windows/mft.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 10db6f112..484fcdb9a 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -4,95 +4,20 @@ import datetime import logging -import struct from typing import Dict from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions -from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion, format_hints +from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags +from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols + from volatility3.plugins import yarascan vollog = logging.getLogger(__name__) -try: - import yara -except ImportError: - vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") - raise - -signatures = { - 'mft_objects': """rule mft_headers - { - strings: - $header1 = "FILE0" - $header2 = "FILE*" - $header3 = "BAAD" - condition: - any of them - }""" -} - -# https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee9338b881c0fa25937/volatility/plugins/mftparser.py#L60 -ATTRIBUTE_TYPE_ID = { - 0x10:"STANDARD_INFORMATION", - 0x20:"ATTRIBUTE_LIST", - 0x30:"FILE_NAME", - 0x40:"OBJECT_ID", - 0x50:"SECURITY_DESCRIPTOR", - 0x60:"VOLUME_NAME", - 0x70:"VOLUME_INFORMATION", - 0x80:"DATA", - 0x90:"INDEX_ROOT", - 0xa0:"INDEX_ALLOCATION", - 0xb0:"BITMAP", - 0xc0:"REPARSE_POINT", - 0xd0:"EA_INFORMATION", #Extended Attribute - 0xe0:"EA", - 0xf0:"PROPERTY_SET", - 0x100:"LOGGED_UTILITY_STREAM", -} - -VERBOSE_STANDARD_INFO_FLAGS = { - 0x1:"Read Only", - 0x2:"Hidden", - 0x4:"System", - 0x20:"Archive", - 0x40:"Device", - 0x80:"Normal", - 0x100:"Temporary", - 0x200:"Sparse File", - 0x400:"Reparse Point", - 0x800:"Compressed", - 0x1000:"Offline", - 0x2000:"Content not indexed", - 0x4000:"Encrypted", - 0x10000000:"Directory", - 0x20000000:"Index view", -} - -FILE_NAME_NAMESPACE = { - 0x0:"POSIX", # Case sensitive, allows all Unicode chars except '/' and NULL - 0x1:"Win32", # Case insensitive, allows most Unicide except specials ('/', '\', ';', '>', '<', '?') - 0x2:"DOS", # Case insensitive, upper case, no special chars, name is 8 or fewer chars in name and 3 or less extension - 0x3:"Win32 & DOS", # Used when original name fits in DOS namespace and 2 names are not needed -} - -MFT_FLAGS = { - 0x0: "Removed", - 0x1: "File", # "In Use", - 0x2: "Directory", # if flag & 0x0002 == 0 this is a regular file - 0x3: "Directory" -} - -INDEX_ENTRY_FLAGS = { - 0x1:"Child Node Exists", - 0x2:"Last entry in list", -} - - class MFTScan(interfaces.plugins.PluginInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" @@ -108,198 +33,94 @@ class MFTScan(interfaces.plugins.PluginInterface): version = (2, 0, 0)), ] - # https://docs.python.org/3/library/struct.html - @classmethod - def unpack_data(self, mft_record: bytes, offset: int, data_type: str) -> bytes: - """Helper to unpack values from the raw mft_record - - Args: - mft_record: 1024 bytes starting from header value as returned by layer read - offset: how far in to the record to read - data_type: what is the data type to unpack - - Returns: - bytes: the unpacked data - """ - - if data_type == 'unsigned long': - return struct.unpack(' Dict: - """Takes an MFT Record and attempts to parse, MFT, SI and FN attributes - - Args: - mft_record: 1024 bytes starting from header value as returned by layer read - - Returns: - Dict: a Dictionary that contains the Parse MFT Record - """ - # https://github.com/Invoke-IR/ForensicPosters - - flags = self.unpack_data(mft_record, 22, 'unsigned short') - file_type = MFT_FLAGS.get(flags, 'Unknown') - - mft_entry = { - "signature": mft_record[:4].decode(), - "FixupArrayOffset": self.unpack_data(mft_record, 4, 'unsigned short'), - "NumFixupEntries": self.unpack_data(mft_record, 6, 'unsigned short'), - "LSN": self.unpack_data(mft_record, 8, 'unsigned long long'), - "SequenceValue": self.unpack_data(mft_record, 16, 'unsigned short'), - "link_count": self.unpack_data(mft_record, 18, 'unsigned short'), - "FirstAttrOffset": self.unpack_data(mft_record, 20, 'unsigned short'), - "flags": file_type, - "record_number": self.unpack_data(mft_record, 44, 'unsigned long'), - "attributes": { - "SI": {}, - "FN": [] - } - } - - attr_offset = mft_entry['FirstAttrOffset'] - # Check at most for 6 entries - for i in range(6): - # If we attempt to overread the entry continue out - if attr_offset > 1000: - continue - - # attr_header - attr_type = self.unpack_data(mft_record, attr_offset, 'int') - attr_len = self.unpack_data(mft_record, attr_offset+4, 'int') - - # As we look for strucutres of header + 1K we can not unpack non resident structures - nr_flag = self.unpack_data(mft_record, attr_offset+8, 'unsigned char') - - # Skip headers - attr_data = attr_offset+24 # Len of Common and Resident Headers - - if attr_type in ATTRIBUTE_TYPE_ID: - vollog.debug(f'Found Attribute {ATTRIBUTE_TYPE_ID[attr_type]}') - - if ATTRIBUTE_TYPE_ID[attr_type] == 'STANDARD_INFORMATION': - creation_time_win = self.unpack_data(mft_record, attr_data, 'unsigned long long') - modified_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') - altered_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') - access_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') - flags = self.unpack_data(mft_record, attr_data+32, 'unsigned short') - permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - - mft_entry['attributes']['SI'] = { - "creation_time": conversion.wintime_to_datetime(creation_time_win), - "modified_time": conversion.wintime_to_datetime(modified_time_win), - "updated_time": conversion.wintime_to_datetime(altered_time_win), - "accessed_time": conversion.wintime_to_datetime(access_time_win), - "flags": permissions - } - - if ATTRIBUTE_TYPE_ID[attr_type] == 'FILE_NAME': - parent_record = self.unpack_data(mft_record, attr_data, 'unsigned long long') - creation_time_win = self.unpack_data(mft_record, attr_data+8, 'unsigned long long') - modified_time_win = self.unpack_data(mft_record, attr_data+16, 'unsigned long long') - altered_time_win = self.unpack_data(mft_record, attr_data+24, 'unsigned long long') - access_time_win = self.unpack_data(mft_record, attr_data+32, 'unsigned long long') - - name_len = self.unpack_data(mft_record, attr_data+64, 'unsigned char') - name_space = self.unpack_data(mft_record, attr_data+65, 'unsigned char') - - # Unicode and partially corruprted records can break us here. - file_name = mft_record[attr_data+66:attr_data+66+(2*name_len)] - #file_name = utility.array_to_string(file_name) - try: - file_name = file_name.replace(b'\x00', b'').decode() - except: - file_name = str(file_name.replace(b'\x00', b'')) - - flags = self.unpack_data(mft_record, attr_data+56, 'unsigned short') - permissions = VERBOSE_STANDARD_INFO_FLAGS.get(flags, 'Unknown') - - mft_entry['attributes']['FN'].append( - { - "creation_time": conversion.wintime_to_datetime(creation_time_win), - "modified_time": conversion.wintime_to_datetime(modified_time_win), - "updated_time": conversion.wintime_to_datetime(altered_time_win), - "accessed_time": conversion.wintime_to_datetime(access_time_win), - "allocated_size": self.unpack_data(mft_record, attr_data+40, 'unsigned long long'), - "real_size": self.unpack_data(mft_record, attr_data+48, 'unsigned long long'), - "flags": permissions, - "file_name": file_name, - "name_space": name_space - }) - - # Update Offset for next Attribute - attr_offset += attr_len - - return mft_entry def _generator(self): - rules = yara.compile(sources = signatures) - layer = self.context.layers[self.config['primary']] + + # Yara Rule to scan for MFT Header Signatures + rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) + + # Read in the Symbol File + symbol_table = MFTIntermedSymbols.create( + self.context, + self.config_path, + "windows", + "mft" + ) + + # get each of the individual Field Sets + mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + header_object = symbol_table + constants.BANG + "ATTR_HEADER" + si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Scan the layer for Raw MFT records and parse the fields for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): - - # For each matching rule try to read 1024 bytes (size of an MFT record) at the offset. try: - mft_record = layer.read(offset, 1024, False) - mft_entry = self.parse_mft_record(mft_record) + mft_record = self.context.object(mft_object, offset=offset, layer_name=layer.name) + # We will update this on each pass in the next loop and use it as the new offset. + attr_base_offset = mft_record.FirstAttrOffset + + # There is no field that has a count of Attributes + # Keep Attempting to read attributes until we get an invalid attr_header.AttrType + while True: + attr_header = self.context.object(header_object, offset=offset+attr_base_offset, layer_name=layer.name) + attr_resident_header = self.context.object(header_object, offset=offset+attr_base_offset+16, layer_name=layer.name) + + vollog.debug(f"Attr Type: {attr_header.AttrType}") + + # If this is not a valid type then exit the loop + if not AttributeTypes(attr_header.AttrType).value: + break + + # Offset past the headers to the attribute data + attr_data_offset = offset+attr_base_offset+24 + + # Standard Information Attribute + if attr_header.AttrType == 0x10: + attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) + + yield 0, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + MFTFlags(mft_record.Flags).name, + renderers.NotApplicableValue(), + AttributeTypes(attr_header.AttrType).name, + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + renderers.NotApplicableValue(), + ) + + # File Name Attribute + if attr_header.AttrType == 0x30: + attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) + file_name = attr_data.get_full_name() + + yield 1, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + MFTFlags(mft_record.Flags).name, + PermissionFlags(attr_data.Flags).name, + AttributeTypes(attr_header.AttrType).name, + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + file_name + ) + + # Update the base offset to point to the next attribute + attr_base_offset += attr_header.Length + except exceptions.PagedInvalidAddressException: - mft_entry = None - #except Exception as err: - # vollog.error(err) - # mft_entry = None + pass - if mft_entry: - vollog.debug(mft_entry) - - # Tree Grid is large and variable - si = mft_entry['attributes']['SI'] - fn = mft_entry['attributes']['FN'] - - signature = mft_entry.get('signature', renderers.NotAvailableValue()) - record_number = mft_entry.get('record_number', renderers.NotAvailableValue()) - link_count = mft_entry.get('link_count', renderers.NotAvailableValue()) - permissions = mft_entry.get('flags', renderers.NotAvailableValue()) - - si_creation_time = si.get('creation_time', renderers.NotAvailableValue()) - si_modified_time = si.get('modified_time', renderers.NotAvailableValue()) - si_updated_time = si.get('updated_time', renderers.NotAvailableValue()) - si_accessed_time = si.get('accessed_time', renderers.NotAvailableValue()) - - yield 0, ( - format_hints.Hex(offset), - signature, - record_number, - link_count, - permissions, - 'Standard Information', - renderers.NotApplicableValue(), - si_creation_time, - si_modified_time, - si_updated_time, - si_accessed_time) - - for entry in fn: - # As this is variable and may or may not exist - # And could have 0-6 entries lets do it per row. - yield 1, ( - format_hints.Hex(offset), - signature, - record_number, - link_count, - permissions, - 'FileName', - entry.get('file_name',renderers.NotAvailableValue()), - entry.get('creation_time', renderers.NotAvailableValue()), - entry.get('modified_time', renderers.NotAvailableValue()), - entry.get('updated_time', renderers.NotAvailableValue()), - entry.get('accessed_time', renderers.NotAvailableValue())) def run(self): return renderers.TreeGrid([ @@ -307,11 +128,12 @@ class MFTScan(interfaces.plugins.PluginInterface): ('Record Type', str), ('Record Number', int), ('Link Count', int), + ('MFT Type', str), ('Permissions', str), ('Attribute Type', str), - ('Filename', str), ('Created', datetime.datetime), ('Modified', datetime.datetime), ('Updated', datetime.datetime), - ('Accessed', datetime.datetime) + ('Accessed', datetime.datetime), + ('Filename', str), ],self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py new file mode 100644 index 000000000..09f6346cc --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -0,0 +1,104 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import enum + +from volatility3.framework import exceptions, objects, renderers +from volatility3.framework.objects import utility + + +class AttributeTypes(enum.Enum): + STANDARD_INFORMATION = 0x10 + ATTRIBUTE_LIST = 0x20 + FILE_NAME = 0x30 + OBJECT_ID = 0x40 + SECURITY_DESCRIPTOR = 0x50 + VOLUME_NAME = 0x60 + VOLUME_INFORMATION = 0x70 + DATA = 0x80 + INDEX_ROOT = 0x90 + INDEX_ALLOCATION = 0xa0 + BITMAP = 0xb0 + REPARSE_POINT = 0xc0 + EA_INFORMATION = 0xd0 + EA = 0xe0 + PROPERTY_SET = 0xf0 + LOGGED_UTILITY_STREAM = 0x100 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(AttributeTypes.Unknown) + +class NameSpace(enum.Enum): + POSIX = 0x0 + Win32 = 0x1 + DOS = 0x2 + Win32DOS = 0x3 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(NameSpace.Unknown) + + +class MFTFlags(enum.Enum): + Removed = 0x00 + File = 0x1 + Directory = 0x2 + DirInUse = 0x3 + Unknown = None + + @classmethod + def _missing_(cls, value): + return cls(MFTFlags.Unknown) + + +class PermissionFlags(enum.Enum): + ReadOnly = 0x1 + Hidden = 0x2 + System = 0x4 + Archive = 0x20 + ArchiveHidden = 0x22 + ArchiveSystem = 0x24 + ArchiveHiddenSystem = 0x26 + Device = 0x40 + Normal = 0x80 + Temporary = 0x100 + TempArchive = 0x120 + SparseFile = 0x200 + ReparsePoint = 0x400 + Compressed = 0x800 + Offline = 0x1000 + NotIndexed = 0x2000 + Encrypted = 0x4000 + Directory = 0x10000000 + IndexView = 0x20000000 + unknown = None + + @classmethod + def _missing_(cls, value): + return cls(PermissionFlags.unknown) + + +class MFTEntry(objects.StructType): + """This represents the base MFT Record""" + + def get_signature(self) -> str: + signature = self.Signature.cast('string', max_length = 4, encoding = 'latin-1') + return signature + + +class MFTFileName(objects.StructType): + """This represents an MFT $FILE_NAME Attribute""" + + def get_full_name(self) -> str: + output = self.Name.cast("string", + encoding = "utf16", + max_length = self.NameLength*2, + errors = "replace") + return output + + def get_file_namespace(self) -> str: + pass diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json new file mode 100644 index 000000000..e045ed0fc --- /dev/null +++ b/volatility3/framework/symbols/windows/mft.json @@ -0,0 +1,371 @@ +{ + "metadata": { + "producer": { + "version": "0.0.1", + "name": "kevthehermit-by-hand", + "comment": "Using structures defined in File System Forensic Analysis pg 353+", + "datetime": "2022-01-03T13:37:00" + }, + "format": "6.1.0" + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": true, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "int", + "size": 1, + "signed": false, + "endian": "little" + }, + "wchar": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + } + }, + "symbols": {}, + "enums": {}, + "user_types": { + "MFT_ENTRY": { + "fields": { + "Signature": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "UpdateSequenceOffset": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "NumFixupEntries": { + "offset": 6, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "LSN": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "SequenceValue": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "LinkCount": { + "offset": 18, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "FirstAttrOffset": { + "offset": 20, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "Flags": { + "offset": 22, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "RealSize": { + "offset": 24, + "type":{ + "kind": "base", + "name": "unsigned int" + } + }, + "AlocatedSize": { + "offset": 28, + "type":{ + "kind": "base", + "name": "unsigned int" + } + }, + "BaseReference": { + "offset": 32, + "type":{ + "kind": "base", + "name": "unsigned long long" + } + }, + "NextAttrID": { + "offset": 40, + "type":{ + "kind": "base", + "name": "unsigned short" + } + }, + "RecordNumber": { + "offset": 44, + "type":{ + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 1024 + },"ATTR_HEADER": { + "fields": { + "AttrType": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned int" + } + },"Length": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "NonResidentFlag": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned char" } + }, + "NameLength": { + "offset": 9, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "NameOffset": { + "offset": 10, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "Flags": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "AttributeID": { + "offset": 14, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 16 + },"RESIDENT_HEADER": { + "fields": { + "AttrSize": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned int" + } + },"AttrOffset": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "IndexFlag": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned short" } + } + }, + "kind": "struct", + "size": 8 + }, + "STANDARD_INFORMATION_ENTRY": { + "fields": { + "CreationTime": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ModifiedTime": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "UpdatedTime": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AccessedTime": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "flags": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 1024 + }, + "FILE_NAME_ENTRY": { + "fields": { + "ParentDirectory": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "CreationTime": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ModifiedTime": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "UpdatedTime": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AccessedTime": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "AllocatedFileSize": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "RealFileSize": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Flags": { + "offset": 56, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ReparseValue": { + "offset": 60, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "NameLength": { + "offset": 64, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "NameSpace": { + "offset": 65, + "type": { + "kind": "base", + "name": "unsigned char" + } + }, + "Name": { + "offset": 66, + "type": { + "count": 10, + "kind": "array", + "subtype": { + "kind": "base", + "name": "wchar" + } + } + } + }, + "kind": "struct", + "size": 1024 + } + } +} \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/mft.py b/volatility3/framework/symbols/windows/mft.py new file mode 100644 index 000000000..921d75cd8 --- /dev/null +++ b/volatility3/framework/symbols/windows/mft.py @@ -0,0 +1,15 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mft + + +class MFTIntermedSymbols(intermed.IntermediateSymbolTable): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.set_type_class('FILE_NAME_ENTRY', mft.MFTFileName) + self.set_type_class('MFT_ENTRY', mft.MFTEntry) From 793d08faf487c1d440bae016d8ca1e87766da7cc Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 9 Jan 2022 22:27:32 +0000 Subject: [PATCH 06/13] Add TimeLiner interface to MFTScan plugin --- volatility3/framework/plugins/windows/mftscan.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 484fcdb9a..616c0d738 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -14,11 +14,11 @@ from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols -from volatility3.plugins import yarascan +from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) -class MFTScan(interfaces.plugins.PluginInterface): +class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) @@ -121,6 +121,18 @@ class MFTScan(interfaces.plugins.PluginInterface): except exceptions.PagedInvalidAddressException: pass + def generate_timeline(self): + for row in self._generator(): + if row[-1] != 'N/A': + filename = row[-1] + created = f'File {row[-1]} Created' + updated = f'File {row[-1]} Updated' + modified = f'File {row[-1]} Modified' + accessed = f'File {row[-1]} Accessed' + yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) + yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) + yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) + yield (f'File {filename} accessed', timeliner.TimeLinerType.ACCESSED, row[10]) def run(self): return renderers.TreeGrid([ From 1c6cd0fb528b02e8f35ac65f1173241ff84dfe26 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 15:53:35 +0000 Subject: [PATCH 07/13] Move mftscan enums to ISF file. --- .../framework/plugins/windows/mftscan.py | 33 +++++--- .../symbols/windows/extensions/mft.py | 77 ------------------- .../framework/symbols/windows/mft.json | 70 ++++++++++++++++- 3 files changed, 93 insertions(+), 87 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 616c0d738..991191d5b 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -11,7 +11,6 @@ from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols.windows.extensions.mft import AttributeTypes, NameSpace, PermissionFlags, MFTFlags from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols from volatility3.plugins import timeliner, yarascan @@ -53,6 +52,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Get the Enums + attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") + namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") + mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") + permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): @@ -70,14 +75,20 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.debug(f"Attr Type: {attr_header.AttrType}") # If this is not a valid type then exit the loop - if not AttributeTypes(attr_header.AttrType).value: + if attr_header.AttrType not in attr_types.choices.values(): break # Offset past the headers to the attribute data attr_data_offset = offset+attr_base_offset+24 + + # MFT Flags determine the file type or dir + if mft_record.Flags in mft_flags.choices.values(): + mft_flag = mft_flags.lookup(mft_record.Flags) + else: + mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType == 0x10: + if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) yield 0, ( @@ -85,9 +96,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, - MFTFlags(mft_record.Flags).name, + mft_flag, renderers.NotApplicableValue(), - AttributeTypes(attr_header.AttrType).name, + attr_types.lookup(attr_header.AttrType), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -96,18 +107,22 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType == 0x30: + if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) file_name = attr_data.get_full_name() + if attr_data.Flags in permission_flags.choices.values(): + permissions = permission_flags.lookup(attr_data.Flags) + else: + permissions = hex(attr_data.Flags) yield 1, ( format_hints.Hex(attr_data_offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, - MFTFlags(mft_record.Flags).name, - PermissionFlags(attr_data.Flags).name, - AttributeTypes(attr_header.AttrType).name, + mft_flag, + permissions, + attr_types.lookup(attr_header.AttrType), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 09f6346cc..0713c969a 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -8,80 +8,6 @@ from volatility3.framework import exceptions, objects, renderers from volatility3.framework.objects import utility -class AttributeTypes(enum.Enum): - STANDARD_INFORMATION = 0x10 - ATTRIBUTE_LIST = 0x20 - FILE_NAME = 0x30 - OBJECT_ID = 0x40 - SECURITY_DESCRIPTOR = 0x50 - VOLUME_NAME = 0x60 - VOLUME_INFORMATION = 0x70 - DATA = 0x80 - INDEX_ROOT = 0x90 - INDEX_ALLOCATION = 0xa0 - BITMAP = 0xb0 - REPARSE_POINT = 0xc0 - EA_INFORMATION = 0xd0 - EA = 0xe0 - PROPERTY_SET = 0xf0 - LOGGED_UTILITY_STREAM = 0x100 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(AttributeTypes.Unknown) - -class NameSpace(enum.Enum): - POSIX = 0x0 - Win32 = 0x1 - DOS = 0x2 - Win32DOS = 0x3 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(NameSpace.Unknown) - - -class MFTFlags(enum.Enum): - Removed = 0x00 - File = 0x1 - Directory = 0x2 - DirInUse = 0x3 - Unknown = None - - @classmethod - def _missing_(cls, value): - return cls(MFTFlags.Unknown) - - -class PermissionFlags(enum.Enum): - ReadOnly = 0x1 - Hidden = 0x2 - System = 0x4 - Archive = 0x20 - ArchiveHidden = 0x22 - ArchiveSystem = 0x24 - ArchiveHiddenSystem = 0x26 - Device = 0x40 - Normal = 0x80 - Temporary = 0x100 - TempArchive = 0x120 - SparseFile = 0x200 - ReparsePoint = 0x400 - Compressed = 0x800 - Offline = 0x1000 - NotIndexed = 0x2000 - Encrypted = 0x4000 - Directory = 0x10000000 - IndexView = 0x20000000 - unknown = None - - @classmethod - def _missing_(cls, value): - return cls(PermissionFlags.unknown) - - class MFTEntry(objects.StructType): """This represents the base MFT Record""" @@ -99,6 +25,3 @@ class MFTFileName(objects.StructType): max_length = self.NameLength*2, errors = "replace") return output - - def get_file_namespace(self) -> str: - pass diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index e045ed0fc..a99b82e9e 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -53,7 +53,75 @@ } }, "symbols": {}, - "enums": {}, + "enums": { + "AttrTypeEnum": { + "base": "unsigned char", + "constants": { + "STANDARD_INFORMATION": 16, + "ATTRIBUTE_LIST": 32, + "FILE_NAME": 48, + "OBJECT_ID": 64, + "SECURITY_DESCRIPTOR": 80, + "VOLUME_NAME": 96, + "VOLUME_INFORMATION": 112, + "DATA": 128, + "INDEX_ROOT": 114, + "INDEX_ALLOCATION": 160, + "BITMAP": 176, + "REPARSE_POINT": 192, + "EA_INFORMATION": 208, + "EA": 224, + "PROPERTY_SET": 240, + "LOGGED_UTILITY_STREAM": 256 + }, + "size": 1 + }, + "NameSpaceEnum": { + "base":"unsigned char", + "constants": { + "POSIX": 0, + "Win32": 1, + "DOS": 2, + "Win32 DOS": 3 + }, + "size": 1 + }, + "MFTFlagsEnum": { + "base":"unsigned char", + "constants": { + "Removed": 0, + "File": 1, + "Directory": 2, + "DirInUse": 3 + }, + "size": 1 + }, + "PermissionFlagEnum": { + "base":"unsigned char", + "constants": { + "ReadOnly": 1, + "Hidden": 2, + "System": 4, + "Archive": 32, + "ArchiveHidden": 34, + "ArchiveSystem": 36, + "ArchiveHiddenSystem": 38, + "Device": 60, + "Normal": 128, + "Temporary": 256, + "TempArchive": 288, + "SparseFile": 512, + "ReparsePoint": 1024, + "Compressed": 2048, + "Offline": 4096, + "NotIndexed": 8192, + "Encrypted": 16384, + "Directory": 268435456, + "IndexView": 536870912 + }, + "size": 1 + } + }, "user_types": { "MFT_ENTRY": { "fields": { From a8f5b0381664b963a565161d88c64fc1b4053102 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 15:57:23 +0000 Subject: [PATCH 08/13] Apply yapf to mftscan plugin --- .../framework/plugins/windows/mftscan.py | 84 +++++++++---------- .../symbols/windows/extensions/mft.py | 10 +-- 2 files changed, 41 insertions(+), 53 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 991191d5b..070061b17 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -17,6 +17,7 @@ from volatility3.plugins import timeliner, yarascan vollog = logging.getLogger(__name__) + class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for MFT FILE objects present in a particular windows memory image.""" @@ -32,7 +33,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): version = (2, 0, 0)), ] - def _generator(self): layer = self.context.layers[self.config['primary']] @@ -40,12 +40,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) # Read in the Symbol File - symbol_table = MFTIntermedSymbols.create( - self.context, - self.config_path, - "windows", - "mft" - ) + symbol_table = MFTIntermedSymbols.create(self.context, self.config_path, "windows", "mft") # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" @@ -57,20 +52,26 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") - permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") - + permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + + "PermissionFlagEnum") + # Scan the layer for Raw MFT records and parse the fields - for offset, rule_name, name, value in layer.scan(context = self.context, scanner = yarascan.YaraScanner(rules = rules)): + for offset, rule_name, name, value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): try: - mft_record = self.context.object(mft_object, offset=offset, layer_name=layer.name) + mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while True: - attr_header = self.context.object(header_object, offset=offset+attr_base_offset, layer_name=layer.name) - attr_resident_header = self.context.object(header_object, offset=offset+attr_base_offset+16, layer_name=layer.name) + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) + attr_resident_header = self.context.object(header_object, + offset = offset + attr_base_offset + 16, + layer_name = layer.name) vollog.debug(f"Attr Type: {attr_header.AttrType}") @@ -79,17 +80,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset+attr_base_offset+24 + attr_data_offset = offset + attr_base_offset + 24 # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): mft_flag = mft_flags.lookup(mft_record.Flags) else: mft_flag = hex(mft_record.Flags) - + # Standard Information Attribute if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): - attr_data = self.context.object(si_object, offset=attr_data_offset, layer_name=layer.name) + attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name) yield 0, ( format_hints.Hex(attr_data_offset), @@ -108,28 +109,21 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # File Name Attribute if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): - attr_data = self.context.object(fn_object, offset=attr_data_offset, layer_name=layer.name) + attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name) file_name = attr_data.get_full_name() if attr_data.Flags in permission_flags.choices.values(): permissions = permission_flags.lookup(attr_data.Flags) else: permissions = hex(attr_data.Flags) - yield 1, ( - format_hints.Hex(attr_data_offset), - mft_record.get_signature(), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - permissions, - attr_types.lookup(attr_header.AttrType), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - file_name - ) - + yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(), + mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, + attr_types.lookup(attr_header.AttrType), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), file_name) + # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length @@ -151,16 +145,16 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): def run(self): return renderers.TreeGrid([ - ('Offset', format_hints.Hex), - ('Record Type', str), - ('Record Number', int), - ('Link Count', int), - ('MFT Type', str), - ('Permissions', str), - ('Attribute Type', str), - ('Created', datetime.datetime), - ('Modified', datetime.datetime), - ('Updated', datetime.datetime), - ('Accessed', datetime.datetime), - ('Filename', str), - ],self._generator()) + ('Offset', format_hints.Hex), + ('Record Type', str), + ('Record Number', int), + ('Link Count', int), + ('MFT Type', str), + ('Permissions', str), + ('Attribute Type', str), + ('Created', datetime.datetime), + ('Modified', datetime.datetime), + ('Updated', datetime.datetime), + ('Accessed', datetime.datetime), + ('Filename', str), + ], self._generator()) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 0713c969a..ba79b7c8b 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,10 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import enum - -from volatility3.framework import exceptions, objects, renderers -from volatility3.framework.objects import utility +from volatility3.framework import objects class MFTEntry(objects.StructType): @@ -20,8 +17,5 @@ class MFTFileName(objects.StructType): """This represents an MFT $FILE_NAME Attribute""" def get_full_name(self) -> str: - output = self.Name.cast("string", - encoding = "utf16", - max_length = self.NameLength*2, - errors = "replace") + output = self.Name.cast("string", encoding = "utf16", max_length = self.NameLength * 2, errors = "replace") return output From c5987a45d2362562329ca0e977ec87cf76babca9 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 20:54:22 +0000 Subject: [PATCH 09/13] Relative Offset MFT Header --- .../framework/plugins/windows/mftscan.py | 13 +++----- .../framework/symbols/windows/mft.json | 30 ++++++++++++++++++- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 070061b17..3d8d96221 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -44,13 +44,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Get the Enums attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") - namespave_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") + namespace_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "PermissionFlagEnum") @@ -69,9 +70,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): attr_header = self.context.object(header_object, offset = offset + attr_base_offset, layer_name = layer.name) - attr_resident_header = self.context.object(header_object, - offset = offset + attr_base_offset + 16, - layer_name = layer.name) vollog.debug(f"Attr Type: {attr_header.AttrType}") @@ -80,7 +78,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + 24 + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( + attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -134,10 +133,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for row in self._generator(): if row[-1] != 'N/A': filename = row[-1] - created = f'File {row[-1]} Created' - updated = f'File {row[-1]} Updated' - modified = f'File {row[-1]} Modified' - accessed = f'File {row[-1]} Accessed' yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index a99b82e9e..2470dcbd5 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -223,7 +223,35 @@ }, "kind": "struct", "size": 1024 - },"ATTR_HEADER": { + }, + "ATTRIBUTE": { + "fields":{ + "Attr_Header": { + "offset": 0, + "type": { + "kind": "struct", + "name": "mft!ATTR_HEADER" + } + }, + "Resident_Header": { + "offset": 16, + "type": { + "kind": "struct", + "name": "mft!RESIDENT_HEADER" + } + }, + "Attr_Data": { + "offset": 24, + "type": { + "kind": "struct", + "name": "mft!ATTR_HEADER" + } + } + }, + "kind": "struct", + "size": 96 + }, + "ATTR_HEADER": { "fields": { "AttrType": { "offset": 0, From 4aaba89d024a06a00760978b2f8bb7f099087c72 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 21:47:37 +0000 Subject: [PATCH 10/13] Unity timeliner output for mftscan --- .../framework/plugins/windows/mftscan.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 3d8d96221..03f269735 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -78,8 +78,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): break # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( - attribute_object).relative_child_offset("Attr_Data") + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type(attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -130,13 +129,18 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pass def generate_timeline(self): + print("MFT Timeline") for row in self._generator(): - if row[-1] != 'N/A': - filename = row[-1] - yield (f'File {filename} created', timeliner.TimeLinerType.CREATED, row[7]) - yield (f'File {filename} modified', timeliner.TimeLinerType.MODIFIED, row[8]) - yield (f'File {filename} updated', timeliner.TimeLinerType.CHANGED, row[9]) - yield (f'File {filename} accessed', timeliner.TimeLinerType.ACCESSED, row[10]) + _depth, row_data = row + + # Only Output FN Records + if row_data[6] == 'FILE_NAME': + filename = row_data[-1] + description = f"MFT FILE_NAME entry for {filename}" + yield (description, timeliner.TimeLinerType.CREATED, row_data[7]) + yield (description, timeliner.TimeLinerType.MODIFIED, row_data[8]) + yield (description, timeliner.TimeLinerType.CHANGED, row_data[9]) + yield (description, timeliner.TimeLinerType.ACCESSED, row_data[10]) def run(self): return renderers.TreeGrid([ From 32abab8733d697ed86e3a6208d9c03ce08233117 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 22:38:05 +0000 Subject: [PATCH 11/13] Remove debug print from mftscan --- volatility3/framework/plugins/windows/mftscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 03f269735..7cc57f111 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -129,7 +129,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): pass def generate_timeline(self): - print("MFT Timeline") for row in self._generator(): _depth, row_data = row From f31bc853c4d7eb05041c854f26704dea21656d0d Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sat, 15 Jan 2022 23:30:39 +0000 Subject: [PATCH 12/13] remove MFTIntermedSymbols --- .../framework/plugins/windows/mftscan.py | 43 +++++++++++-------- volatility3/framework/symbols/windows/mft.py | 15 ------- 2 files changed, 25 insertions(+), 33 deletions(-) delete mode 100644 volatility3/framework/symbols/windows/mft.py diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 7cc57f111..6487ffe97 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,13 +5,12 @@ import datetime import logging -from typing import Dict - from volatility3.framework import constants, renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework import exceptions from volatility3.framework.renderers import conversion, format_hints -from volatility3.framework.symbols.windows.mft import MFTIntermedSymbols +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mft from volatility3.plugins import timeliner, yarascan @@ -40,7 +39,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) # Read in the Symbol File - symbol_table = MFTIntermedSymbols.create(self.context, self.config_path, "windows", "mft") + symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, + config_path = self.config_path, + sub_path = "windows", + filename = "mft", + class_types = { + 'FILE_NAME_ENTRY': mft.MFTFileName, + 'MFT_ENTRY': mft.MFTEntry + }) # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" @@ -57,28 +63,25 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields - for offset, rule_name, name, value in layer.scan(context = self.context, - scanner = yarascan.YaraScanner(rules = rules)): + for offset, _rule_name, _name, _value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): try: mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) + # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while True: - attr_header = self.context.object(header_object, - offset = offset + attr_base_offset, - layer_name = layer.name) - + while attr_header.AttrType in attr_types.choices.values(): vollog.debug(f"Attr Type: {attr_header.AttrType}") - # If this is not a valid type then exit the loop - if attr_header.AttrType not in attr_types.choices.values(): - break - # Offset past the headers to the attribute data - attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type(attribute_object).relative_child_offset("Attr_Data") + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( + attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir if mft_record.Flags in mft_flags.choices.values(): @@ -124,9 +127,13 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Update the base offset to point to the next attribute attr_base_offset += attr_header.Length + # Get the next attribute + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) - except exceptions.PagedInvalidAddressException: - pass + except Exception as err: + vollog.debug(f'Error Parsing MFT Record: {err}') def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/symbols/windows/mft.py b/volatility3/framework/symbols/windows/mft.py deleted file mode 100644 index 921d75cd8..000000000 --- a/volatility3/framework/symbols/windows/mft.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows.extensions import mft - - -class MFTIntermedSymbols(intermed.IntermediateSymbolTable): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.set_type_class('FILE_NAME_ENTRY', mft.MFTFileName) - self.set_type_class('MFT_ENTRY', mft.MFTEntry) From e3a7ac566840ee052ffbefe3fb370e4142704706 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 16 Jan 2022 02:20:55 +0000 Subject: [PATCH 13/13] Use lookups on mft enums instead of choices --- .../framework/plugins/windows/mftscan.py | 39 +++++++++---------- .../framework/symbols/windows/mft.json | 22 +++++------ 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 6487ffe97..16f0c9c95 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -55,12 +55,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" - # Get the Enums - attr_types = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "AttrTypeEnum") - namespace_enum = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "NameSpaceEnum") - mft_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + "MFTFlagsEnum") - permission_flags = self.context.symbol_space.get_enumeration(symbol_table + constants.BANG + - "PermissionFlagEnum") # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan(context = self.context, @@ -74,23 +68,26 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) + # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr_header.AttrType in attr_types.choices.values(): - vollog.debug(f"Attr Type: {attr_header.AttrType}") + + while attr_header.AttrType.is_valid_choice: + vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") # Offset past the headers to the attribute data attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( attribute_object).relative_child_offset("Attr_Data") # MFT Flags determine the file type or dir - if mft_record.Flags in mft_flags.choices.values(): - mft_flag = mft_flags.lookup(mft_record.Flags) - else: + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + mft_flag = mft_record.Flags.lookup() + except ValueError: mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType == attr_types.choices.get('STANDARD_INFORMATION'): + if attr_header.AttrType.lookup() == 'STANDARD_INFORMATION': attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name) yield 0, ( @@ -100,7 +97,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_record.LinkCount, mft_flag, renderers.NotApplicableValue(), - attr_types.lookup(attr_header.AttrType), + attr_header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -109,17 +106,19 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # File Name Attribute - if attr_header.AttrType == attr_types.choices.get('FILE_NAME'): + if attr_header.AttrType.lookup() == 'FILE_NAME': attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name) file_name = attr_data.get_full_name() - if attr_data.Flags in permission_flags.choices.values(): - permissions = permission_flags.lookup(attr_data.Flags) - else: + + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + permissions = attr_data.Flags.lookup() + except ValueError: permissions = hex(attr_data.Flags) yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, - attr_types.lookup(attr_header.AttrType), + attr_header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -132,8 +131,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset = offset + attr_base_offset, layer_name = layer.name) - except Exception as err: - vollog.debug(f'Error Parsing MFT Record: {err}') + except exceptions.PagedInvalidAddressException: + pass def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index 2470dcbd5..b71be6444 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -181,8 +181,8 @@ "Flags": { "offset": 22, "type":{ - "kind": "base", - "name": "unsigned short" + "kind": "enum", + "name": "MFTFlagsEnum" } }, "RealSize": { @@ -256,8 +256,8 @@ "AttrType": { "offset": 0, "type": { - "kind": "base", - "name": "unsigned int" + "kind": "enum", + "name": "AttrTypeEnum" } },"Length": { "offset": 4, @@ -289,9 +289,9 @@ "Flags": { "offset": 12, "type": { - "kind": "base", - "name": "unsigned short" - } + "kind": "enum", + "name": "MFTFlagsEnum" + } }, "AttributeID": { "offset": 14, @@ -361,8 +361,8 @@ "flags": { "offset": 32, "type": { - "kind": "base", - "name": "unsigned short" + "kind": "enum", + "name": "PermissionFlagEnum" } } }, @@ -423,8 +423,8 @@ "Flags": { "offset": 56, "type": { - "kind": "base", - "name": "unsigned int" + "kind": "enum", + "name": "PermissionFlagEnum" } }, "ReparseValue": {