From 84401847ff50009c120dcbbfdf17868b86c2a01b Mon Sep 17 00:00:00 2001 From: Valentin Obst Date: Wed, 20 Dec 2023 18:38:07 +0100 Subject: [PATCH 001/348] add sanity check in Linux find_aslr to skip unrelocated init_task --- volatility3/framework/automagic/linux.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2eebcc2dc..fda71b766 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -156,6 +156,18 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): and init_task.state.cast("unsigned int") != 0 ): continue + elif init_task.active_mm.cast("long unsigned int") == module.get_symbol( + "init_mm" + ).address and init_task.tasks.next.cast( + "long unsigned int" + ) == init_task.tasks.prev.cast( + "long unsigned int" + ): + # The idle task steals `mm` from previously running task, i.e., + # `init_mm` is only used as long as no CPU has ever been idle. + # This catches cases where we found a fragment of the + # unrelocated ELF file instead of the running kernel. + continue # This we get for free aslr_shift = ( From 2920694643d6951a980fe32df0e3afe295e5d418 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 17:58:49 -0500 Subject: [PATCH 002/348] placeholder --- .../framework/plugins/windows/mftscan.py | 367 +++++++++--------- 1 file changed, 187 insertions(+), 180 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 9e6585345..0687c7796 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -20,6 +20,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._record_map = {} + @classmethod def get_requirements(cls): return [ @@ -33,8 +37,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] - def _generator(self): - layer = self.context.layers[self.config["primary"]] + def enumerate_mft_records(self, attr_callback): + phys_layer = self.context.layers[self.config["primary"]].config["memory_layer"] + + layer = self.context.layers[phys_layer] # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( @@ -47,87 +53,36 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): config_path=self.config_path, sub_path="windows", filename="mft", - class_types={"FILE_NAME_ENTRY": mft.MFTFileName, "MFT_ENTRY": mft.MFTEntry}, + class_types={"FILE_NAME_ENTRY": mft.MFTFileName, "MFT_ENTRY": mft.MFTEntry, "ATTRIBUTE": mft.MFTAttribute}, ) # get each of the individual Field Sets - mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" - si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" - fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + self.mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + self.attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" + self.si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + self.fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields - for offset, _rule_name, _name, _value in layer.scan( + for offset, _, _, _ in layer.scan( context=self.context, scanner=yarascan.YaraScanner(rules=rules) ): - with contextlib.suppress(exceptions.PagedInvalidAddressException): + with contextlib.suppress(exceptions.InvalidAddressException): mft_record = self.context.object( - mft_object, offset=offset, layer_name=layer.name + self.mft_object, offset=offset, layer_name=layer.name ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset attr = self.context.object( - attribute_object, + self.attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr.Attr_Header.AttrType.is_valid_choice: - vollog.debug(f"Attr Type: {attr.Attr_Header.AttrType.lookup()}") - - # MFT Flags determine the file type or dir - # If we don't have a valid enum, coerce to hex so we can keep the record - try: - mft_flag = mft_record.Flags.lookup() - except ValueError: - mft_flag = hex(mft_record.Flags) - - # Standard Information Attribute - if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": - attr_data = attr.Attr_Data.cast(si_object) - yield 0, ( - format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - renderers.NotApplicableValue(), - attr.Attr_Header.AttrType.lookup(), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - renderers.NotApplicableValue(), - ) - - # File Name Attribute - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - attr_data = attr.Attr_Data.cast(fn_object) - file_name = attr_data.get_full_name() - - # If we don't have a valid enum, coerce to hex so we can keep the record - try: - permissions = attr_data.Flags.lookup() - except ValueError: - permissions = hex(attr_data.Flags) - - yield 1, ( - format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - permissions, - attr.Attr_Header.AttrType.lookup(), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - file_name, - ) + for record in attr_callback(mft_record, attr): + yield record # If there's no advancement the loop will never end, so break it now if attr.Attr_Header.Length == 0: @@ -135,12 +90,69 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Update the base offset to point to the next attribute attr_base_offset += attr.Attr_Header.Length + # Get the next attribute attr = self.context.object( - attribute_object, + self.attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) + def parse_mft_records(self, mft_record, attr): + # MFT Flags determine the file type or dir + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + mft_flag = mft_record.Flags.lookup() + except ValueError: + mft_flag = hex(mft_record.Flags) + + # Standard Information Attribute + if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": + attr_data = attr.Attr_Data.cast(self.si_object) + yield 0, ( + format_hints.Hex(attr_data.vol.offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + mft_flag, + renderers.NotApplicableValue(), + attr.Attr_Header.AttrType.lookup(), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + renderers.NotApplicableValue(), + ) + + # File Name Attribute + elif attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(self.fn_object) + file_name = attr_data.get_full_name() + + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + permissions = attr_data.Flags.lookup() + except ValueError: + permissions = hex(attr_data.Flags) + + yield 1, ( + format_hints.Hex(attr_data.vol.offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + mft_flag, + permissions, + attr.Attr_Header.AttrType.lookup(), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + file_name, + ) + + def _generator(self): + for record in self.enumerate_mft_records(self.parse_mft_records): + yield record + def generate_timeline(self): for row in self._generator(): _depth, row_data = row @@ -173,127 +185,98 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self._generator(), ) - -class ADS(interfaces.plugins.PluginInterface): +class ADS(MFTScan): """Scans for Alternate Data Stream""" _required_framework_version = (2, 0, 0) - @classmethod - def get_requirements(cls): - return [ - requirements.TranslationLayerRequirement( - name="primary", - description="Memory layer for the kernel", - architectures=["Intel32", "Intel64"], - ), - requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) - ), - ] + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # which DATA attribute should be displayed + self._display_first_data = False + + def _parse_data_record(self, mft_record, attr): + if attr.Attr_Header.NonResidentFlag: + return + + # regular $DATA + if self._display_first_data: + content = attr.get_resident_filecontent() + if content: + content = format_hints.HexBytes(content) + else: + content = renderers.NotAvailableValue() + + yield 0, ( + format_hints.Hex(self._record_map[mft_record.RecordNumber][2]), + mft_record.get_signature(), + mft_record.RecordNumber, + attr.Attr_Header.AttrType.lookup(), + self._record_map[mft_record.RecordNumber][0], + content, + ) + + # ADS $DATA + elif attr.Attr_Header.NameLength > 0: + ads_name = attr.get_resident_filename() + if not ads_name: + ads_name = renderers.NotAvailableValue() + + content = attr.get_resident_filecontent() + if content: + content = format_hints.HexBytes(content) + else: + content = renderers.NotAvailableValue() + + yield 0, ( + format_hints.Hex(self._record_map[mft_record.RecordNumber][2]), + mft_record.get_signature(), + mft_record.RecordNumber, + attr.Attr_Header.AttrType.lookup(), + self._record_map[mft_record.RecordNumber][0], + ads_name, + content, + ) + + def parse_data_records(self, mft_record, attr): + rec_num = mft_record.RecordNumber + if rec_num not in self._record_map: + # file name, DATA count, offset + self._record_map[rec_num] = [renderers.NotAvailableValue(), 0, None] + + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(self.fn_object) + rec_name = attr_data.get_full_name() + self._record_map[rec_num][0] = rec_name + elif attr.Attr_Header.AttrType.lookup() == "DATA": + # first data + self._record_map[rec_num][2] = attr.Attr_Data.vol.offset + + display_data = False + + # first DATA attribute of this record + if self._record_map[rec_num][1] == 0: + if self._display_first_data: + display_data = True + else: + self._record_map[rec_num][1] = 1 + + # at the second DATA attribute of this record + elif not self._display_first_data: + display_data = True + + if display_data: + for record in self._parse_data_record( + mft_record, attr + ): + yield record def _generator(self): - layer = self.context.layers[self.config["primary"]] - - # Yara Rule to scan for MFT Header Signatures - rules = yarascan.YaraScan.process_yara_options( - {"yara_string": "/FILE0|FILE\\*|BAAD/"} - ) - - # Read in the Symbol File - symbol_table = intermed.IntermediateSymbolTable.create( - context=self.context, - config_path=self.config_path, - sub_path="windows", - filename="mft", - class_types={ - "MFT_ENTRY": mft.MFTEntry, - "FILE_NAME_ENTRY": mft.MFTFileName, - "ATTRIBUTE": mft.MFTAttribute, - }, - ) - - # get each of the individual Field Sets - mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" - fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" - - # Scan the layer for Raw MFT records and parse the fields - for offset, _rule_name, _name, _value in layer.scan( - context=self.context, scanner=yarascan.YaraScanner(rules=rules) + for record in self.enumerate_mft_records( + self.parse_data_records ): - with contextlib.suppress(exceptions.PagedInvalidAddressException): - mft_record = self.context.object( - mft_object, offset=offset, layer_name=layer.name - ) - # We will update this on each pass in the next loop and use it as the new offset. - attr_base_offset = mft_record.FirstAttrOffset - - attr = self.context.object( - attribute_object, - offset=offset + attr_base_offset, - layer_name=layer.name, - ) - - # There is no field that has a count of Attributes - # Keep Attempting to read attributes until we get an invalid attr.AttrType - is_ads = False - file_name = renderers.NotAvailableValue - # The First $DATA Attr is the 'principal' file itself not the ADS - while attr.Attr_Header.AttrType.is_valid_choice: - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - attr_data = attr.Attr_Data.cast(fn_object) - file_name = attr_data.get_full_name() - if attr.Attr_Header.AttrType.lookup() == "DATA": - if is_ads: - if not attr.Attr_Header.NonResidentFlag: - # Resident files are the most interesting. - if attr.Attr_Header.NameLength > 0: - ads_name = attr.get_resident_filename() - if not ads_name: - ads_name = renderers.NotAvailableValue - - content = attr.get_resident_filecontent() - if content: - # Preparing for Disassembly - disasm = interfaces.renderers.BaseAbsentValue - architecture = layer.metadata.get( - "architecture", None - ) - if architecture: - disasm = interfaces.renderers.Disassembly( - content, 0, architecture.lower() - ) - content = format_hints.HexBytes(content) - else: - content = renderers.NotAvailableValue() - disasm = interfaces.renderers.BaseAbsentValue() - - yield 0, ( - format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), - mft_record.RecordNumber, - attr.Attr_Header.AttrType.lookup(), - file_name, - ads_name, - content, - disasm, - ) - else: - is_ads = True - - # If there's no advancement the loop will never end, so break it now - if attr.Attr_Header.Length == 0: - break - - # Update the base offset to point to the next attribute - attr_base_offset += attr.Attr_Header.Length - # Get the next attribute - attr = self.context.object( - attribute_object, - offset=offset + attr_base_offset, - layer_name=layer.name, - ) + yield record def run(self): return renderers.TreeGrid( @@ -305,7 +288,31 @@ class ADS(interfaces.plugins.PluginInterface): ("Filename", str), ("ADS Filename", str), ("Hexdump", format_hints.HexBytes), - ("Disasm", interfaces.renderers.Disassembly), ], self._generator(), ) + +class ResidentData(ADS): + """Scans for Alternate Data Stream""" + + _required_framework_version = (2, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # which DATA attribute should be displayed + self._display_first_data = True + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Record Type", str), + ("Record Number", int), + ("MFT Type", str), + ("Filename", str), + ("Hexdump", format_hints.HexBytes), + ], + self._generator(), + ) + From 8926823199e35cd26858f6782943f4ec4e53e20e Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 18:08:08 -0500 Subject: [PATCH 003/348] Format fixes --- .../framework/plugins/windows/mftscan.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 0687c7796..d6f4684ec 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -53,7 +53,11 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): config_path=self.config_path, sub_path="windows", filename="mft", - class_types={"FILE_NAME_ENTRY": mft.MFTFileName, "MFT_ENTRY": mft.MFTEntry, "ATTRIBUTE": mft.MFTAttribute}, + class_types={ + "FILE_NAME_ENTRY": mft.MFTFileName, + "MFT_ENTRY": mft.MFTEntry, + "ATTRIBUTE": mft.MFTAttribute + }, ) # get each of the individual Field Sets @@ -185,6 +189,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self._generator(), ) + class ADS(MFTScan): """Scans for Alternate Data Stream""" @@ -242,7 +247,7 @@ class ADS(MFTScan): def parse_data_records(self, mft_record, attr): rec_num = mft_record.RecordNumber if rec_num not in self._record_map: - # file name, DATA count, offset + # file name, DATA count, offset self._record_map[rec_num] = [renderers.NotAvailableValue(), 0, None] if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": @@ -267,15 +272,11 @@ class ADS(MFTScan): display_data = True if display_data: - for record in self._parse_data_record( - mft_record, attr - ): + for record in self._parse_data_record(mft_record, attr): yield record def _generator(self): - for record in self.enumerate_mft_records( - self.parse_data_records - ): + for record in self.enumerate_mft_records(self.parse_data_records): yield record def run(self): @@ -292,6 +293,7 @@ class ADS(MFTScan): self._generator(), ) + class ResidentData(ADS): """Scans for Alternate Data Stream""" @@ -315,4 +317,3 @@ class ResidentData(ADS): ], self._generator(), ) - From 105b4bab25511dc8f5b2461f5f2a53422b55dafa Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 18:09:18 -0500 Subject: [PATCH 004/348] Format fixes --- volatility3/framework/plugins/windows/mftscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index d6f4684ec..540ab531c 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -56,7 +56,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): class_types={ "FILE_NAME_ENTRY": mft.MFTFileName, "MFT_ENTRY": mft.MFTEntry, - "ATTRIBUTE": mft.MFTAttribute + "ATTRIBUTE": mft.MFTAttribute, }, ) From 9b8eea015a4544752805f0dcfa2bde33b33e3084 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Thu, 18 Jul 2024 11:57:13 -0500 Subject: [PATCH 005/348] Address feedback --- .../framework/plugins/windows/mftscan.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 540ab531c..e2dfa12a3 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,6 +5,8 @@ import contextlib import datetime import logging +from typing import Generator, Iterable + from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints @@ -23,6 +25,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._record_map = {} + self.mft_object = None + self.attribute_object = None + self.si_object = None + self.fn_object = None @classmethod def get_requirements(cls): @@ -201,12 +207,17 @@ class ADS(MFTScan): # which DATA attribute should be displayed self._display_first_data = False - def _parse_data_record(self, mft_record, attr): + def _parse_data_record( + self, + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + ) -> Generator[Iterable, None, None]: + # we only care about resident data if attr.Attr_Header.NonResidentFlag: return # regular $DATA - if self._display_first_data: + elif self._display_first_data: content = attr.get_resident_filecontent() if content: content = format_hints.HexBytes(content) @@ -244,7 +255,11 @@ class ADS(MFTScan): content, ) - def parse_data_records(self, mft_record, attr): + def parse_data_records( + self, + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + ) -> Generator[Iterable, None, None]: rec_num = mft_record.RecordNumber if rec_num not in self._record_map: # file name, DATA count, offset From 2c73d8812a24eb8e34b84cfc6be61bf06a0c18b5 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 21 Jul 2024 08:55:27 -0500 Subject: [PATCH 006/348] Split layer gathering and add KeyError checks --- volatility3/framework/plugins/windows/mftscan.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index e2dfa12a3..6a4033a6f 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -44,7 +44,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ] def enumerate_mft_records(self, attr_callback): - phys_layer = self.context.layers[self.config["primary"]].config["memory_layer"] + try: + primary = self.context.layers[self.config["primary"]] + except KeyError: + vollog.error("Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue.") + return + + try: + phys_layer = primary.config["memory_layer"] + except KeyError: + vollog.error("Unable to obtain memory layer from primary layer. Please file a bug on GitHub about this issue.") + return layer = self.context.layers[phys_layer] From a8c6db2fc4b7d7a618477639e64d2c554c8961b2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 21 Jul 2024 11:56:52 -0500 Subject: [PATCH 007/348] Formatting fixes --- volatility3/framework/plugins/windows/mftscan.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 6a4033a6f..ec8dd5fe8 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -47,13 +47,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): try: primary = self.context.layers[self.config["primary"]] except KeyError: - vollog.error("Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue.") + vollog.error( + "Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue." + ) return try: phys_layer = primary.config["memory_layer"] except KeyError: - vollog.error("Unable to obtain memory layer from primary layer. Please file a bug on GitHub about this issue.") + vollog.error( + "Unable to obtain memory layer from primary layer. Please file a bug on GitHub about this issue." + ) return layer = self.context.layers[phys_layer] From dea0e156890e620ef78aeeadce8139a81e8e9042 Mon Sep 17 00:00:00 2001 From: atcuno Date: Mon, 22 Jul 2024 16:27:05 -0500 Subject: [PATCH 008/348] Address feedback --- .../framework/plugins/windows/mftscan.py | 338 +++++++++++------- 1 file changed, 215 insertions(+), 123 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index ec8dd5fe8..0d858cec1 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,7 +5,7 @@ import contextlib import datetime import logging -from typing import Generator, Iterable +from typing import Generator, Iterable, Dict, Tuple from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -22,13 +22,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._record_map = {} - self.mft_object = None - self.attribute_object = None - self.si_object = None - self.fn_object = None + _version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -43,9 +37,15 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] - def enumerate_mft_records(self, attr_callback): + @staticmethod + def enumerate_mft_records( + context: interfaces.context.ContextInterface, + config: interfaces.configuration.HierarchicalDict, + config_path: str, + attr_callback + ) -> interfaces.objects.ObjectInterface: try: - primary = self.context.layers[self.config["primary"]] + primary = context.layers[config["primary"]] except KeyError: vollog.error( "Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue." @@ -60,7 +60,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) return - layer = self.context.layers[phys_layer] + layer = context.layers[phys_layer] # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( @@ -69,8 +69,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Read in the Symbol File symbol_table = intermed.IntermediateSymbolTable.create( - context=self.context, - config_path=self.config_path, + context=context, + config_path=config_path, sub_path="windows", filename="mft", class_types={ @@ -81,23 +81,21 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # get each of the individual Field Sets - self.mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - self.attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" - self.si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" - self.fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" # Scan the layer for Raw MFT records and parse the fields for offset, _, _, _ in layer.scan( - context=self.context, scanner=yarascan.YaraScanner(rules=rules) + context=context, scanner=yarascan.YaraScanner(rules=rules) ): with contextlib.suppress(exceptions.InvalidAddressException): - mft_record = self.context.object( - self.mft_object, offset=offset, layer_name=layer.name + mft_record = context.object( + mft_object, offset=offset, layer_name=layer.name ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - attr = self.context.object( - self.attribute_object, + attr = context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -105,7 +103,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while attr.Attr_Header.AttrType.is_valid_choice: - for record in attr_callback(mft_record, attr): + for record in attr_callback(mft_record, attr, symbol_table): yield record # If there's no advancement the loop will never end, so break it now @@ -115,13 +113,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Update the base offset to point to the next attribute attr_base_offset += attr.Attr_Header.Length # Get the next attribute - attr = self.context.object( - self.attribute_object, + attr = context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) - def parse_mft_records(self, mft_record, attr): + @staticmethod + def parse_mft_records(mft_record, attr, symbol_table): # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record try: @@ -131,7 +130,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Standard Information Attribute if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": - attr_data = attr.Attr_Data.cast(self.si_object) + si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + attr_data = attr.Attr_Data.cast(si_object) yield 0, ( format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), @@ -149,7 +149,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # File Name Attribute elif attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - attr_data = attr.Attr_Data.cast(self.fn_object) + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() # If we don't have a valid enum, coerce to hex so we can keep the record @@ -173,8 +175,114 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_name, ) + @staticmethod + def parse_data_record( + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + record_map: Dict[int, Tuple[str, int, int]], + return_first_record: bool, + ) -> Generator[Iterable, None, None]: + """ + Returns the parsed data from a MFT record + """ + # we only care about resident data + if attr.Attr_Header.NonResidentFlag: + return + + content = attr.get_resident_filecontent() + if content: + content = format_hints.HexBytes(content) + else: + content = renderers.NotAvailableValue() + + # past the first $DATA record, attempt to get the ADS name + # NotApplicableValue = 1st Data + # NotAvailableValue = > 1st Data, but name was not parsable + ads_name = renderers.NotApplicableValue() + if not return_first_record and attr.Attr_Header.NameLength > 0: + ads_name = attr.get_resident_filename() + if not ads_name: + ads_name = renderers.NotAvailableValue() + + yield ( + format_hints.Hex(record_map[mft_record.RecordNumber][2]), + mft_record.get_signature(), + mft_record.RecordNumber, + attr.Attr_Header.AttrType.lookup(), + record_map[mft_record.RecordNumber][0], + ads_name, + content, + ) + + @classmethod + def _do_parse_data_records( + cls, + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table, + record_map: Dict[int, Tuple[str, int, int]], + return_first_record: bool, + ) -> Generator[Iterable, None, None]: + """ + Parses DATA records while maintaining the FILE_NAME association + from previous parsing of the record + Suports returning the first/main $DATA as well as however many + ADS records a file might have + """ + rec_num = mft_record.RecordNumber + if rec_num not in record_map: + # file name, DATA count, offset + record_map[rec_num] = [renderers.NotAvailableValue(), 0, None] + + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + attr_data = attr.Attr_Data.cast(fn_object) + rec_name = attr_data.get_full_name() + record_map[rec_num][0] = rec_name + elif attr.Attr_Header.AttrType.lookup() == "DATA": + # first data + record_map[rec_num][2] = attr.Attr_Data.vol.offset + + display_data = False + + # first DATA attribute of this record + if record_map[rec_num][1] == 0 and return_first_record: + if return_first_record: + display_data = True + else: + record_map[rec_num][1] = 1 + + # at the second DATA attribute of this record + elif not return_first_record: + display_data = True + + if display_data: + for record in cls.parse_data_record( + mft_record, attr, record_map, return_first_record + ): + yield record + + @classmethod + def parse_data_records( + cls, + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table, + return_first_record: bool, + ): + """ + Callback for parsing data records through enumerate_mft_records + """ + record_map = {} + for record in cls._do_parse_data_records( + mft_record, attr, symbol_table, record_map, return_first_record + ): + yield record + def _generator(self): - for record in self.enumerate_mft_records(self.parse_mft_records): + for record in self.enumerate_mft_records( + self.context, self.config, self.config_path, self.parse_mft_records + ): yield record def generate_timeline(self): @@ -210,103 +318,53 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) -class ADS(MFTScan): +class ADS(interfaces.plugins.PluginInterface): """Scans for Alternate Data Stream""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 7, 0) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + _version = (1, 0, 0) - # which DATA attribute should be displayed - self._display_first_data = False + @classmethod + def get_requirements(cls): + return [ + requirements.PluginRequirement( + name="MFTScan", plugin=MFTScan, version=(2, 0, 0) + ), + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + ] - def _parse_data_record( - self, + @staticmethod + def parse_ads_data_records( mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, - ) -> Generator[Iterable, None, None]: - # we only care about resident data - if attr.Attr_Header.NonResidentFlag: - return - - # regular $DATA - elif self._display_first_data: - content = attr.get_resident_filecontent() - if content: - content = format_hints.HexBytes(content) - else: - content = renderers.NotAvailableValue() - - yield 0, ( - format_hints.Hex(self._record_map[mft_record.RecordNumber][2]), - mft_record.get_signature(), - mft_record.RecordNumber, - attr.Attr_Header.AttrType.lookup(), - self._record_map[mft_record.RecordNumber][0], - content, - ) - - # ADS $DATA - elif attr.Attr_Header.NameLength > 0: - ads_name = attr.get_resident_filename() - if not ads_name: - ads_name = renderers.NotAvailableValue() - - content = attr.get_resident_filecontent() - if content: - content = format_hints.HexBytes(content) - else: - content = renderers.NotAvailableValue() - - yield 0, ( - format_hints.Hex(self._record_map[mft_record.RecordNumber][2]), - mft_record.get_signature(), - mft_record.RecordNumber, - attr.Attr_Header.AttrType.lookup(), - self._record_map[mft_record.RecordNumber][0], - ads_name, - content, - ) - - def parse_data_records( - self, - mft_record: interfaces.objects.ObjectInterface, - attr: interfaces.objects.ObjectInterface, - ) -> Generator[Iterable, None, None]: - rec_num = mft_record.RecordNumber - if rec_num not in self._record_map: - # file name, DATA count, offset - self._record_map[rec_num] = [renderers.NotAvailableValue(), 0, None] - - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - attr_data = attr.Attr_Data.cast(self.fn_object) - rec_name = attr_data.get_full_name() - self._record_map[rec_num][0] = rec_name - elif attr.Attr_Header.AttrType.lookup() == "DATA": - # first data - self._record_map[rec_num][2] = attr.Attr_Data.vol.offset - - display_data = False - - # first DATA attribute of this record - if self._record_map[rec_num][1] == 0: - if self._display_first_data: - display_data = True - else: - self._record_map[rec_num][1] = 1 - - # at the second DATA attribute of this record - elif not self._display_first_data: - display_data = True - - if display_data: - for record in self._parse_data_record(mft_record, attr): - yield record + symbol_table, + ): + return MFTScan.parse_data_records(mft_record, attr, symbol_table, False) def _generator(self): - for record in self.enumerate_mft_records(self.parse_data_records): - yield record + for ( + offset, + rec_type, + rec_num, + attr_type, + file_name, + ads_name, + content, + ) in MFTScan.enumerate_mft_records( + self.context, self.config, self.config_path, self.parse_ads_data_records + ): + yield ( + 0, + (offset, rec_type, rec_num, attr_type, file_name, ads_name, content), + ) def run(self): return renderers.TreeGrid( @@ -323,16 +381,50 @@ class ADS(MFTScan): ) -class ResidentData(ADS): +class ResidentData(interfaces.plugins.PluginInterface): """Scans for Alternate Data Stream""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 7, 0) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + _version = (1, 0, 0) - # which DATA attribute should be displayed - self._display_first_data = True + @classmethod + def get_requirements(cls): + return [ + requirements.PluginRequirement( + name="MFTScan", plugin=MFTScan, version=(2, 0, 0) + ), + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + ] + + @staticmethod + def parse_first_data_records( + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table, + ): + return MFTScan.parse_data_records(mft_record, attr, symbol_table, True) + + def _generator(self): + for ( + offset, + rec_type, + rec_num, + attr_type, + file_name, + _, + content, + ) in MFTScan.enumerate_mft_records( + self.context, self.config, self.config_path, self.parse_first_data_records + ): + yield (0, (offset, rec_type, rec_num, attr_type, file_name, content)) def run(self): return renderers.TreeGrid( From da843b0367b7353274626f79d09ddb4377a9148b Mon Sep 17 00:00:00 2001 From: atcuno Date: Mon, 22 Jul 2024 16:27:39 -0500 Subject: [PATCH 009/348] Address feedback --- volatility3/framework/plugins/windows/mftscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 0d858cec1..6a2aac440 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -42,7 +42,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): context: interfaces.context.ContextInterface, config: interfaces.configuration.HierarchicalDict, config_path: str, - attr_callback + attr_callback, ) -> interfaces.objects.ObjectInterface: try: primary = context.layers[config["primary"]] From 33716be15b4088e7743bc938f2c0e83ab8b5ea18 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Wed, 31 Jul 2024 13:56:38 -0500 Subject: [PATCH 010/348] Rework for proper ADS recovery. Memory OOM issues --- .../framework/plugins/windows/mftscan.py | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 6a2aac440..430f3fc02 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -84,8 +84,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_object = symbol_table + constants.BANG + "MFT_ENTRY" attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" + record_map = {} + # Scan the layer for Raw MFT records and parse the fields - for offset, _, _, _ in layer.scan( + for offset, _rule_name, _name, _value in layer.scan( context=context, scanner=yarascan.YaraScanner(rules=rules) ): with contextlib.suppress(exceptions.InvalidAddressException): @@ -103,7 +105,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while attr.Attr_Header.AttrType.is_valid_choice: - for record in attr_callback(mft_record, attr, symbol_table): + for record in attr_callback(record_map, mft_record, attr, symbol_table): yield record # If there's no advancement the loop will never end, so break it now @@ -120,7 +122,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) @staticmethod - def parse_mft_records(mft_record, attr, symbol_table): + def parse_mft_records(record_map, mft_record, attr, symbol_table): # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record try: @@ -189,21 +191,27 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if attr.Attr_Header.NonResidentFlag: return + # we aren't looking ADS when we want the first data record + if return_first_record: + ads_name = renderers.NotApplicableValue() + + # skip records without a name if we want ADS entries + elif attr.Attr_Header.NameLength == 0: + return + + else: + # past the first $DATA record, attempt to get the ADS name + # NotAvailableValue = > 1st Data, but name was not parsable + ads_name = attr.get_resident_filename() + if not ads_name: + ads_name = renderers.NotAvailableValue() + content = attr.get_resident_filecontent() if content: content = format_hints.HexBytes(content) else: content = renderers.NotAvailableValue() - # past the first $DATA record, attempt to get the ADS name - # NotApplicableValue = 1st Data - # NotAvailableValue = > 1st Data, but name was not parsable - ads_name = renderers.NotApplicableValue() - if not return_first_record and attr.Attr_Header.NameLength > 0: - ads_name = attr.get_resident_filename() - if not ads_name: - ads_name = renderers.NotAvailableValue() - yield ( format_hints.Hex(record_map[mft_record.RecordNumber][2]), mft_record.get_signature(), @@ -246,14 +254,15 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): display_data = False # first DATA attribute of this record - if record_map[rec_num][1] == 0 and return_first_record: + if record_map[rec_num][1] == 0: if return_first_record: display_data = True - else: - record_map[rec_num][1] = 1 + + record_map[rec_num][1] = 1 # at the second DATA attribute of this record - elif not return_first_record: + elif record_map[rec_num][1] == 1 and not return_first_record: + print("at second record") display_data = True if display_data: @@ -265,6 +274,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def parse_data_records( cls, + record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, symbol_table, @@ -273,7 +283,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ Callback for parsing data records through enumerate_mft_records """ - record_map = {} for record in cls._do_parse_data_records( mft_record, attr, symbol_table, record_map, return_first_record ): @@ -343,11 +352,12 @@ class ADS(interfaces.plugins.PluginInterface): @staticmethod def parse_ads_data_records( + record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, symbol_table, ): - return MFTScan.parse_data_records(mft_record, attr, symbol_table, False) + return MFTScan.parse_data_records(record_map, mft_record, attr, symbol_table, False) def _generator(self): for ( @@ -382,7 +392,7 @@ class ADS(interfaces.plugins.PluginInterface): class ResidentData(interfaces.plugins.PluginInterface): - """Scans for Alternate Data Stream""" + """Scans for MFT Records with Resident Data""" _required_framework_version = (2, 7, 0) @@ -406,11 +416,12 @@ class ResidentData(interfaces.plugins.PluginInterface): @staticmethod def parse_first_data_records( + record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, symbol_table, ): - return MFTScan.parse_data_records(mft_record, attr, symbol_table, True) + return MFTScan.parse_data_records(record_map, mft_record, attr, symbol_table, True) def _generator(self): for ( From da41124b1cb936ab168555568cf17512f719b6b4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 14:20:02 -0500 Subject: [PATCH 011/348] Fix formatting with black --- volatility3/framework/plugins/windows/mftscan.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 430f3fc02..2929b0522 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -105,7 +105,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while attr.Attr_Header.AttrType.is_valid_choice: - for record in attr_callback(record_map, mft_record, attr, symbol_table): + for record in attr_callback( + record_map, mft_record, attr, symbol_table + ): yield record # If there's no advancement the loop will never end, so break it now @@ -357,7 +359,9 @@ class ADS(interfaces.plugins.PluginInterface): attr: interfaces.objects.ObjectInterface, symbol_table, ): - return MFTScan.parse_data_records(record_map, mft_record, attr, symbol_table, False) + return MFTScan.parse_data_records( + record_map, mft_record, attr, symbol_table, False + ) def _generator(self): for ( @@ -421,7 +425,9 @@ class ResidentData(interfaces.plugins.PluginInterface): attr: interfaces.objects.ObjectInterface, symbol_table, ): - return MFTScan.parse_data_records(record_map, mft_record, attr, symbol_table, True) + return MFTScan.parse_data_records( + record_map, mft_record, attr, symbol_table, True + ) def _generator(self): for ( From 685dc97298719b9bcfb24d5a020b7d4550580f4c Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 30 Sep 2024 18:11:23 -0500 Subject: [PATCH 012/348] fix resident data bug for duplicate mft record numbers --- .../framework/plugins/windows/mftscan.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 2929b0522..16425ca22 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -215,11 +215,11 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): content = renderers.NotAvailableValue() yield ( - format_hints.Hex(record_map[mft_record.RecordNumber][2]), + format_hints.Hex(record_map[mft_record.vol.offset][2]), mft_record.get_signature(), mft_record.RecordNumber, attr.Attr_Header.AttrType.lookup(), - record_map[mft_record.RecordNumber][0], + record_map[mft_record.vol.offset][0], ads_name, content, ) @@ -239,31 +239,29 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Suports returning the first/main $DATA as well as however many ADS records a file might have """ - rec_num = mft_record.RecordNumber - if rec_num not in record_map: + if mft_record.vol.offset not in record_map: # file name, DATA count, offset - record_map[rec_num] = [renderers.NotAvailableValue(), 0, None] - + record_map[mft_record.vol.offset] = [renderers.NotAvailableValue(), 0, None] if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" attr_data = attr.Attr_Data.cast(fn_object) rec_name = attr_data.get_full_name() - record_map[rec_num][0] = rec_name + record_map[mft_record.vol.offset][0] = rec_name elif attr.Attr_Header.AttrType.lookup() == "DATA": # first data - record_map[rec_num][2] = attr.Attr_Data.vol.offset + record_map[mft_record.vol.offset][2] = attr.Attr_Data.vol.offset display_data = False # first DATA attribute of this record - if record_map[rec_num][1] == 0: + if record_map[mft_record.vol.offset][1] == 0: if return_first_record: display_data = True - record_map[rec_num][1] = 1 + record_map[mft_record.vol.offset][1] = 1 # at the second DATA attribute of this record - elif record_map[rec_num][1] == 1 and not return_first_record: + elif record_map[mft_record.vol.offset][1] == 1 and not return_first_record: print("at second record") display_data = True From 79f81f0af9193878f0290d0ae05d5178e71502c6 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 15 Oct 2024 14:11:27 -0500 Subject: [PATCH 013/348] Address feedback --- .../framework/plugins/windows/mftscan.py | 37 +++++-------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 16425ca22..94dce8c4d 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -40,12 +40,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @staticmethod def enumerate_mft_records( context: interfaces.context.ContextInterface, - config: interfaces.configuration.HierarchicalDict, config_path: str, + primary_layer_name: str, attr_callback, ) -> interfaces.objects.ObjectInterface: try: - primary = context.layers[config["primary"]] + primary = context.layers[primary_layer_name] except KeyError: vollog.error( "Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue." @@ -105,10 +105,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while attr.Attr_Header.AttrType.is_valid_choice: - for record in attr_callback( + yield from attr_callback( record_map, mft_record, attr, symbol_table - ): - yield record + ) # If there's no advancement the loop will never end, so break it now if attr.Attr_Header.Length == 0: @@ -225,12 +224,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) @classmethod - def _do_parse_data_records( + def parse_data_records( cls, + record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, symbol_table, - record_map: Dict[int, Tuple[str, int, int]], return_first_record: bool, ) -> Generator[Iterable, None, None]: """ @@ -262,7 +261,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # at the second DATA attribute of this record elif record_map[mft_record.vol.offset][1] == 1 and not return_first_record: - print("at second record") display_data = True if display_data: @@ -271,26 +269,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ): yield record - @classmethod - def parse_data_records( - cls, - record_map: Dict[int, Tuple[str, int, int]], - mft_record: interfaces.objects.ObjectInterface, - attr: interfaces.objects.ObjectInterface, - symbol_table, - return_first_record: bool, - ): - """ - Callback for parsing data records through enumerate_mft_records - """ - for record in cls._do_parse_data_records( - mft_record, attr, symbol_table, record_map, return_first_record - ): - yield record - def _generator(self): for record in self.enumerate_mft_records( - self.context, self.config, self.config_path, self.parse_mft_records + self.context, self.config_path, self.config["primary"], self.parse_mft_records ): yield record @@ -371,7 +352,7 @@ class ADS(interfaces.plugins.PluginInterface): ads_name, content, ) in MFTScan.enumerate_mft_records( - self.context, self.config, self.config_path, self.parse_ads_data_records + self.context, self.config_path, self.config["primary"], self.parse_ads_data_records ): yield ( 0, @@ -437,7 +418,7 @@ class ResidentData(interfaces.plugins.PluginInterface): _, content, ) in MFTScan.enumerate_mft_records( - self.context, self.config, self.config_path, self.parse_first_data_records + self.context, self.config_path, self.config["primary"], self.parse_first_data_records ): yield (0, (offset, rec_type, rec_num, attr_type, file_name, content)) From adee265b1775a15913c0ce7d591cd9c6ccb32e18 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 15 Oct 2024 14:12:23 -0500 Subject: [PATCH 014/348] Address feedback --- .../framework/plugins/windows/mftscan.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 94dce8c4d..014516b7c 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -105,9 +105,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while attr.Attr_Header.AttrType.is_valid_choice: - yield from attr_callback( - record_map, mft_record, attr, symbol_table - ) + yield from attr_callback(record_map, mft_record, attr, symbol_table) # If there's no advancement the loop will never end, so break it now if attr.Attr_Header.Length == 0: @@ -271,7 +269,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): def _generator(self): for record in self.enumerate_mft_records( - self.context, self.config_path, self.config["primary"], self.parse_mft_records + self.context, + self.config_path, + self.config["primary"], + self.parse_mft_records, ): yield record @@ -352,7 +353,10 @@ class ADS(interfaces.plugins.PluginInterface): ads_name, content, ) in MFTScan.enumerate_mft_records( - self.context, self.config_path, self.config["primary"], self.parse_ads_data_records + self.context, + self.config_path, + self.config["primary"], + self.parse_ads_data_records, ): yield ( 0, @@ -418,7 +422,10 @@ class ResidentData(interfaces.plugins.PluginInterface): _, content, ) in MFTScan.enumerate_mft_records( - self.context, self.config_path, self.config["primary"], self.parse_first_data_records + self.context, + self.config_path, + self.config["primary"], + self.parse_first_data_records, ): yield (0, (offset, rec_type, rec_num, attr_type, file_name, content)) From 18f7f035ee83080d2ea660acfef05453c8e04ab5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 18 Oct 2024 12:29:09 +1100 Subject: [PATCH 015/348] linux: fix datetime import and remove unused ones --- .../framework/symbols/linux/extensions/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0dab0db43..d9392b9ba 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,13 +7,12 @@ import logging import functools import binascii import stat -from datetime import datetime +import datetime import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, Dict from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion -from volatility3.framework.configuration import requirements from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1882,7 +1881,7 @@ class kernel_cap_t(kernel_cap_struct): class timespec64(objects.StructType): - def to_datetime(self) -> datetime: + def to_datetime(self) -> datetime.datetime: """Returns the respective aware datetime""" dt = conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) @@ -1958,7 +1957,7 @@ class inode(objects.StructType): else: return None - def _time_member_to_datetime(self, member) -> datetime: + def _time_member_to_datetime(self, member) -> datetime.datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 @@ -1977,7 +1976,7 @@ class inode(objects.StructType): "Unsupported kernel inode type implementation" ) - def get_access_time(self) -> datetime: + def get_access_time(self) -> datetime.datetime: """Returns the inode's last access time This is updated when inode contents are read @@ -1986,7 +1985,7 @@ class inode(objects.StructType): """ return self._time_member_to_datetime("i_atime") - def get_modification_time(self) -> datetime: + def get_modification_time(self) -> datetime.datetime: """Returns the inode's last modification time This is updated when the inode contents change @@ -1996,7 +1995,7 @@ class inode(objects.StructType): return self._time_member_to_datetime("i_mtime") - def get_change_time(self) -> datetime: + def get_change_time(self) -> datetime.datetime: """Returns the inode's last change time This is updated when the inode metadata changes From f8224684078dd9937da4b0924f504a0bfb25b8bf Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 18 Oct 2024 12:40:02 +1100 Subject: [PATCH 016/348] linux: Implement boot time support in the Volatility 3 core framework --- .../framework/constants/linux/__init__.py | 3 + .../framework/symbols/linux/__init__.py | 98 +++++++++ .../symbols/linux/extensions/__init__.py | 197 ++++++++++++++++++ 3 files changed, 298 insertions(+) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 3eabc2341..a481e1ba9 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -302,3 +302,6 @@ class ELF_CLASS(IntEnum): ELFCLASSNONE = 0 ELFCLASS32 = 1 ELFCLASS64 = 2 + + +NSEC_PER_SEC = 1e9 diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 219f120ee..95b476263 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,11 +3,15 @@ # import math import contextlib +import datetime +import dataclasses from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects +from volatility3.framework.renderers import conversion +from volatility3.framework.constants.linux import NSEC_PER_SEC from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -830,3 +834,97 @@ class PageCache(object): page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page + + +@dataclasses.dataclass +class TimespecVol3(object): + """Internal helper class to handle all required timespec operations, convertions and + adjustments. + + NOTE: This is intended for exclusive use with get_boottime() and its related functions. + """ + + tv_sec: int = 0 + tv_nsec: int = 0 + + @classmethod + def new_from_timespec(cls, timespec) -> "TimespecVol3": + """Creates a new instance from a TimespecVol3 or timespec64 object""" + if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): + raise TypeError("It requires either a TimespecVol3 or timespec64 type") + + tv_sec = int(timespec.tv_sec) + tv_nsec = int(timespec.tv_nsec) + return cls(tv_sec=tv_sec, tv_nsec=tv_nsec) + + @classmethod + def new_from_nsec(cls, nsec) -> "TimespecVol3": + """Creates a new instance from an integer in nanoseconds""" + + # Based on ns_to_timespec64() + if nsec > 0: + tv_sec = nsec // NSEC_PER_SEC + tv_nsec = nsec % NSEC_PER_SEC + elif nsec < 0: + tv_sec = -((-nsec - 1) // NSEC_PER_SEC) - 1 + rem = (-nsec - 1) % NSEC_PER_SEC + tv_nsec = NSEC_PER_SEC - rem - 1 + else: + tv_sec = tv_nsec = 0 + + return cls(tv_sec=tv_sec, tv_nsec=tv_nsec) + + def to_datetime(self) -> datetime.datetime: + """Converts this TimespecVol3 to a UTC aware datetime""" + return conversion.unixtime_to_datetime( + self.tv_sec + self.tv_nsec / NSEC_PER_SEC + ) + + def to_timedelta(self) -> datetime.timedelta: + """Converts this TimespecVol3 to timedelta""" + return datetime.timedelta(seconds=self.tv_sec + self.tv_nsec / NSEC_PER_SEC) + + def __add__(self, timespec) -> "TimespecVol3": + """Returns a new TimespecVol3 object that sums the current values with those + in the timespec argument""" + if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): + raise TypeError("Cannot add a TimespecVol3 to this object") + + result = TimespecVol3( + tv_sec=self.tv_sec + timespec.tv_sec, + tv_nsec=self.tv_nsec + timespec.tv_nsec, + ) + + result.normalize() + + return result + + def __sub__(self, timespec) -> "TimespecVol3": + """Returns a new TimespecVol3 object that subtracts the values in the timespec + argument from the current object's values""" + if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): + raise TypeError("Cannot add a TimespecVol3 to this object") + + result = TimespecVol3( + tv_sec=self.tv_sec - timespec.tv_sec, + tv_nsec=self.tv_nsec - timespec.tv_nsec, + ) + result.normalize() + + return result + + def normalize(self): + """Normalize any overflow in tv_sec and tv_nsec after previous addition or subtractions""" + # Based on kernel's set_normalized_timespec64() + while self.tv_nsec >= NSEC_PER_SEC: + self.tv_nsec -= NSEC_PER_SEC + self.tv_sec += 1 + + while self.tv_nsec < 0: + self.tv_nsec += NSEC_PER_SEC + self.tv_sec -= 1 + + def negate(self): + """Negates the sign of both tv_sec and tv_nsec""" + self.tv_sec = -self.tv_sec + self.tv_nsec = -self.tv_nsec diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d9392b9ba..b3e836102 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -382,6 +382,203 @@ class task_struct(generic.GenericIntelProcess): threads_seen.add(task.vol.offset) yield task + def _get_task_start_time(self) -> datetime.timedelta: + """Returns the task's monotonic start_time as a timedelta. + + Returns: + The task's start time as a timedelta object. + """ + for member_name in ("start_boottime", "real_start_time", "start_time"): + if self.has_member(member_name): + start_time_obj = self.member(member_name) + start_time_obj_type = start_time_obj.vol.type_name + start_time_obj_type_name = start_time_obj_type.split(constants.BANG)[1] + if start_time_obj_type_name != "timespec": + # kernels >= 3.17 real_start_time and start_time are u64 + # kernels >= 5.5 uses start_boottime which is also a u64 + start_time = linux.TimespecVol3.new_from_nsec(start_time_obj) + else: + # kernels < 3.17 real_start_time and start_time are timespec + start_time = linux.TimespecVol3.new_from_timespec(start_time_obj) + + # This is relative to the boot time so it makes sense to be a timedelta. + return start_time.to_timedelta() + + raise AttributeError("Unsupported task_struct start_time member") + + def get_time_namespace(self) -> Optional[interfaces.objects.ObjectInterface]: + """Returns the task's time namespace""" + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if not self.has_member("nsproxy"): + # kernels < 2.6.19: ab516013ad9ca47f1d3a936fa81303bfbf734d52 + return None + + if not vmlinux.get_type("nsproxy").has_member("time_ns"): + # kernels < 5.6 769071ac9f20b6a447410c7eaa55d1a5233ef40c + return None + + return self.nsproxy.time_ns + + def get_time_namespace_id(self) -> int: + """Returns the task's time namespace ID.""" + time_ns = self.get_time_namespace() + if not time_ns: + # kernels < 5.6 + return + + # We are good. ns_common (ns) was introduced in kernels 3.19. So by the time the + # time namespace was added in kernels 5.6, it already included the ns member. + return time_ns.ns.inum + + def _get_time_namespace_offsets( + self, + ) -> Optional[interfaces.objects.ObjectInterface]: + """Returns the time offsets from the task's time namespace.""" + time_ns = self.get_time_namespace() + if not time_ns: + # kernels < 5.6 + return + + if not time_ns.has_member("offsets"): + # kernels < 5.6 af993f58d69ee9c1f421dfc87c3ed231c113989c + return None + + return time_ns.offsets + + def get_time_namespace_monotonic_offset( + self, + ) -> Optional[interfaces.objects.ObjectInterface]: + """Gets task's time namespace monotonic offset + + Returns: + a kernel's timespec64 object with the monotonic offset + """ + time_namespace_offsets = self._get_time_namespace_offsets() + if not time_namespace_offsets: + return None + + return time_namespace_offsets.monotonic + + def _get_time_namespace_boottime_offset( + self, + ) -> Optional[interfaces.objects.ObjectInterface]: + """Gets task's time namespace boottime offset + + Returns: + a kernel's timespec64 object with the boottime offset + """ + time_namespace_offsets = self._get_time_namespace_offsets() + if not time_namespace_offsets: + return None + + return time_namespace_offsets.boottime + + def _get_boottime_raw(self) -> "linux.TimespecVol3": + """Returns the boot time in a TimespecVol3.""" + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if vmlinux.has_symbol("tk_core"): + # kernels >= 3.17 | tk_core | 3fdb14fd1df70325e1e91e1203a699a4803ed741 + tk_core = vmlinux.object_from_symbol("tk_core") + timekeeper = tk_core.timekeeper + if not timekeeper.offs_real.has_member("tv64"): + # kernels >= 4.10 - Tested on Ubuntu 6.8.0-41 + boottime_nsec = timekeeper.offs_real - timekeeper.offs_boot + else: + # 3.17 <= kernels < 4.10 - Tested on Ubuntu 4.4.0-142 + boottime_nsec = timekeeper.offs_real.tv64 - timekeeper.offs_boot.tv64 + return linux.TimespecVol3.new_from_nsec(boottime_nsec) + + elif vmlinux.has_symbol("timekeeper") and vmlinux.get_type( + "timekeeper" + ).has_member("wall_to_monotonic"): + # 3.4 <= kernels < 3.17 - Tested on Ubuntu 3.13.0-185 + timekeeper = vmlinux.object_from_symbol("timekeeper") + + # timekeeper.wall_to_monotonic is timespec + boottime = linux.TimespecVol3.new_from_timespec( + timekeeper.wall_to_monotonic + ) + + boottime += timekeeper.total_sleep_time + + boottime.negate() + boottime.normalize() + + return boottime + + elif vmlinux.has_symbol("wall_to_monotonic"): + # kernels < 3.4 - Tested on Debian7 3.2.0-4 (3.2.57-3+deb7u2) + wall_to_monotonic = vmlinux.object_from_symbol("wall_to_monotonic") + boottime = linux.TimespecVol3.new_from_timespec(wall_to_monotonic) + if vmlinux.has_symbol("total_sleep_time"): + # 2.6.23 <= kernels < 3.4 7c3f1a573237b90ef331267260358a0ec4ac9079 + total_sleep_time = vmlinux.object_from_symbol("total_sleep_time") + full_type_name = total_sleep_time.vol.type_name + type_name = full_type_name.split(constants.BANG)[1] + if type_name == "timespec": + # kernels >= 2.6.32 total_sleep_time is a timespec + boottime += total_sleep_time + else: + # kernels < 2.6.32 total_sleep_time is an unsigned long as seconds + boottime.tv_sec += total_sleep_time + + boottime.negate() + boottime.normalize() + + return boottime + + raise exceptions.VolatilityException("Unsupported") + + def get_boottime(self, root_time_namespace: bool = True) -> datetime.datetime: + """Returns the boot time in UTC as a datetime. + + Args: + root_time_namespace: If True, it returns the boot time as seen from the root + time namespace. Otherwise, it returns the boot time relative to the + task's time namespace. + + Returns: + A datetime with the UTC boot time. + """ + boottime = self._get_boottime_raw() + if not boottime: + return None + + if not root_time_namespace: + # Shift boot timestamp according to the task's time namespace offset + boottime_offset_timespec = self._get_time_namespace_boottime_offset() + if boottime_offset_timespec: + # Time namespace support is from kernels 5.6 + boottime -= boottime_offset_timespec + + return boottime.to_datetime() + + def get_create_time(self) -> datetime.datetime: + """Retrieves the task's start time from its time namespace. + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + task: A reference task + + Returns: + A datetime with task's start time + """ + # Typically, we want to see the creation time seen from the root time namespace + boottime = self.get_boottime(root_time_namespace=True) + + # The kernel exports only tv_sec to procfs, see kernel's show_stat(). + # This means user-space tools, like those in the procps package (e.g., ps, top, etc.), + # only use the boot time seconds to compute dates relatives to this. + boottime = boottime.replace(microsecond=0) + + task_start_time_timedelta = self._get_task_start_time() + + # NOTE: Do NOT apply the task's time namespace offsets here. While the kernel uses + # timens_add_boottime_ns(), it's not needed here since we're seeing it from the + # root time namespace, not within the task's own time namespace + return boottime + task_start_time_timedelta + class fs_struct(objects.StructType): def get_root_dentry(self): From 69512dc9911fc6ce8d0e1ab65926663341513974 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 18 Oct 2024 12:58:02 +1100 Subject: [PATCH 017/348] Linux: pslist: Add creation time column and timeline support to the linux.pslist plugin --- volatility3/framework/plugins/linux/pslist.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 1888bd7b8..b05d69c7a 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -1,6 +1,7 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime from typing import Any, Callable, Iterable, List, Tuple from volatility3.framework import interfaces, renderers @@ -9,15 +10,16 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux.extensions import elf +from volatility3.plugins import timeliner from volatility3.plugins.linux import elfs -class PsList(interfaces.plugins.PluginInterface): +class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 0, 0) - _version = (2, 2, 1) + _version = (2, 3, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -81,7 +83,7 @@ class PsList(interfaces.plugins.PluginInterface): @classmethod def get_task_fields( cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False - ) -> Tuple[int, int, int, str]: + ) -> Tuple[int, int, int, int, str, datetime.datetime]: """Extract the fields needed for the final output Args: @@ -96,13 +98,14 @@ class PsList(interfaces.plugins.PluginInterface): tid = task.pid ppid = task.parent.tgid if task.parent else 0 name = utility.array_to_string(task.comm) + start_time = task.get_create_time() if decorate_comm: if task.is_kernel_thread: name = f"[{name}]" elif task.is_user_thread: name = f"{{{name}}}" - task_fields = (task.vol.offset, pid, tid, ppid, name) + task_fields = (task.vol.offset, pid, tid, ppid, name, start_time) return task_fields def _get_file_output(self, task: interfaces.objects.ObjectInterface) -> str: @@ -177,7 +180,9 @@ class PsList(interfaces.plugins.PluginInterface): else: file_output = "Disabled" - offset, pid, tid, ppid, name = self.get_task_fields(task, decorate_comm) + offset, pid, tid, ppid, name, creation_time = self.get_task_fields( + task, decorate_comm + ) yield 0, ( format_hints.Hex(offset), @@ -185,6 +190,7 @@ class PsList(interfaces.plugins.PluginInterface): tid, ppid, name, + creation_time or renderers.NotAvailableValue(), file_output, ) @@ -233,8 +239,23 @@ class PsList(interfaces.plugins.PluginInterface): ("TID", int), ("PPID", int), ("COMM", str), + ("CREATION TIME", datetime.datetime), ("File output", str), ] return renderers.TreeGrid( columns, self._generator(filter_func, include_threads, decorate_comm, dump) ) + + def generate_timeline(self): + pids = self.config.get("pid") + filter_func = self.create_pid_filter(pids) + for task in self.list_tasks( + self.context, self.config["kernel"], filter_func, include_threads=True + ): + offset, user_pid, user_tid, _user_ppid, name, creation_time = ( + self.get_task_fields(task) + ) + + description = f"Process {user_pid}/{user_tid} {name} ({offset})" + + yield (description, timeliner.TimeLinerType.CREATED, creation_time) From d25df2357acf157641a6b30a7a9a30d0261bf147 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 18 Oct 2024 12:59:23 +1100 Subject: [PATCH 018/348] Linux: pslist: Add the boottime plugin --- .../framework/plugins/linux/boottime.py | 98 +++++++++++++++++++ volatility3/framework/plugins/timeliner.py | 13 ++- 2 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 volatility3/framework/plugins/linux/boottime.py diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py new file mode 100644 index 000000000..eeee418f8 --- /dev/null +++ b/volatility3/framework/plugins/linux/boottime.py @@ -0,0 +1,98 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import datetime +from typing import List, Tuple, Iterable + + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.plugins import timeliner +from volatility3.plugins.linux import pslist + + +class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """Shows the time the system was started""" + + _required_framework_version = (2, 11, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 3, 0) + ), + ] + + @classmethod + def get_time_namespaces_bootime( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Iterable[Tuple[int, int, int, str, datetime.datetime]]: + """Enumerates tasks' boot times based on their time namespaces. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + pids: Pid list + unique: Filter unique time namespaces + + Yields: + A tuple with the fields to show in the plugin output. + """ + time_namespace_ids = set() + for task in pslist.PsList.list_tasks(context, vmlinux_module_name): + time_namespace_id = task.get_time_namespace_id() + # If it cannot get the time namespace i.e. kernels < 5.6, this still works + # using None to just get the first tasks + if time_namespace_id in time_namespace_ids: + continue + time_namespace_ids.add(time_namespace_id) + boottime = task.get_boottime(root_time_namespace=False) + + fields = ( + time_namespace_id, + boottime, + ) + yield fields + + def _generator(self): + for ( + time_namespace_id, + boottime, + ) in self.get_time_namespaces_bootime( + self.context, + self.config["kernel"], + ): + fields = [ + time_namespace_id or renderers.NotAvailableValue(), + boottime, + ] + yield 0, fields + + def generate_timeline(self): + for ( + time_namespace_id, + boottime, + ) in self.get_time_namespaces_bootime( + self.context, + self.config["kernel"], + ): + description = f"System boot time for time namespace {time_namespace_id}" + + yield description, timeliner.TimeLinerType.BOOTTIME, boottime + + def run(self): + columns = [ + ("TIME NS", int), + ("Boot Time", datetime.datetime), + ] + return renderers.TreeGrid(columns, self._generator()) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index c754e43ef..70da0c4fb 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -23,6 +23,7 @@ class TimeLinerType(enum.IntEnum): MODIFIED = 2 ACCESSED = 3 CHANGED = 4 + BOOTTIME = 5 class TimeLinerInterface(metaclass=abc.ABCMeta): @@ -171,6 +172,10 @@ class Timeliner(interfaces.plugins.PluginInterface): TimeLinerType.CHANGED, renderers.NotApplicableValue(), ), + times.get( + TimeLinerType.BOOTTIME, + renderers.NotApplicableValue(), + ), ], ) ) @@ -178,11 +183,11 @@ class Timeliner(interfaces.plugins.PluginInterface): # Write each entry because the body file doesn't need to be sorted if fp: times = self.timeline[(plugin_name, item)] - # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime + # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime|boottime if self._any_time_present(times): fp.write( - "|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format( + "|{} - {}|0|0|0|0|0|{}|{}|{}|{}|{}\n".format( plugin_name, self._sanitize_body_format(item), self._text_format( @@ -197,6 +202,9 @@ class Timeliner(interfaces.plugins.PluginInterface): self._text_format( times.get(TimeLinerType.CREATED, "0") ), + self._text_format( + times.get(TimeLinerType.BOOTTIME, "0") + ), ) ) except Exception as e: @@ -320,6 +328,7 @@ class Timeliner(interfaces.plugins.PluginInterface): ("Modified Date", datetime.datetime), ("Accessed Date", datetime.datetime), ("Changed Date", datetime.datetime), + ("Boot Date", datetime.datetime), ], generator=self._generator(plugins_to_run), ) From 1651ecb1d70bb32868be864c473ef4a346141381 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 18 Oct 2024 13:58:27 +1100 Subject: [PATCH 019/348] linux: boottime api: Fix explicit returns mixed with implicit (fall through) returns --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b3e836102..2784aeda7 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -424,7 +424,7 @@ class task_struct(generic.GenericIntelProcess): time_ns = self.get_time_namespace() if not time_ns: # kernels < 5.6 - return + return None # We are good. ns_common (ns) was introduced in kernels 3.19. So by the time the # time namespace was added in kernels 5.6, it already included the ns member. @@ -437,7 +437,7 @@ class task_struct(generic.GenericIntelProcess): time_ns = self.get_time_namespace() if not time_ns: # kernels < 5.6 - return + return None if not time_ns.has_member("offsets"): # kernels < 5.6 af993f58d69ee9c1f421dfc87c3ed231c113989c From 094cdf18f14d795f385e504506390ba1602c0b55 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 10:59:18 +1100 Subject: [PATCH 020/348] Linux: netfilter plugin: Fix hooked field to match vol2 output --- volatility3/framework/plugins/linux/netfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index f576c073c..66075f907 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -190,7 +190,7 @@ class AbstractNetfilter(ABC): priority = int(hook_ops.priority) hook_ops_hook = hook_ops.hook module_name = self.get_module_name_for_address(hook_ops_hook) - hooked = module_name is not None + hooked = module_name is None yield netns, proto_name, hook_name, priority, hook_ops_hook, module_name, hooked From 4895af47bdec7779c7806dca47c657ae686a6e0b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 13:46:19 +1100 Subject: [PATCH 021/348] Linux: Boottime timeliner: Rollback timeliner event type changes and use the created time for the boot time plugin --- volatility3/framework/plugins/linux/boottime.py | 2 +- volatility3/framework/plugins/timeliner.py | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py index eeee418f8..8f63ee7f8 100644 --- a/volatility3/framework/plugins/linux/boottime.py +++ b/volatility3/framework/plugins/linux/boottime.py @@ -88,7 +88,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ): description = f"System boot time for time namespace {time_namespace_id}" - yield description, timeliner.TimeLinerType.BOOTTIME, boottime + yield description, timeliner.TimeLinerType.CREATED, boottime def run(self): columns = [ diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 70da0c4fb..c754e43ef 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -23,7 +23,6 @@ class TimeLinerType(enum.IntEnum): MODIFIED = 2 ACCESSED = 3 CHANGED = 4 - BOOTTIME = 5 class TimeLinerInterface(metaclass=abc.ABCMeta): @@ -172,10 +171,6 @@ class Timeliner(interfaces.plugins.PluginInterface): TimeLinerType.CHANGED, renderers.NotApplicableValue(), ), - times.get( - TimeLinerType.BOOTTIME, - renderers.NotApplicableValue(), - ), ], ) ) @@ -183,11 +178,11 @@ class Timeliner(interfaces.plugins.PluginInterface): # Write each entry because the body file doesn't need to be sorted if fp: times = self.timeline[(plugin_name, item)] - # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime|boottime + # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime if self._any_time_present(times): fp.write( - "|{} - {}|0|0|0|0|0|{}|{}|{}|{}|{}\n".format( + "|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format( plugin_name, self._sanitize_body_format(item), self._text_format( @@ -202,9 +197,6 @@ class Timeliner(interfaces.plugins.PluginInterface): self._text_format( times.get(TimeLinerType.CREATED, "0") ), - self._text_format( - times.get(TimeLinerType.BOOTTIME, "0") - ), ) ) except Exception as e: @@ -328,7 +320,6 @@ class Timeliner(interfaces.plugins.PluginInterface): ("Modified Date", datetime.datetime), ("Accessed Date", datetime.datetime), ("Changed Date", datetime.datetime), - ("Boot Date", datetime.datetime), ], generator=self._generator(plugins_to_run), ) From c4274c942c4e94e909de44bbe3e12b47a15143f2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 13:49:58 +1100 Subject: [PATCH 022/348] Linux: Fix exception message in TimespecVol3::__sub__() --- volatility3/framework/symbols/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 95b476263..8c28ca562 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -903,7 +903,7 @@ class TimespecVol3(object): """Returns a new TimespecVol3 object that subtracts the values in the timespec argument from the current object's values""" if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): - raise TypeError("Cannot add a TimespecVol3 to this object") + raise TypeError("Cannot substract this object to a TimespecVol3") result = TimespecVol3( tv_sec=self.tv_sec - timespec.tv_sec, From 57ffd5b939df456b7e573d91d6b23bc01f36f353 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 14:24:16 +1100 Subject: [PATCH 023/348] Linux: Boottime API: Refactor TimespecVol3::negate() to return a new object instead of modifying the original. It also normalizes its values, aligning with the behavior of the other addition and subtraction operators --- volatility3/framework/symbols/linux/__init__.py | 14 +++++++++++--- .../framework/symbols/linux/extensions/__init__.py | 10 ++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 8c28ca562..4123674ed 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -909,6 +909,7 @@ class TimespecVol3(object): tv_sec=self.tv_sec - timespec.tv_sec, tv_nsec=self.tv_nsec - timespec.tv_nsec, ) + result.normalize() return result @@ -925,6 +926,13 @@ class TimespecVol3(object): self.tv_sec -= 1 def negate(self): - """Negates the sign of both tv_sec and tv_nsec""" - self.tv_sec = -self.tv_sec - self.tv_nsec = -self.tv_nsec + """Returns a new TimespecVol3 object with the values of the current object negated""" + + result = TimespecVol3( + tv_sec=-self.tv_sec, + tv_nsec=-self.tv_nsec, + ) + + result.normalize() + + return result diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 2784aeda7..6fd55cb56 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -502,10 +502,7 @@ class task_struct(generic.GenericIntelProcess): boottime += timekeeper.total_sleep_time - boottime.negate() - boottime.normalize() - - return boottime + return boottime.negate() elif vmlinux.has_symbol("wall_to_monotonic"): # kernels < 3.4 - Tested on Debian7 3.2.0-4 (3.2.57-3+deb7u2) @@ -523,10 +520,7 @@ class task_struct(generic.GenericIntelProcess): # kernels < 2.6.32 total_sleep_time is an unsigned long as seconds boottime.tv_sec += total_sleep_time - boottime.negate() - boottime.normalize() - - return boottime + return boottime.negate() raise exceptions.VolatilityException("Unsupported") From bee2a398eb4d3f3b28644ed2039bb516ccc3e75e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 15:13:26 +1100 Subject: [PATCH 024/348] Linux: Minor: Add comment/header on each set of constants --- volatility3/framework/constants/linux/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 056c384ae..7bfa9268d 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -304,6 +304,7 @@ class ELF_CLASS(IntEnum): ELFCLASS64 = 2 +# PTrace PT_OPT_FLAG_SHIFT = 3 PTRACE_EVENT_FORK = 1 @@ -341,4 +342,5 @@ class PT_FLAGS(Flag): return str(self).replace(self.__class__.__name__ + ".", "") +# Boot time NSEC_PER_SEC = 1e9 From 42d918c05d6a2d08d025d63f51b9c6764076be12 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 15:21:13 +1100 Subject: [PATCH 025/348] Linux: Boottime API: Refactor Timespec Methods. Move TimespecVol3 methods to an abstract class Timespec64Abstract, which is now inherited by Timespec64 and Timespec64Concrete. --- .../framework/symbols/linux/__init__.py | 106 ------------- .../symbols/linux/extensions/__init__.py | 140 ++++++++++++++++-- 2 files changed, 127 insertions(+), 119 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 84f546a9f..1f2812c10 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,15 +3,11 @@ # import math import contextlib -import datetime -import dataclasses from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects -from volatility3.framework.renderers import conversion -from volatility3.framework.constants.linux import NSEC_PER_SEC from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -836,105 +832,3 @@ class PageCache(object): page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page - - -@dataclasses.dataclass -class TimespecVol3(object): - """Internal helper class to handle all required timespec operations, convertions and - adjustments. - - NOTE: This is intended for exclusive use with get_boottime() and its related functions. - """ - - tv_sec: int = 0 - tv_nsec: int = 0 - - @classmethod - def new_from_timespec(cls, timespec) -> "TimespecVol3": - """Creates a new instance from a TimespecVol3 or timespec64 object""" - if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): - raise TypeError("It requires either a TimespecVol3 or timespec64 type") - - tv_sec = int(timespec.tv_sec) - tv_nsec = int(timespec.tv_nsec) - return cls(tv_sec=tv_sec, tv_nsec=tv_nsec) - - @classmethod - def new_from_nsec(cls, nsec) -> "TimespecVol3": - """Creates a new instance from an integer in nanoseconds""" - - # Based on ns_to_timespec64() - if nsec > 0: - tv_sec = nsec // NSEC_PER_SEC - tv_nsec = nsec % NSEC_PER_SEC - elif nsec < 0: - tv_sec = -((-nsec - 1) // NSEC_PER_SEC) - 1 - rem = (-nsec - 1) % NSEC_PER_SEC - tv_nsec = NSEC_PER_SEC - rem - 1 - else: - tv_sec = tv_nsec = 0 - - return cls(tv_sec=tv_sec, tv_nsec=tv_nsec) - - def to_datetime(self) -> datetime.datetime: - """Converts this TimespecVol3 to a UTC aware datetime""" - return conversion.unixtime_to_datetime( - self.tv_sec + self.tv_nsec / NSEC_PER_SEC - ) - - def to_timedelta(self) -> datetime.timedelta: - """Converts this TimespecVol3 to timedelta""" - return datetime.timedelta(seconds=self.tv_sec + self.tv_nsec / NSEC_PER_SEC) - - def __add__(self, timespec) -> "TimespecVol3": - """Returns a new TimespecVol3 object that sums the current values with those - in the timespec argument""" - if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): - raise TypeError("Cannot add a TimespecVol3 to this object") - - result = TimespecVol3( - tv_sec=self.tv_sec + timespec.tv_sec, - tv_nsec=self.tv_nsec + timespec.tv_nsec, - ) - - result.normalize() - - return result - - def __sub__(self, timespec) -> "TimespecVol3": - """Returns a new TimespecVol3 object that subtracts the values in the timespec - argument from the current object's values""" - if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): - raise TypeError("Cannot substract this object to a TimespecVol3") - - result = TimespecVol3( - tv_sec=self.tv_sec - timespec.tv_sec, - tv_nsec=self.tv_nsec - timespec.tv_nsec, - ) - - result.normalize() - - return result - - def normalize(self): - """Normalize any overflow in tv_sec and tv_nsec after previous addition or subtractions""" - # Based on kernel's set_normalized_timespec64() - while self.tv_nsec >= NSEC_PER_SEC: - self.tv_nsec -= NSEC_PER_SEC - self.tv_sec += 1 - - while self.tv_nsec < 0: - self.tv_nsec += NSEC_PER_SEC - self.tv_sec -= 1 - - def negate(self): - """Returns a new TimespecVol3 object with the values of the current object negated""" - - result = TimespecVol3( - tv_sec=-self.tv_sec, - tv_nsec=-self.tv_nsec, - ) - - result.normalize() - - return result diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e952ed968..ea20850b7 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import abc import collections.abc import logging import functools @@ -18,12 +19,13 @@ from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES -from volatility3.framework.constants.linux import CAPABILITIES, PT_FLAGS +from volatility3.framework.constants.linux import CAPABILITIES, PT_FLAGS, NSEC_PER_SEC from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf + vollog = logging.getLogger(__name__) # Keep these in a basic module, to prevent import cycles when symbol providers require them @@ -431,10 +433,10 @@ class task_struct(generic.GenericIntelProcess): if start_time_obj_type_name != "timespec": # kernels >= 3.17 real_start_time and start_time are u64 # kernels >= 5.5 uses start_boottime which is also a u64 - start_time = linux.TimespecVol3.new_from_nsec(start_time_obj) + start_time = Timespec64Concrete.new_from_nsec(start_time_obj) else: # kernels < 3.17 real_start_time and start_time are timespec - start_time = linux.TimespecVol3.new_from_timespec(start_time_obj) + start_time = Timespec64Concrete.new_from_timespec(start_time_obj) # This is relative to the boot time so it makes sense to be a timedelta. return start_time.to_timedelta() @@ -508,8 +510,8 @@ class task_struct(generic.GenericIntelProcess): return time_namespace_offsets.boottime - def _get_boottime_raw(self) -> "linux.TimespecVol3": - """Returns the boot time in a TimespecVol3.""" + def _get_boottime_raw(self) -> "Timespec64Concrete": + """Returns the boot time in a Timespec64Concrete object.""" vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) if vmlinux.has_symbol("tk_core"): @@ -522,7 +524,7 @@ class task_struct(generic.GenericIntelProcess): else: # 3.17 <= kernels < 4.10 - Tested on Ubuntu 4.4.0-142 boottime_nsec = timekeeper.offs_real.tv64 - timekeeper.offs_boot.tv64 - return linux.TimespecVol3.new_from_nsec(boottime_nsec) + return Timespec64Concrete.new_from_nsec(boottime_nsec) elif vmlinux.has_symbol("timekeeper") and vmlinux.get_type( "timekeeper" @@ -531,7 +533,7 @@ class task_struct(generic.GenericIntelProcess): timekeeper = vmlinux.object_from_symbol("timekeeper") # timekeeper.wall_to_monotonic is timespec - boottime = linux.TimespecVol3.new_from_timespec( + boottime = Timespec64Concrete.new_from_timespec( timekeeper.wall_to_monotonic ) @@ -542,7 +544,7 @@ class task_struct(generic.GenericIntelProcess): elif vmlinux.has_symbol("wall_to_monotonic"): # kernels < 3.4 - Tested on Debian7 3.2.0-4 (3.2.57-3+deb7u2) wall_to_monotonic = vmlinux.object_from_symbol("wall_to_monotonic") - boottime = linux.TimespecVol3.new_from_timespec(wall_to_monotonic) + boottime = Timespec64Concrete.new_from_timespec(wall_to_monotonic) if vmlinux.has_symbol("total_sleep_time"): # 2.6.23 <= kernels < 3.4 7c3f1a573237b90ef331267260358a0ec4ac9079 total_sleep_time = vmlinux.object_from_symbol("total_sleep_time") @@ -2168,12 +2170,124 @@ class kernel_cap_t(kernel_cap_struct): return cap_value & self.get_kernel_cap_full() -class timespec64(objects.StructType): - def to_datetime(self) -> datetime.datetime: - """Returns the respective aware datetime""" +class Timespec64Abstract(abc.ABC): + """Abstract class to handle all required timespec64 operations, convertions and + adjustments.""" - dt = conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) - return dt + @classmethod + def new_from_timespec(cls, other) -> "Timespec64Concrete": + """Creates a new instance from an Timespec64Abstract subclass object""" + if not isinstance(other, Timespec64Abstract): + raise TypeError("Requires an object subclass of Timespec64Abstract") + + tv_sec = int(other.tv_sec) + tv_nsec = int(other.tv_nsec) + return Timespec64Concrete(tv_sec=tv_sec, tv_nsec=tv_nsec) + + @classmethod + def new_from_nsec(cls, nsec) -> "Timespec64Concrete": + """Creates a new instance from an integer in nanoseconds""" + + # Based on ns_to_timespec64() + if nsec > 0: + tv_sec = nsec // NSEC_PER_SEC + tv_nsec = nsec % NSEC_PER_SEC + elif nsec < 0: + tv_sec = -((-nsec - 1) // NSEC_PER_SEC) - 1 + rem = (-nsec - 1) % NSEC_PER_SEC + tv_nsec = NSEC_PER_SEC - rem - 1 + else: + tv_sec = tv_nsec = 0 + + return Timespec64Concrete(tv_sec=tv_sec, tv_nsec=tv_nsec) + + def to_datetime(self) -> datetime.datetime: + """Converts this Timespec64Abstract subclass object to a UTC aware datetime""" + + # pylint: disable=E1101 + return conversion.unixtime_to_datetime( + self.tv_sec + self.tv_nsec / NSEC_PER_SEC + ) + + def to_timedelta(self) -> datetime.timedelta: + """Converts this Timespec64Abstract subclass object to timedelta""" + # pylint: disable=E1101 + return datetime.timedelta(seconds=self.tv_sec + self.tv_nsec / NSEC_PER_SEC) + + def __add__(self, other) -> "Timespec64Concrete": + """Returns a new Timespec64Concrete object that sums the current values with those + in the timespec argument""" + if not isinstance(other, Timespec64Abstract): + raise TypeError("Requires an object subclass of Timespec64Abstract") + + # pylint: disable=E1101 + result = Timespec64Concrete( + tv_sec=self.tv_sec + other.tv_sec, + tv_nsec=self.tv_nsec + other.tv_nsec, + ) + + result.normalize() + + return result + + def __sub__(self, other) -> "Timespec64Concrete": + """Returns a new Timespec64Abstract object that subtracts the values in the timespec + argument from the current object's values""" + if not isinstance(other, Timespec64Abstract): + raise TypeError("Requires an object subclass of Timespec64Abstract") + + # pylint: disable=E1101 + result = Timespec64Concrete( + tv_sec=self.tv_sec - other.tv_sec, + tv_nsec=self.tv_nsec - other.tv_nsec, + ) + + result.normalize() + + return result + + def normalize(self): + """Normalize any overflow in tv_sec and tv_nsec.""" + # Based on kernel's set_normalized_timespec64() + + # pylint: disable=E1101 + while self.tv_nsec >= NSEC_PER_SEC: + self.tv_nsec -= NSEC_PER_SEC + self.tv_sec += 1 + + while self.tv_nsec < 0: + self.tv_nsec += NSEC_PER_SEC + self.tv_sec -= 1 + + def negate(self): + """Returns a new Timespec64Concrete object with the values of the current object negated""" + # pylint: disable=E1101 + result = Timespec64Concrete( + tv_sec=-self.tv_sec, + tv_nsec=-self.tv_nsec, + ) + + result.normalize() + + return result + + +class Timespec64Concrete(Timespec64Abstract): + """Handle all required timespec64 operations, convertions and adjustments. + This is used to dynamically create timespec64-like objects, each its own variables + and the same methods as a timespec64 object extension. + """ + + def __init__(self, tv_sec=0, tv_nsec=0): + self.tv_sec = tv_sec + self.tv_nsec = tv_nsec + + +class timespec64(Timespec64Abstract, objects.StructType): + """Handle all required timespec64 operations, convertions and adjustments. + This works as an extension of the timespec64 object while maintaining the same methods + as a Timespec64Concrete object. + """ class inode(objects.StructType): From 7abe92cf2f2ddac1a5e7f15a27afe341e1810901 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 15:36:46 +1100 Subject: [PATCH 026/348] Linux: Boottime API: Minor. Move negate() up --- .../symbols/linux/extensions/__init__.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index ea20850b7..f20d93755 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2244,6 +2244,18 @@ class Timespec64Abstract(abc.ABC): result.normalize() + return self + other.negate() + + def negate(self): + """Returns a new Timespec64Concrete object with the values of the current object negated""" + # pylint: disable=E1101 + result = Timespec64Concrete( + tv_sec=-self.tv_sec, + tv_nsec=-self.tv_nsec, + ) + + result.normalize() + return result def normalize(self): @@ -2259,18 +2271,6 @@ class Timespec64Abstract(abc.ABC): self.tv_nsec += NSEC_PER_SEC self.tv_sec -= 1 - def negate(self): - """Returns a new Timespec64Concrete object with the values of the current object negated""" - # pylint: disable=E1101 - result = Timespec64Concrete( - tv_sec=-self.tv_sec, - tv_nsec=-self.tv_nsec, - ) - - result.normalize() - - return result - class Timespec64Concrete(Timespec64Abstract): """Handle all required timespec64 operations, convertions and adjustments. From e197fbaf5fe7a7fba8fabce73533725af9110d74 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 15:39:40 +1100 Subject: [PATCH 027/348] Linux: Boottime API: Refactor __sub__ to operate through __add__() and negate() for improved clarity and reuse --- .../framework/symbols/linux/extensions/__init__.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f20d93755..c80ffa515 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2236,17 +2236,9 @@ class Timespec64Abstract(abc.ABC): if not isinstance(other, Timespec64Abstract): raise TypeError("Requires an object subclass of Timespec64Abstract") - # pylint: disable=E1101 - result = Timespec64Concrete( - tv_sec=self.tv_sec - other.tv_sec, - tv_nsec=self.tv_nsec - other.tv_nsec, - ) - - result.normalize() - return self + other.negate() - def negate(self): + def negate(self) -> "Timespec64Concrete": """Returns a new Timespec64Concrete object with the values of the current object negated""" # pylint: disable=E1101 result = Timespec64Concrete( From a05397e8b68c280ba74c58bc968cc6692d25e613 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 15:57:00 +1100 Subject: [PATCH 028/348] Linux: Boottime API: Minor. Fix docstring typo --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index c80ffa515..a9127907e 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2266,7 +2266,7 @@ class Timespec64Abstract(abc.ABC): class Timespec64Concrete(Timespec64Abstract): """Handle all required timespec64 operations, convertions and adjustments. - This is used to dynamically create timespec64-like objects, each its own variables + This is used to dynamically create timespec64-like objects, each with its own variables and the same methods as a timespec64 object extension. """ From d5a0b93383fda59267bdd9b42e716b70ad66595c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:28:40 +0100 Subject: [PATCH 029/348] add TAINT_FLAGS constant --- .../framework/constants/linux/__init__.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 7c485d3c3..9f25c9225 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -347,3 +347,88 @@ class PT_FLAGS(Flag): MODULE_MAXIMUM_CORE_SIZE = 20000000 MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 MODULE_MINIMUM_SIZE = 4096 + + +TAINT_FLAGS = { + "P": { + "shift": 1 << 0, + "desc": "PROPRIETARY_MODULE", + "when_present": True, + "module": True, + }, + "G": { + "shift": 1 << 0, + "desc": "PROPRIETARY_MODULE", + "when_present": False, + "module": True, + }, + "F": { + "shift": 1 << 1, + "desc": "FORCED_MODULE", + "when_present": True, + "module": False, + }, + # CPU_OUT_OF_SPEC was TAINT_UNSAFE_SMP on < 3.15-rc1 : https://lore.kernel.org/linux-kernel//20140303080432.GA25489@localhost/t/#:~:text=liked%20your%20proposal%3A-,%3E%20Right,-%2C%20I%20was%20about + "S": { + "shift": 1 << 2, + "desc": "CPU_OUT_OF_SPEC", + "when_present": True, + "module": False, + }, + "R": { + "shift": 1 << 3, + "desc": "FORCED_RMMOD", + "when_present": True, + "module": False, + }, + "M": { + "shift": 1 << 4, + "desc": "MACHINE_CHECK", + "when_present": True, + "module": False, + }, + "B": {"shift": 1 << 5, "desc": "BAD_PAGE", "when_present": True, "module": False}, + "U": {"shift": 1 << 6, "desc": "USER", "when_present": True, "module": False}, + "D": {"shift": 1 << 7, "desc": "DIE", "when_present": True, "module": False}, + "A": { + "shift": 1 << 8, + "desc": "OVERRIDDEN_ACPI_TABLE", + "when_present": True, + "module": False, + }, + "W": {"shift": 1 << 9, "desc": "WARN", "when_present": True, "module": False}, + "C": {"shift": 1 << 10, "desc": "CRAP", "when_present": True, "module": True}, + "I": { + "shift": 1 << 11, + "desc": "FIRMWARE_WORKAROUND", + "when_present": True, + "module": False, + }, + "O": {"shift": 1 << 12, "desc": "OOT_MODULE", "when_present": True, "module": True}, + "E": { + "shift": 1 << 13, + "desc": "UNSIGNED_MODULE", + "when_present": True, + "module": True, + }, + "L": { + "shift": 1 << 14, + "desc": "SOFTLOCKUP", + "when_present": True, + "module": False, + }, + "K": {"shift": 1 << 15, "desc": "LIVEPATCH", "when_present": True, "module": True}, + "X": {"shift": 1 << 16, "desc": "AUX", "when_present": True, "module": True}, + "T": {"shift": 1 << 17, "desc": "RANDSTRUCT", "when_present": True, "module": True}, + "N": {"shift": 1 << 18, "desc": "TEST", "when_present": True, "module": True}, +} +"""Flags used to taint kernel and modules, for debugging purposes. + +Map based on 6.12-rc5. + +Documentation : + - https://www.kernel.org/doc/Documentation/admin-guide/sysctl/kernel.rst#:~:text=guide/sysrq.rst.-,tainted,-%3D%3D%3D%3D%3D%3D%3D%0A%0ANon%2Dzero%20if + - https://www.kernel.org/doc/Documentation/admin-guide/tainted-kernels.rst#:~:text=More%20detailed%20explanation%20for%20tainting + - taint_flag kernel struct + - taint_flags kernel constant +""" From e1b343a284436ee4d92a7b8a6daf0a98dd01fdeb Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:33:54 +0100 Subject: [PATCH 030/348] add module taints parsing apis --- .../symbols/linux/extensions/__init__.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index aa3e8c675..89e2cde27 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -279,6 +279,77 @@ class module(generic.GenericIntelProcess): return None + def _module_flags_taints_pre_4_10_rc1(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Returns: + The raw taints string. + """ + taints_string = "" + for char, infos in linux_constants.TAINT_FLAGS.items(): + if infos["module"] and self.taints_value & infos["shift"]: + taints_string += char + + return taints_string + + def _module_flags_taints_post_4_10_rc1(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self.taint_flags_list): + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taint_flag.module and (self.taints_value & (1 << i)): + taints_string += c_true + elif taint_flag.module and c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self.taint_flags_list: + return self._module_flags_taints_post_4_10_rc1() + return self._module_flags_taints_pre_4_10_rc1() + + def get_taints_parsed(self) -> List[str]: + """Convert the module's taints string to a 1-1 descriptor mapping. + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for c in self.get_taints_as_plain_string(): + infos = linux_constants.TAINT_FLAGS.get(c) + if not infos: + comprehensive_taints.append(f"") + elif infos["when_present"]: + comprehensive_taints.append(infos["desc"]) + + return comprehensive_taints + @property def section_symtab(self): if self.has_member("kallsyms"): @@ -307,6 +378,17 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("module -> strtab: Unable to get strtab") + @property + def taints_value(self) -> int: + return self.taints + + @property + def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: + kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if kernel.has_symbol("taint_flags"): + return list(kernel.object_from_symbol("taint_flags")) + return None + class task_struct(generic.GenericIntelProcess): def add_process_layer( From c88ebe89270355d188770c74b39eb8acef1f3549 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:35:54 +0100 Subject: [PATCH 031/348] introduce modxview linux plugin --- .../framework/plugins/linux/modxview.py | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 volatility3/framework/plugins/linux/modxview.py diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py new file mode 100644 index 000000000..66b644164 --- /dev/null +++ b/volatility3/framework/plugins/linux/modxview.py @@ -0,0 +1,195 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import List, Dict, Set, Iterator +from volatility3.plugins.linux import lsmod, check_modules, hidden_modules +from volatility3.framework import interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.symbols.linux import extensions +from volatility3.framework.constants import architectures + +vollog = logging.getLogger(__name__) + + +class Modxview(interfaces.plugins.PluginInterface): + """Centralize lsmod, check_modules and hidden_modules results to efficiently + spot modules presence and taints.""" + + _version = (1, 0, 0) + _required_framework_version = (2, 11, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="check_modules", + plugin=check_modules.Check_modules, + version=(0, 0, 0), + ), + requirements.PluginRequirement( + name="hidden_modules", + plugin=hidden_modules.Hidden_modules, + version=(1, 0, 0), + ), + requirements.BooleanRequirement( + name="plain_taints", + description="Display the plain taints string for each module.", + optional=True, + default=False, + ), + ] + + @classmethod + def run_lsmod( + cls, context: interfaces.context.ContextInterface, kernel_name: str + ) -> List[extensions.module]: + """Wrapper for the lsmod plugin.""" + return list(lsmod.Lsmod.list_modules(context, kernel_name)) + + @classmethod + def run_check_modules( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + ) -> List[extensions.module]: + """Wrapper for the check_modules plugin. + Here, we extract the /sys/module/ list.""" + kernel = context.modules[kernel_name] + sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( + context, kernel_name + ) + + # Convert get_kset_modules() offsets back to module objects + return [ + kernel.object(object_type="module", offset=m_offset, absolute=True) + for m_offset in sysfs_modules.values() + ] + + @classmethod + def run_hidden_modules( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + known_modules_addresses: Set[int], + ) -> List[extensions.module]: + """Wrapper for the hidden_modules plugin.""" + modules_memory_boundaries = ( + hidden_modules.Hidden_modules.get_modules_memory_boundaries( + context, kernel_name + ) + ) + return list( + hidden_modules.Hidden_modules.get_hidden_modules( + context, + kernel_name, + known_modules_addresses, + modules_memory_boundaries, + ) + ) + + @classmethod + def flatten_run_modules_results( + cls, run_results: Dict[str, List[extensions.module]], deduplicate: bool = True + ) -> Iterator[extensions.module]: + """Flatten a dictionary mapping plugin names and modules list, to a single merged list. + This is useful to get a generic lookup list of all the detected modules. + + Args: + run_results: dictionary of plugin names mapping a list of detected modules + deduplicate: remove duplicate modules, based on their offsets + + Returns: + Iterator of modules objects + """ + seen_addresses = set() + for modules in run_results.values(): + for module in modules: + if deduplicate and module.vol.offset in seen_addresses: + continue + yield module + + @classmethod + def run_modules_scanners( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + run_hidden_modules: bool = True, + ) -> Dict[str, List[extensions.module]]: + """Run module scanning plugins and aggregate the results. + + Args: + run_hidden_modules: specify if the hidden_modules plugin should be run + Returns: + Dictionary mapping each plugin to its corresponding result + """ + + kernel = context.modules[kernel_name] + run_results = {} + run_results["lsmod"] = cls.run_lsmod(context, kernel_name) + run_results["check_modules"] = cls.run_check_modules(context, kernel_name) + if run_hidden_modules: + known_module_addresses = set( + context.layers[kernel.layer_name].canonicalize(module.vol.offset) + for module in run_results["lsmod"] + run_results["check_modules"] + ) + run_results["hidden_modules"] = cls.run_hidden_modules( + context, kernel_name, known_module_addresses + ) + + return run_results + + def _generator(self): + kernel_name = self.config["kernel"] + run_results = self.run_modules_scanners(self.context, kernel_name) + modules_offsets = {} + for key in ["lsmod", "check_modules", "hidden_modules"]: + modules_offsets[key] = set(module.vol.offset for module in run_results[key]) + + seen_addresses = set() + for modules_list in run_results.values(): + for module in modules_list: + if module.vol.offset in seen_addresses: + continue + seen_addresses.add(module.vol.offset) + + if self.config.get("plain_taints"): + taints = module.get_taints_as_plain_string() + else: + taints = ",".join(module.get_taints_parsed()) + + yield ( + 0, + ( + module.get_name() or NotAvailableValue(), + format_hints.Hex(module.vol.offset), + module.vol.offset in modules_offsets["lsmod"], + module.vol.offset in modules_offsets["check_modules"], + module.vol.offset in modules_offsets["hidden_modules"], + taints or NotAvailableValue(), + ), + ) + + def run(self): + columns = [ + ("Name", str), + ("Address", format_hints.Hex), + ("In /proc/modules", bool), + ("In /sys/module/", bool), + ("Hidden", bool), + ("Taints", str), + ] + + return TreeGrid( + columns, + self._generator(), + ) From 9440f53429a1f9c7d77d51eeb75c2b5938da040f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:05:25 +0100 Subject: [PATCH 032/348] use a dict of dataclasses for taint_flags --- .../framework/constants/linux/__init__.py | 112 +++++++----------- .../symbols/linux/extensions/__init__.py | 12 +- 2 files changed, 47 insertions(+), 77 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 9f25c9225..6cf8585f5 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -6,6 +6,7 @@ Linux-specific values that aren't found in debug symbols """ from enum import IntEnum, Flag +from dataclasses import dataclass KERNEL_NAME = "__kernel__" @@ -349,78 +350,47 @@ MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 MODULE_MINIMUM_SIZE = 4096 +@dataclass +class TaintFlag: + shift: int + desc: str + when_present: bool + module: bool + + TAINT_FLAGS = { - "P": { - "shift": 1 << 0, - "desc": "PROPRIETARY_MODULE", - "when_present": True, - "module": True, - }, - "G": { - "shift": 1 << 0, - "desc": "PROPRIETARY_MODULE", - "when_present": False, - "module": True, - }, - "F": { - "shift": 1 << 1, - "desc": "FORCED_MODULE", - "when_present": True, - "module": False, - }, - # CPU_OUT_OF_SPEC was TAINT_UNSAFE_SMP on < 3.15-rc1 : https://lore.kernel.org/linux-kernel//20140303080432.GA25489@localhost/t/#:~:text=liked%20your%20proposal%3A-,%3E%20Right,-%2C%20I%20was%20about - "S": { - "shift": 1 << 2, - "desc": "CPU_OUT_OF_SPEC", - "when_present": True, - "module": False, - }, - "R": { - "shift": 1 << 3, - "desc": "FORCED_RMMOD", - "when_present": True, - "module": False, - }, - "M": { - "shift": 1 << 4, - "desc": "MACHINE_CHECK", - "when_present": True, - "module": False, - }, - "B": {"shift": 1 << 5, "desc": "BAD_PAGE", "when_present": True, "module": False}, - "U": {"shift": 1 << 6, "desc": "USER", "when_present": True, "module": False}, - "D": {"shift": 1 << 7, "desc": "DIE", "when_present": True, "module": False}, - "A": { - "shift": 1 << 8, - "desc": "OVERRIDDEN_ACPI_TABLE", - "when_present": True, - "module": False, - }, - "W": {"shift": 1 << 9, "desc": "WARN", "when_present": True, "module": False}, - "C": {"shift": 1 << 10, "desc": "CRAP", "when_present": True, "module": True}, - "I": { - "shift": 1 << 11, - "desc": "FIRMWARE_WORKAROUND", - "when_present": True, - "module": False, - }, - "O": {"shift": 1 << 12, "desc": "OOT_MODULE", "when_present": True, "module": True}, - "E": { - "shift": 1 << 13, - "desc": "UNSIGNED_MODULE", - "when_present": True, - "module": True, - }, - "L": { - "shift": 1 << 14, - "desc": "SOFTLOCKUP", - "when_present": True, - "module": False, - }, - "K": {"shift": 1 << 15, "desc": "LIVEPATCH", "when_present": True, "module": True}, - "X": {"shift": 1 << 16, "desc": "AUX", "when_present": True, "module": True}, - "T": {"shift": 1 << 17, "desc": "RANDSTRUCT", "when_present": True, "module": True}, - "N": {"shift": 1 << 18, "desc": "TEST", "when_present": True, "module": True}, + "P": TaintFlag( + shift=1 << 0, desc="PROPRIETARY_MODULE", when_present=True, module=True + ), + "G": TaintFlag( + shift=1 << 0, desc="PROPRIETARY_MODULE", when_present=False, module=True + ), + "F": TaintFlag(shift=1 << 1, desc="FORCED_MODULE", when_present=True, module=False), + "S": TaintFlag( + shift=1 << 2, desc="CPU_OUT_OF_SPEC", when_present=True, module=False + ), + "R": TaintFlag(shift=1 << 3, desc="FORCED_RMMOD", when_present=True, module=False), + "M": TaintFlag(shift=1 << 4, desc="MACHINE_CHECK", when_present=True, module=False), + "B": TaintFlag(shift=1 << 5, desc="BAD_PAGE", when_present=True, module=False), + "U": TaintFlag(shift=1 << 6, desc="USER", when_present=True, module=False), + "D": TaintFlag(shift=1 << 7, desc="DIE", when_present=True, module=False), + "A": TaintFlag( + shift=1 << 8, desc="OVERRIDDEN_ACPI_TABLE", when_present=True, module=False + ), + "W": TaintFlag(shift=1 << 9, desc="WARN", when_present=True, module=False), + "C": TaintFlag(shift=1 << 10, desc="CRAP", when_present=True, module=True), + "I": TaintFlag( + shift=1 << 11, desc="FIRMWARE_WORKAROUND", when_present=True, module=False + ), + "O": TaintFlag(shift=1 << 12, desc="OOT_MODULE", when_present=True, module=True), + "E": TaintFlag( + shift=1 << 13, desc="UNSIGNED_MODULE", when_present=True, module=True + ), + "L": TaintFlag(shift=1 << 14, desc="SOFTLOCKUP", when_present=True, module=False), + "K": TaintFlag(shift=1 << 15, desc="LIVEPATCH", when_present=True, module=True), + "X": TaintFlag(shift=1 << 16, desc="AUX", when_present=True, module=True), + "T": TaintFlag(shift=1 << 17, desc="RANDSTRUCT", when_present=True, module=True), + "N": TaintFlag(shift=1 << 18, desc="TEST", when_present=True, module=True), } """Flags used to taint kernel and modules, for debugging purposes. diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 89e2cde27..42c1a470d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -287,8 +287,8 @@ class module(generic.GenericIntelProcess): The raw taints string. """ taints_string = "" - for char, infos in linux_constants.TAINT_FLAGS.items(): - if infos["module"] and self.taints_value & infos["shift"]: + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if taint_flag.module and self.taints_value & taint_flag.shift: taints_string += char return taints_string @@ -342,11 +342,11 @@ class module(generic.GenericIntelProcess): """ comprehensive_taints = [] for c in self.get_taints_as_plain_string(): - infos = linux_constants.TAINT_FLAGS.get(c) - if not infos: + taint_flag = linux_constants.TAINT_FLAGS.get(c) + if not taint_flag: comprehensive_taints.append(f"") - elif infos["when_present"]: - comprehensive_taints.append(infos["desc"]) + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) return comprehensive_taints From 9d08c4681ae1cf18ddf4bd53ff970f6a9bc26573 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:07:23 +0100 Subject: [PATCH 033/348] add module offset to seen_addresses --- volatility3/framework/plugins/linux/modxview.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 66b644164..f44984926 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -116,6 +116,7 @@ class Modxview(interfaces.plugins.PluginInterface): for module in modules: if deduplicate and module.vol.offset in seen_addresses: continue + seen_addresses.add(module.vol.offset) yield module @classmethod From b209ea36a284ae1a75ba18222cbe8db1c1eede4f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:12:52 +0100 Subject: [PATCH 034/348] remove slashes in columns --- volatility3/framework/plugins/linux/modxview.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index f44984926..d79f5e7a9 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -184,8 +184,8 @@ class Modxview(interfaces.plugins.PluginInterface): columns = [ ("Name", str), ("Address", format_hints.Hex), - ("In /proc/modules", bool), - ("In /sys/module/", bool), + ("In procfs", bool), + ("In sysfs", bool), ("Hidden", bool), ("Taints", str), ] From 2efb4e7d28d60a337aabae448de24da37c3feb7e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 5 Nov 2024 19:56:15 +1100 Subject: [PATCH 035/348] Merge branch 'develop' into linux_boottime_support --- .../framework/constants/linux/__init__.py | 8 + .../framework/plugins/linux/hidden_modules.py | 246 ++++++++++++++++++ .../framework/symbols/linux/__init__.py | 6 +- .../symbols/linux/extensions/__init__.py | 225 +++++++++------- 4 files changed, 383 insertions(+), 102 deletions(-) create mode 100644 volatility3/framework/plugins/linux/hidden_modules.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 7bfa9268d..6e49e6f37 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -344,3 +344,11 @@ class PT_FLAGS(Flag): # Boot time NSEC_PER_SEC = 1e9 + + +# Valid sizes for modules. Note that the Linux kernel does not define these values; they +# are based on empirical observations of typical memory allocations for kernel modules. +# We use this to verify that the found module falls within reasonable limits. +MODULE_MAXIMUM_CORE_SIZE = 20000000 +MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 +MODULE_MINIMUM_SIZE = 4096 diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py new file mode 100644 index 000000000..fd4b28943 --- /dev/null +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -0,0 +1,246 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import List, Set, Tuple, Iterable +from volatility3.framework import renderers, interfaces, exceptions, objects +from volatility3.framework.constants import architectures +from volatility3.framework.renderers import format_hints +from volatility3.framework.configuration import requirements +from volatility3.plugins.linux import lsmod + +vollog = logging.getLogger(__name__) + + +class Hidden_modules(interfaces.plugins.PluginInterface): + """Carves memory to find hidden kernel modules""" + + _required_framework_version = (2, 10, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + ] + + @staticmethod + def get_modules_memory_boundaries( + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Tuple[int]: + """Determine the boundaries of the module allocation area + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + + Returns: + A tuple containing the minimum and maximum addresses for the module allocation area. + """ + vmlinux = context.modules[vmlinux_module_name] + if vmlinux.has_symbol("mod_tree"): + # Kernel >= 5.19 58d208de3e8d87dbe196caf0b57cc58c7a3836ca + mod_tree = vmlinux.object_from_symbol("mod_tree") + modules_addr_min = mod_tree.addr_min + modules_addr_max = mod_tree.addr_max + elif vmlinux.has_symbol("module_addr_min"): + # 2.6.27 <= kernel < 5.19 3a642e99babe0617febb6f402e1e063479f489db + modules_addr_min = vmlinux.object_from_symbol("module_addr_min") + modules_addr_max = vmlinux.object_from_symbol("module_addr_max") + + if isinstance(modules_addr_min, objects.Void): + raise exceptions.VolatilityException( + "Your ISF symbols lack type information. You may need to update the" + "ISF using the latest version of dwarf2json" + ) + else: + raise exceptions.VolatilityException( + "Cannot find the module memory allocation area. Unsupported kernel" + ) + + return modules_addr_min, modules_addr_max + + @classmethod + def _get_module_address_alignment( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> int: + """Obtain the module memory address alignment. + + struct module is aligned to the L1 cache line, which is typically 64 bytes for most + common i386/AMD64/ARM64 configurations. In some cases, it can be 128 bytes, but this + will still work. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + + Returns: + The struct module alignment + """ + # FIXME: When dwarf2json/ISF supports type alignments. Read it directly from the type metadata + # Additionally, while 'context' and 'vmlinux_module_name' are currently unused, they will be + # essential for retrieving type metadata in the future. + return 64 + + @staticmethod + def _validate_alignment_patterns( + addresses: Iterable[int], + address_alignment: int, + ) -> bool: + """Check if the memory addresses meet our alignments patterns + + Args: + addresses: Iterable with the address values + address_alignment: Number of bytes for alignment validation + + Returns: + True if all the addresses meet the alignment + """ + return all(addr % address_alignment == 0 for addr in addresses) + + @classmethod + def get_hidden_modules( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + known_module_addresses: Set[int], + modules_memory_boundaries: Tuple, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Enumerate hidden modules by taking advantage of memory address alignment patterns + + This technique is much faster and uses less memory than the traditional scan method + in Volatility2, but it doesn't work with older kernels. + + From kernels 4.2 struct module allocation are aligned to the L1 cache line size. + In i386/amd64/arm64 this is typically 64 bytes. However, this can be changed in + the Linux kernel configuration via CONFIG_X86_L1_CACHE_SHIFT. The alignment can + also be obtained from the DWARF info i.e. DW_AT_alignment<64>, but dwarf2json + doesn't support this feature yet. + In kernels < 4.2, alignment attributes are absent in the struct module, meaning + alignment cannot be guaranteed. Therefore, for older kernels, it's better to use + the traditional scan technique. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + known_module_addresses: Set with known module addresses + modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. + Yields: + module objects + """ + vmlinux = context.modules[vmlinux_module_name] + vmlinux_layer = context.layers[vmlinux.layer_name] + + module_addr_min, module_addr_max = modules_memory_boundaries + module_address_alignment = cls._get_module_address_alignment( + context, vmlinux_module_name + ) + if not cls._validate_alignment_patterns( + known_module_addresses, module_address_alignment + ): + vollog.warning( + f"Module addresses aren't aligned to {module_address_alignment} bytes. " + "Switching to 1 byte aligment scan method." + ) + module_address_alignment = 1 + + mkobj_offset = vmlinux.get_type("module").relative_child_offset("mkobj") + mod_offset = vmlinux.get_type("module_kobject").relative_child_offset("mod") + offset_to_mkobj_mod = mkobj_offset + mod_offset + mod_member_template = vmlinux.get_type("module_kobject").child_template("mod") + mod_size = mod_member_template.size + mod_member_data_format = mod_member_template.data_format + + for module_addr in range( + module_addr_min, module_addr_max, module_address_alignment + ): + if module_addr in known_module_addresses: + continue + + try: + # This is just a pre-filter. Module readability and consistency are verified in module.is_valid() + self_referential_bytes = vmlinux_layer.read( + module_addr + offset_to_mkobj_mod, mod_size + ) + self_referential = objects.convert_data_to_value( + self_referential_bytes, int, mod_member_data_format + ) + if self_referential != module_addr: + continue + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ): + continue + + module = vmlinux.object("module", offset=module_addr, absolute=True) + if module and module.is_valid(): + yield module + + @classmethod + def get_lsmod_module_addresses( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Set[int]: + """Obtain a set the known module addresses from linux.lsmod plugin + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + + Returns: + A set containing known kernel module addresses + """ + vmlinux = context.modules[vmlinux_module_name] + vmlinux_layer = context.layers[vmlinux.layer_name] + + known_module_addresses = { + vmlinux_layer.canonicalize(module.vol.offset) + for module in lsmod.Lsmod.list_modules(context, vmlinux_module_name) + } + return known_module_addresses + + def _generator(self): + vmlinux_module_name = self.config["kernel"] + known_module_addresses = self.get_lsmod_module_addresses( + self.context, vmlinux_module_name + ) + modules_memory_boundaries = self.get_modules_memory_boundaries( + self.context, vmlinux_module_name + ) + for module in self.get_hidden_modules( + self.context, + vmlinux_module_name, + known_module_addresses, + modules_memory_boundaries, + ): + module_addr = module.vol.offset + module_name = module.get_name() or renderers.NotAvailableValue() + fields = (format_hints.Hex(module_addr), module_name) + yield (0, fields) + + def run(self): + if self.context.symbol_space.verify_table_versions( + "dwarf2json", lambda version, _: (not version) or version < (0, 8, 0) + ): + raise exceptions.SymbolSpaceError( + "Invalid symbol table, please ensure the ISF table produced by dwarf2json was created with version 0.8.0 or later" + ) + + headers = [ + ("Address", format_hints.Hex), + ("Name", str), + ] + return renderers.TreeGrid(headers, self._generator()) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 1f2812c10..3289775b6 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -615,7 +615,7 @@ class IDStorage(ABC): return nodep - def _iter_node(self, nodep, height) -> int: + def _iter_node(self, nodep, height) -> Iterator[int]: node = self.nodep_to_node(nodep) node_slots = node.slots for off in range(self.CHUNK_SIZE): @@ -632,7 +632,7 @@ class IDStorage(ABC): for child_node in self._iter_node(nodep, height - 1): yield child_node - def get_entries(self, root: interfaces.objects.ObjectInterface) -> int: + def get_entries(self, root: interfaces.objects.ObjectInterface) -> Iterator[int]: """Walks the tree data structure Args: @@ -818,7 +818,7 @@ class PageCache(object): self._page_cache = page_cache self._idstorage = IDStorage.choose_id_storage(context, kernel_module_name) - def get_cached_pages(self) -> interfaces.objects.ObjectInterface: + def get_cached_pages(self) -> Iterator[interfaces.objects.ObjectInterface]: """Returns all page cache contents Yields: diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index a9127907e..0099b17e1 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -14,12 +14,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion -from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY -from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS -from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS -from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES -from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES -from volatility3.framework.constants.linux import CAPABILITIES, PT_FLAGS, NSEC_PER_SEC +from volatility3.framework.constants import linux as linux_constants from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed @@ -33,112 +28,140 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._mod_mem_type = None # Initialize _mod_mem_type to None for memoization + def is_valid(self): + """Determine whether it is a valid module object by verifying the self-referential + in module_kobject. This also confirms that the module is actively allocated and + not a remnant of freed memory or a failed module load attempt by verifying the + module memory section sizes. + """ + layer = self._context.layers[self.vol.layer_name] + # Make sure the entire module content is readable + if not layer.is_valid(self.vol.offset, self.vol.size): + return False - @property - def mod_mem_type(self): + core_size = self.get_core_size() + core_text_size = self.get_core_text_size() + init_size = self.get_init_size() + if not ( + 0 < core_text_size <= linux_constants.MODULE_MAXIMUM_CORE_TEXT_SIZE + and 0 < core_size <= linux_constants.MODULE_MAXIMUM_CORE_SIZE + and core_size + init_size >= linux_constants.MODULE_MINIMUM_SIZE + ): + return False + + if not ( + self.mkobj + and self.mkobj.mod + and self.mkobj.mod.is_readable() + and self.mkobj.mod == self.vol.offset + ): + return False + + return True + + @functools.cached_property + def mod_mem_type(self) -> Dict: """Return the mod_mem_type enum choices if available or an empty dict if not""" # mod_mem_type and module_memory were added in kernel 6.4 which replaces # module_layout for storing the information around core_layout etc. # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information + symbol_table_name = self.get_symbol_table_name() + mod_mem_type_symname = symbol_table_name + constants.BANG + "mod_mem_type" + symbol_space = self._context.symbol_space + try: + mod_mem_type = symbol_space.get_enumeration(mod_mem_type_symname).choices + except exceptions.SymbolError: + mod_mem_type = {} + vollog.debug( + "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" + ) - if self._mod_mem_type is None: - try: - self._mod_mem_type = self._context.symbol_space.get_enumeration( - self.get_symbol_table_name() + constants.BANG + "mod_mem_type" - ).choices - except exceptions.SymbolError: - vollog.debug( - "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" - ) - # set to empty dict to show that the enum was not found, and so shouldn't be searched for again - self._mod_mem_type = {} - return self._mod_mem_type + return mod_mem_type + + def _get_mem_type(self, mod_mem_type_name): + module_mem_index = self.mod_mem_type.get(mod_mem_type_name) + if module_mem_index is None: + raise AttributeError(f"Unknown module memory type '{mod_mem_type_name}'") + + if not (0 <= module_mem_index < self.mem.count): + raise AttributeError( + f"Invalid module memory type index '{module_mem_index}'" + ) + + return self.mem[module_mem_index] + + def _get_mem_size(self, mod_mem_type_name): + return self._get_mem_type(mod_mem_type_name).size + + def _get_mem_base(self, mod_mem_type_name): + return self._get_mem_type(mod_mem_type_name).base def get_module_base(self): if self.has_member("mem"): # kernels 6.4+ - try: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base - except KeyError: - raise AttributeError( - "module -> get_module_base: Unable to get module base. Cannot read base from MOD_TEXT." - ) + return self._get_mem_base("MOD_TEXT") elif self.has_member("core_layout"): return self.core_layout.base elif self.has_member("module_core"): return self.module_core - raise AttributeError("module -> get_module_base: Unable to get module base") + + raise AttributeError("Unable to get module base") def get_init_size(self): if self.has_member("mem"): # kernels 6.4+ - try: - return ( - self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size - + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size - ) - except KeyError: - raise AttributeError( - "module -> get_init_size: Unable to determine .init section size of module. Cannot read size of MOD_INIT_TEXT, MOD_INIT_DATA, and MOD_INIT_RODATA" - ) + return ( + self._get_mem_size("MOD_INIT_TEXT") + + self._get_mem_size("MOD_INIT_DATA") + + self._get_mem_size("MOD_INIT_RODATA") + ) elif self.has_member("init_layout"): return self.init_layout.size elif self.has_member("init_size"): return self.init_size - raise AttributeError( - "module -> get_init_size: Unable to determine .init section size of module" - ) + + raise AttributeError("Unable to determine .init section size of module") def get_core_size(self): if self.has_member("mem"): # kernels 6.4+ - try: - return ( - self.mem[self.mod_mem_type["MOD_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_DATA"]].size - + self.mem[self.mod_mem_type["MOD_RODATA"]].size - + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size - ) - except KeyError: - raise AttributeError( - "module -> get_core_size: Unable to determine core size of module. Cannot read size of MOD_TEXT, MOD_DATA, MOD_RODATA, and MOD_RO_AFTER_INIT." - ) + return ( + self._get_mem_size("MOD_TEXT") + + self._get_mem_size("MOD_DATA") + + self._get_mem_size("MOD_RODATA") + + self._get_mem_size("MOD_RO_AFTER_INIT") + ) elif self.has_member("core_layout"): return self.core_layout.size elif self.has_member("core_size"): return self.core_size - raise AttributeError( - "module -> get_core_size: Unable to determine core size of module" - ) + + raise AttributeError("Unable to determine core size of module") + + def get_core_text_size(self): + if self.has_member("mem"): # kernels 6.4+ + return self._get_mem_size("MOD_TEXT") + elif self.has_member("core_layout"): + return self.core_layout.text_size + elif self.has_member("core_text_size"): + return self.core_text_size + + raise AttributeError("Unable to determine core text size of module") def get_module_core(self): if self.has_member("mem"): # kernels 6.4+ - try: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base - except KeyError: - raise AttributeError( - "module -> get_module_core: Unable to get module core. Cannot read base from MOD_TEXT." - ) + return self._get_mem_base("MOD_TEXT") elif self.has_member("core_layout"): return self.core_layout.base elif self.has_member("module_core"): return self.module_core - raise AttributeError("module -> get_module_core: Unable to get module core") + raise AttributeError("Unable to get module core") def get_module_init(self): if self.has_member("mem"): # kernels 6.4+ - try: - return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base - except KeyError: - raise AttributeError( - "module -> get_module_core: Unable to get module init. Cannot read base from MOD_INIT_TEXT." - ) + return self._get_mem_base("MOD_INIT_TEXT") elif self.has_member("init_layout"): return self.init_layout.base elif self.has_member("module_init"): return self.module_init - raise AttributeError("module -> get_module_init: Unable to get module init") + raise AttributeError("Unable to get module init") def get_name(self): """Get the name of the module as a string""" @@ -340,7 +363,7 @@ class task_struct(generic.GenericIntelProcess): Returns: bool: True, if this task is a kernel thread. Otherwise, False. """ - return (self.flags & constants.linux.PF_KTHREAD) != 0 + return (self.flags & linux_constants.PF_KTHREAD) != 0 @property def is_thread_group_leader(self) -> bool: @@ -417,7 +440,11 @@ class task_struct(generic.GenericIntelProcess): def get_ptrace_tracee_flags(self) -> Optional[str]: """Returns a string with the ptrace flags""" - return PT_FLAGS(self.ptrace).flags if self.is_being_ptraced else None + return ( + linux_constants.PT_FLAGS(self.ptrace).flags + if self.is_being_ptraced + else None + ) def _get_task_start_time(self) -> datetime.timedelta: """Returns the task's monotonic start_time as a timedelta. @@ -1492,7 +1519,7 @@ class vfsmount(objects.StructType): bool: 'True' if the given argument points to the the same 'vfsmount' as 'self'. """ - if type(vfsmount_ptr) == objects.Pointer: + if isinstance(vfsmount_ptr, objects.Pointer): return self.vol.offset == vfsmount_ptr else: raise exceptions.VolatilityException( @@ -1715,18 +1742,18 @@ class socket(objects.StructType): def get_state(self): socket_state_idx = self.state - if 0 <= socket_state_idx < len(SOCKET_STATES): - return SOCKET_STATES[socket_state_idx] + if 0 <= socket_state_idx < len(linux_constants.SOCKET_STATES): + return linux_constants.SOCKET_STATES[socket_state_idx] class sock(objects.StructType): def get_family(self): family_idx = self.__sk_common.skc_family - if 0 <= family_idx < len(SOCK_FAMILY): - return SOCK_FAMILY[family_idx] + if 0 <= family_idx < len(linux_constants.SOCK_FAMILY): + return linux_constants.SOCK_FAMILY[family_idx] def get_type(self): - return SOCK_TYPES.get(self.sk_type, "") + return linux_constants.SOCK_TYPES.get(self.sk_type, "") def get_inode(self): if not self.sk_socket: @@ -1760,8 +1787,8 @@ class unix_sock(objects.StructType): # Unix socket states reuse (a subset) of the inet_sock states contants if self.sk.get_type() == "STREAM": state_idx = self.sk.__sk_common.skc_state - if 0 <= state_idx < len(TCP_STATES): - return TCP_STATES[state_idx] + if 0 <= state_idx < len(linux_constants.TCP_STATES): + return linux_constants.TCP_STATES[state_idx] else: # Return the generic socket state return self.sk.sk_socket.get_state() @@ -1773,15 +1800,15 @@ class unix_sock(objects.StructType): class inet_sock(objects.StructType): def get_family(self): family_idx = self.sk.__sk_common.skc_family - if 0 <= family_idx < len(SOCK_FAMILY): - return SOCK_FAMILY[family_idx] + if 0 <= family_idx < len(linux_constants.SOCK_FAMILY): + return linux_constants.SOCK_FAMILY[family_idx] def get_protocol(self): # If INET6 family and a proto is defined, we use that specific IPv6 protocol. # Otherwise, we use the standard IP protocol. - protocol = IP_PROTOCOLS.get(self.sk.sk_protocol) + protocol = linux_constants.IP_PROTOCOLS.get(self.sk.sk_protocol) if self.get_family() == "AF_INET6": - protocol = IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) + protocol = linux_constants.IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) return protocol def get_state(self): @@ -1789,8 +1816,8 @@ class inet_sock(objects.StructType): if self.sk.get_type() == "STREAM": state_idx = self.sk.__sk_common.skc_state - if 0 <= state_idx < len(TCP_STATES): - return TCP_STATES[state_idx] + if 0 <= state_idx < len(linux_constants.TCP_STATES): + return linux_constants.TCP_STATES[state_idx] else: # Return the generic socket state return self.sk.sk_socket.get_state() @@ -1873,8 +1900,8 @@ class inet_sock(objects.StructType): class netlink_sock(objects.StructType): def get_protocol(self): protocol_idx = self.sk.sk_protocol - if 0 <= protocol_idx < len(NETLINK_PROTOCOLS): - return NETLINK_PROTOCOLS[protocol_idx] + if 0 <= protocol_idx < len(linux_constants.NETLINK_PROTOCOLS): + return linux_constants.NETLINK_PROTOCOLS[protocol_idx] def get_state(self): # Return the generic socket state @@ -1916,8 +1943,8 @@ class packet_sock(objects.StructType): eth_proto = socket_module.htons(self.num) if eth_proto == 0: return None - elif eth_proto in ETH_PROTOCOLS: - return ETH_PROTOCOLS[eth_proto] + elif eth_proto in linux_constants.ETH_PROTOCOLS: + return linux_constants.ETH_PROTOCOLS[eth_proto] else: return f"0x{eth_proto:x}" @@ -1929,13 +1956,13 @@ class packet_sock(objects.StructType): class bt_sock(objects.StructType): def get_protocol(self): type_idx = self.sk.sk_protocol - if 0 <= type_idx < len(BLUETOOTH_PROTOCOLS): - return BLUETOOTH_PROTOCOLS[type_idx] + if 0 <= type_idx < len(linux_constants.BLUETOOTH_PROTOCOLS): + return linux_constants.BLUETOOTH_PROTOCOLS[type_idx] def get_state(self): state_idx = self.sk.__sk_common.skc_state - if 0 <= state_idx < len(BLUETOOTH_STATES): - return BLUETOOTH_STATES[state_idx] + if 0 <= state_idx < len(linux_constants.BLUETOOTH_STATES): + return linux_constants.BLUETOOTH_STATES[state_idx] class xdp_sock(objects.StructType): @@ -2053,7 +2080,7 @@ class kernel_cap_struct(objects.StructType): Returns: int: The latest capability ID supported by the framework. """ - return len(CAPABILITIES) - 1 + return len(linux_constants.CAPABILITIES) - 1 def get_kernel_cap_full(self) -> int: """Return the maximum value allowed for this kernel for a capability @@ -2082,7 +2109,7 @@ class kernel_cap_struct(objects.StructType): """ capabilities = [] - for bit, name in enumerate(CAPABILITIES): + for bit, name in enumerate(linux_constants.CAPABILITIES): if capabilities_bitfield & (1 << bit) != 0: capabilities.append(name) @@ -2143,10 +2170,10 @@ class kernel_cap_struct(objects.StructType): Returns: bool: "True" if the given capability is enabled. """ - if capability not in CAPABILITIES: + if capability not in linux_constants.CAPABILITIES: raise AttributeError(f"Unknown capability with name '{capability}'") - cap_value = 1 << CAPABILITIES.index(capability) + cap_value = 1 << linux_constants.CAPABILITIES.index(capability) return cap_value & self.get_capabilities() != 0 From c0fa2cfcd67d676dab100150daad2f93948c641b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 5 Nov 2024 20:02:55 +1100 Subject: [PATCH 036/348] Linux: Boottime API: User linux_constanst import --- .../symbols/linux/extensions/__init__.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0099b17e1..d05c34304 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2217,12 +2217,12 @@ class Timespec64Abstract(abc.ABC): # Based on ns_to_timespec64() if nsec > 0: - tv_sec = nsec // NSEC_PER_SEC - tv_nsec = nsec % NSEC_PER_SEC + tv_sec = nsec // linux_constants.NSEC_PER_SEC + tv_nsec = nsec % linux_constants.NSEC_PER_SEC elif nsec < 0: - tv_sec = -((-nsec - 1) // NSEC_PER_SEC) - 1 - rem = (-nsec - 1) % NSEC_PER_SEC - tv_nsec = NSEC_PER_SEC - rem - 1 + tv_sec = -((-nsec - 1) // linux_constants.NSEC_PER_SEC) - 1 + rem = (-nsec - 1) % linux_constants.NSEC_PER_SEC + tv_nsec = linux_constants.NSEC_PER_SEC - rem - 1 else: tv_sec = tv_nsec = 0 @@ -2233,13 +2233,15 @@ class Timespec64Abstract(abc.ABC): # pylint: disable=E1101 return conversion.unixtime_to_datetime( - self.tv_sec + self.tv_nsec / NSEC_PER_SEC + self.tv_sec + self.tv_nsec / linux_constants.NSEC_PER_SEC ) def to_timedelta(self) -> datetime.timedelta: """Converts this Timespec64Abstract subclass object to timedelta""" # pylint: disable=E1101 - return datetime.timedelta(seconds=self.tv_sec + self.tv_nsec / NSEC_PER_SEC) + return datetime.timedelta( + seconds=self.tv_sec + self.tv_nsec / linux_constants.NSEC_PER_SEC + ) def __add__(self, other) -> "Timespec64Concrete": """Returns a new Timespec64Concrete object that sums the current values with those @@ -2282,12 +2284,12 @@ class Timespec64Abstract(abc.ABC): # Based on kernel's set_normalized_timespec64() # pylint: disable=E1101 - while self.tv_nsec >= NSEC_PER_SEC: - self.tv_nsec -= NSEC_PER_SEC + while self.tv_nsec >= linux_constants.NSEC_PER_SEC: + self.tv_nsec -= linux_constants.NSEC_PER_SEC self.tv_sec += 1 while self.tv_nsec < 0: - self.tv_nsec += NSEC_PER_SEC + self.tv_nsec += linux_constants.NSEC_PER_SEC self.tv_sec -= 1 From 0ad8054f125c360afadcd25db70c1f67d8265e57 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 7 Nov 2024 20:26:26 +1100 Subject: [PATCH 037/348] intel layer: minor improve lru_cache argument --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 2b0df5372..33d5432cc 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -252,7 +252,7 @@ class Intel(linear.LinearlyMappedLayer): return entry, position - @functools.lru_cache(1025) + @functools.lru_cache(maxsize=1025) def _get_valid_table(self, base_address: int) -> Optional[bytes]: """Extracts the table, validates it and returns it if it's valid.""" table = self._context.layers.read( From 9ae7c2bb109681bbd8b6cffc5a1f5b7f1dfa71cf Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 7 Nov 2024 20:28:21 +1100 Subject: [PATCH 038/348] Data layer interface: Convert address_mask to a cached property --- volatility3/framework/interfaces/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index e2a68780a..78687d8d5 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -136,7 +136,7 @@ class DataLayerInterface( def minimum_address(self) -> int: """Returns the minimum valid address of the space.""" - @property + @functools.cached_property def address_mask(self) -> int: """Returns a mask which encapsulates all the active bits of an address for this layer.""" From 73d4f2fc88e15ac038f962cb5f5b11acf7855091 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 7 Nov 2024 20:31:20 +1100 Subject: [PATCH 039/348] Linux: Add support for PROT_NONE, Intel Side Channel Vulnerability L1TF changes and fix _maxphyaddr in x86-64 --- volatility3/framework/automagic/linux.py | 6 +- .../framework/constants/linux/__init__.py | 8 ++ volatility3/framework/layers/intel.py | 100 ++++++++++++++++-- 3 files changed, 105 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 52a73f45a..27e07e564 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -80,14 +80,14 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): context, table_name, layer_name, progress_callback=progress_callback ) - layer_class: Type = intel.Intel if "init_top_pgt" in table.symbols: - layer_class = intel.Intel32e + layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_top_pgt" elif "init_level4_pgt" in table.symbols: - layer_class = intel.Intel32e + layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" else: + layer_class = intel.LinuxIntel dtb_symbol_name = "swapper_pg_dir" dtb = cls.virtual_to_physical_address( diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 7c485d3c3..303c534ed 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -11,6 +11,14 @@ KERNEL_NAME = "__kernel__" """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" +# Translation Layer constants +PAGE_BIT_PRESENT = 0 +PAGE_BIT_PSE = 7 # Page Size Extension: 4 MB (or 2MB) page +PAGE_BIT_PROTNONE = 8 +PAGE_BIT_PAT_LARGE = 12 # 2MB or 1GB pages +PAGE_PRESENT = 1 << PAGE_BIT_PRESENT +PAGE_PROTNONE = 1 << PAGE_BIT_PROTNONE + # include/linux/sched.h PF_KTHREAD = 0x00200000 # I'm a kernel thread diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 33d5432cc..e6d20d992 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -13,6 +13,7 @@ from volatility3 import classproperty from volatility3.framework import exceptions, interfaces, constants from volatility3.framework.configuration import requirements from volatility3.framework.layers import linear +from volatility3.framework.constants import linux as linux_constants vollog = logging.getLogger(__name__) @@ -163,12 +164,17 @@ class Intel(linear.LinearlyMappedLayer): entry, f"Page Fault at entry {hex(entry)} in page entry", ) - page = self._mask(entry, self._maxphyaddr - 1, position + 1) | self._mask( - offset, position, 0 - ) + + pfn = self.pte_pfn(entry) + page_offset = self._mask(offset, position, 0) + page = pfn << self.page_shift | page_offset return page, 1 << (position + 1), self._base_layer + def pte_pfn(self, entry: int) -> int: + """Extracts the page frame number (PFN) from the page table entry (PTE) entry""" + return entry >> self.page_shift + def _translate_entry(self, offset: int) -> Tuple[int, int]: """Translates a specific offset based on paging tables. @@ -203,10 +209,10 @@ class Intel(linear.LinearlyMappedLayer): "Page Fault at entry " + hex(entry) + " in table " + name, ) # Check if we're a large page - if large_page and (entry & (1 << 7)): + if large_page and (entry & (1 << linux_constants.PAGE_BIT_PSE)): # Mask off the PAT bit - if entry & (1 << 12): - entry -= 1 << 12 + if entry & (1 << linux_constants.PAGE_BIT_PAT_LARGE): + entry -= 1 << linux_constants.PAGE_BIT_PAT_LARGE # We're a large page, the rest is finished below # If we want to implement PSE-36, it would need to be done here break @@ -501,3 +507,85 @@ class WindowsIntel32e(WindowsMixin, Intel32e): def _translate(self, offset: int) -> Tuple[int, int, str]: return self._translate_swap(self, offset, self._bits_per_register // 2) + + +class LinuxMixin(Intel): + @functools.cached_property + def register_mask(self) -> int: + return (1 << self._bits_per_register) - 1 + + @functools.cached_property + def physical_mask(self) -> int: + # From kernels 4.18 the physical mask is dynamic: See AMD SME, Intel Multi-Key Total + # Memory Encryption and CONFIG_DYNAMIC_PHYSICAL_MASK: 94d49eb30e854c84d1319095b5dd0405a7da9362 + physical_mask = (1 << self._maxphyaddr) - 1 + # TODO: Come back once SME support is available in the framework + return physical_mask + + @functools.cached_property + def page_mask(self) -> int: + # Note that within the Intel class it's a class method. However, since it uses + # complement operations and we are working in Python, it would be more careful to + # limit it to the architecture's pointer size. + return ~(self.page_size - 1) & self.register_mask + + @functools.cached_property + def physical_page_mask(self) -> int: + return self.page_mask & self.physical_mask + + @functools.cached_property + def pte_pfn_mask(self) -> int: + return self.physical_page_mask + + @functools.cached_property + def pte_flags_mask(self) -> int: + return ~self.pte_pfn_mask & self.register_mask + + def pte_flags(self, pte) -> int: + return pte & self.pte_flags_mask + + def is_pte_present(self, entry: int) -> bool: + return ( + self.pte_flags(entry) + & (linux_constants.PAGE_PRESENT | linux_constants.PAGE_PROTNONE) + ) != 0 + + def _page_is_valid(self, entry: int) -> bool: + # Overrides the Intel static method with the Linux-specific implementation + return self.is_pte_present(entry) + + def pte_needs_invert(self, entry) -> bool: + # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted + return not (entry & linux_constants.PAGE_PRESENT) + + def protnone_mask(self, entry: int) -> int: + """Gets a mask to XOR with the page table entry to get the correct PFN""" + return ~0 & self.register_mask if self.pte_needs_invert(entry) else 0 + + def pte_pfn(self, entry: int) -> int: + """Extracts the page frame number from the page table entry""" + pfn = entry ^ self.protnone_mask(entry) + return (pfn & self.pte_pfn_mask) >> self.page_shift + + +class LinuxIntel(LinuxMixin, Intel): + pass + + +class LinuxIntelPAE(LinuxMixin, IntelPAE): + pass + + +class LinuxIntel32e(LinuxMixin, Intel32e): + # In the Linux kernel, the __PHYSICAL_MASK_SHIFT is a mask used to extract the + # physical address from a PTE. In Volatility3, this is referred to as _maxphyaddr. + # + # Until kernel version 4.17, Linux x86-64 used a 46-bit mask. With commit + # b83ce5ee91471d19c403ff91227204fb37c95fb2, this was extended to 52 bits, + # applying to both 4 and 5-level page tables. + # + # We initially used 52 bits for all Intel 64-bit systems, but this produced incorrect + # results for PROT_NONE pages. Since the mask value is defined by a preprocessor macro, + # it's difficult to detect the exact bit shift used in the current kernel. + # Using 46 bits has proven reliable for our use case, as seen in tools like crashtool. + _maxphyaddr = 46 From 20317e0d13d049070201562a1617ea89256879dc Mon Sep 17 00:00:00 2001 From: Joren Vrancken Date: Thu, 7 Nov 2024 18:05:36 +0100 Subject: [PATCH 040/348] Improve windows.amcache plugin description --- volatility3/framework/plugins/windows/amcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/amcache.py b/volatility3/framework/plugins/windows/amcache.py index 8d0a4769e..1e918d61c 100644 --- a/volatility3/framework/plugins/windows/amcache.py +++ b/volatility3/framework/plugins/windows/amcache.py @@ -215,7 +215,7 @@ def _get_datetime_str_value( class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): - """Scans for windows services.""" + """Extract information on executed applications from the AmCache.""" _required_framework_version = (2, 0, 0) _version = (1, 0, 0) From 6494260d916838c8f1a854450e217049dff015bb Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 7 Nov 2024 23:33:52 +0000 Subject: [PATCH 041/348] Core: Limit capstone to compatible versions --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bd61edf14..e0d366391 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,8 @@ yara-python>=3.8.0 # This is required for several plugins that perform malware analysis and disassemble code. # It can also improve accuracy of Windows 8 and later memory samples. -capstone>=3.0.5 +# FIXME: Version 6.0.0 is incompatible (#1336) so we'll need an adaptor at some point +capstone>=3.0.5,<6.0.0 # This is required by plugins that decrypt passwords, password hashes, etc. pycryptodome @@ -19,4 +20,4 @@ leechcorepyc>=2.4.0; sys_platform != 'darwin' # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage gcsfs>=2023.1.0 -s3fs>=2023.1.0 \ No newline at end of file +s3fs>=2023.1.0 From 74cf394ad0adb65d9cc149dae0451d7bee3d76a8 Mon Sep 17 00:00:00 2001 From: Joren Vrancken Date: Fri, 8 Nov 2024 10:06:59 +0100 Subject: [PATCH 042/348] Print plugin description on plugin --help --- volatility3/cli/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index b4e993dac..75b62abf6 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -367,7 +367,9 @@ class CommandLine: ) for plugin in sorted(plugin_list): plugin_parser = subparser.add_parser( - plugin, help=plugin_list[plugin].__doc__ + plugin, + help=plugin_list[plugin].__doc__, + description=plugin_list[plugin].__doc__, ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) From b37ba26dd62a9cc85c89e174c20f1afe549eb753 Mon Sep 17 00:00:00 2001 From: Joren Vrancken Date: Fri, 8 Nov 2024 10:19:05 +0100 Subject: [PATCH 043/348] Fix typo in SuspiciousThreads --- volatility3/framework/plugins/windows/suspicious_threads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspicious_threads.py b/volatility3/framework/plugins/windows/suspicious_threads.py index 2679b191a..4bfb6baa5 100644 --- a/volatility3/framework/plugins/windows/suspicious_threads.py +++ b/volatility3/framework/plugins/windows/suspicious_threads.py @@ -13,11 +13,11 @@ from volatility3.plugins.windows import pslist, threads, vadinfo, thrdscan vollog = logging.getLogger(__name__) -class SupsiciousThreads(interfaces.plugins.PluginInterface): +class SuspiciousThreads(interfaces.plugins.PluginInterface): """Lists suspicious userland process threads""" _required_framework_version = (2, 4, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 1fd51772600dc14ad6893e76362c9b8f599be180 Mon Sep 17 00:00:00 2001 From: lesander <4174509+lesander@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:33:20 +0100 Subject: [PATCH 044/348] move dependency definitions to pyproject.toml and bump minimal cpython version to 3.9.0 since 3.8.0 is EOL. --- README.md | 48 +++++++++++++--------------------------- mypy.ini | 4 ---- pyproject.toml | 42 ++++++++++++++++++++++++++++++----- requirements-dev.txt | 9 -------- requirements-minimal.txt | 2 -- requirements.txt | 23 ------------------- setup.py | 24 -------------------- 7 files changed, 51 insertions(+), 101 deletions(-) delete mode 100644 mypy.ini delete mode 100644 requirements-dev.txt delete mode 100644 requirements-minimal.txt delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/README.md b/README.md index 1463c2bde..78eb7c491 100644 --- a/README.md +++ b/README.md @@ -18,62 +18,44 @@ the Volatility Software License (VSL). See the [LICENSE](https://www.volatilityfoundation.org/license/vsl-v1.0) file for more details. -## Requirements +## Installing -Volatility 3 requires Python 3.8.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.9.0 or later and is published on the [PyPi registry](https://pypi.org/project/volatility3). ```shell -pip3 install -r requirements-minimal.txt +pip install volatility3 ``` -Alternately, the minimal packages will be installed automatically when Volatility 3 is installed using pip. However, as noted in the Quick Start section below, Volatility 3 does not *need* to be installed prior to using it. +If you want to use the latest development version of Volatility 3 we recommend you manually clone this repository and install an editable version of the project. +We recommend you use a virtual environment to keep installed dependencies separate from system packages. -```shell -pip3 install . -``` - -To enable the full range of Volatility 3 functionality, use a command like the one below. For partial functionality, comment out any unnecessary packages in [requirements.txt](requirements.txt) prior to running the command. - -```shell -pip3 install -r requirements.txt -``` - -## Downloading Volatility - -The latest stable version of Volatility will always be the stable branch of the GitHub repository. You can get the latest version of the code using the following command: +The latest stable version of Volatility will always be the `stable` branch of the GitHub repository. The default branch is `develop`. ```shell git clone https://github.com/volatilityfoundation/volatility3.git +cd volatility3/ +python3 -m venv venv && . venv/bin/activate +pip install -e .[dev] ``` ## Quick Start -1. Clone the latest version of Volatility from GitHub: - - ```shell - git clone https://github.com/volatilityfoundation/volatility3.git - ``` +1. Install Volatility 3 as documented in the Installing section of the readme. 2. See available options: ```shell - python3 vol.py -h + vol -h ``` -3. To get more information on a Windows memory sample and to make sure -Volatility supports that sample type, run -`python3 vol.py -f windows.info` - - Example: +3. To get more information on a Windows memory sample and to make sure Volatility supports that sample type, run `vol -f windows.info`: ```shell - python3 vol.py -f /home/user/samples/stuxnet.vmem windows.info + vol -f /home/user/samples/stuxnet.vmem windows.info ``` -4. Run some other plugins. The `-f` or `--single-location` is not strictly -required, but most plugins expect a single sample. Some also -require/accept other options. Run `python3 vol.py -h` -for more information on a particular command. +4. Run some other plugins. The `-f` or `--single-location` is not strictly required, but most plugins expect a single sample. +Some also require/accept other options. Run `vol -h` for more information on a particular command. ## Symbol Tables diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 6fb9f9ed3..000000000 --- a/mypy.ini +++ /dev/null @@ -1,4 +0,0 @@ -[mypy] -mypy_path = ./stubs -show_traceback = True -ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 2e1636a43..91848e1fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,15 +6,39 @@ readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, ] -requires-python = ">=3.8.0" +requires-python = ">=3.9.0" license = { text = "VSL" } dynamic = ["dependencies", "optional-dependencies", "version"] +dependencies = [ + "pefile>=2024.8.26", +] + +[project.optional-dependencies] +full = [ + "yara-python>=4.5.1,<5", + "capstone>=5.0.3,<6", + "pycryptodome>=3.21.0,<4", + "leechcorepyc>=2.19.2,<3; sys_platform != 'darwin'", +] + +cloud = [ + "gcsfs>=2024.10.0", + "s3fs>=2024.10.0", +] + +dev = [ + "volatility3[full,cloud]", + "jsonschema>=4.23.0,<5", + "pyinstaller>=6.11.0,<7", + "pyinstaller-hooks-contrib>=2024.9", +] + [project.urls] -Homepage = "https://github.com/volatilityfoundation/volatility3/" -"Bug Tracker" = "https://github.com/volatilityfoundation/volatility3/issues" -Documentation = "https://volatility3.readthedocs.io/" -"Source Code" = "https://github.com/volatilityfoundation/volatility3" +homepage = "https://github.com/volatilityfoundation/volatility3/" +documentation = "https://volatility3.readthedocs.io/" +repository = "https://github.com/volatilityfoundation/volatility3" +issues = "https://github.com/volatilityfoundation/volatility3/issues" [project.scripts] vol = "volatility3.cli:main" @@ -22,11 +46,17 @@ volshell = "volatility3.cli.volshell:main" [tool.setuptools.dynamic] version = { attr = "volatility3.framework.constants._version.PACKAGE_VERSION" } -dependencies = { file = "requirements-minimal.txt" } [tool.setuptools.packages.find] include = ["volatility3*"] +[tool.mypy] +mypy_path = "./stubs" +show_traceback = true + +[tool.mypy.overrides] +ignore_missing_imports = true + [build-system] requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index ae3482290..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ --r requirements.txt - -# This can improve error messages regarding improperly configured ISF files, -# but is only recommended for development -jsonschema>=2.3.0 - -# Used to build executable file -pyinstaller>=6.5.0 -pyinstaller-hooks-contrib>=2024.3 \ No newline at end of file diff --git a/requirements-minimal.txt b/requirements-minimal.txt deleted file mode 100644 index c030b332d..000000000 --- a/requirements-minimal.txt +++ /dev/null @@ -1,2 +0,0 @@ -# These packages are required for core functionality. -pefile>=2023.2.7 #foo \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e0d366391..000000000 --- a/requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Include the minimal requirements --r requirements-minimal.txt - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 - -# This is required for several plugins that perform malware analysis and disassemble code. -# It can also improve accuracy of Windows 8 and later memory samples. -# FIXME: Version 6.0.0 is incompatible (#1336) so we'll need an adaptor at some point -capstone>=3.0.5,<6.0.0 - -# This is required by plugins that decrypt passwords, password hashes, etc. -pycryptodome - -# This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0; sys_platform != 'darwin' - -# This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage -gcsfs>=2023.1.0 -s3fs>=2023.1.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 3af033160..000000000 --- a/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -import setuptools - - -def get_requires(filename): - requirements = [] - with open(filename, "r", encoding="utf-8") as fh: - for line in fh.readlines(): - stripped_line = line.strip() - if stripped_line == "" or stripped_line.startswith(("#", "-r")): - continue - requirements.append(stripped_line) - return requirements - - -setuptools.setup( - extras_require={ - "dev": get_requires("requirements-dev.txt"), - "full": get_requires("requirements.txt"), - }, -) From a04c04c4e3789008987ef18207ed242ff606ba1e Mon Sep 17 00:00:00 2001 From: lesander <4174509+lesander@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:03:08 +0100 Subject: [PATCH 045/348] fix ci --- .github/workflows/install.yml | 6 +----- .github/workflows/test.yaml | 6 ++---- .readthedocs.yml | 5 ++++- MANIFEST.in | 2 +- doc/requirements.txt | 9 --------- pyproject.toml | 15 ++++++++++++++- test/README.md | 6 ++---- test/requirements-testing.txt | 11 ----------- 8 files changed, 24 insertions(+), 36 deletions(-) delete mode 100644 doc/requirements.txt delete mode 100644 test/requirements-testing.txt diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 398ff8ae3..9b9cbed4d 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -20,12 +20,8 @@ jobs: - name: Setup python-pip run: python -m pip install --upgrade pip - - name: Install dependencies - run: | - pip install -r requirements.txt - - name: Install volatility3 run: pip install . - name: Run volatility3 - run: vol --help \ No newline at end of file + run: vol --help diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6358dd45d..930b68526 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,10 +16,8 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install Cmake - pip install build - pip install -r ./test/requirements-testing.txt + python -m pip install --upgrade pip Cmake build + pip install .[test] - name: Build PyPi packages run: | diff --git a/.readthedocs.yml b/.readthedocs.yml index e7c2b25d5..628e79ebf 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -20,4 +20,7 @@ build: # Optionally set the version of Python and requirements required to build your docs python: install: - - requirements: doc/requirements.txt + - method: pip + path: . + extra_requirements: + - docs diff --git a/MANIFEST.in b/MANIFEST.in index 1cec729f6..863621381 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ prune development include * .* -include doc/make.bat doc/Makefile doc/requirements.txt +include pyproject.toml doc/make.bat doc/Makefile recursive-include doc/source * recursive-include volatility3 *.json recursive-exclude doc/source volatility3.*.rst diff --git a/doc/requirements.txt b/doc/requirements.txt deleted file mode 100644 index d3ba51224..000000000 --- a/doc/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -# These packages are required for building the documentation. -sphinx>=4.0.0,<7 -sphinx_autodoc_typehints>=1.4.0 -sphinx-rtd-theme>=0.4.3 - -yara-python -yara-x -pycryptodome -pefile diff --git a/pyproject.toml b/pyproject.toml index 91848e1fd..a966e41b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = [ ] requires-python = ">=3.9.0" license = { text = "VSL" } -dynamic = ["dependencies", "optional-dependencies", "version"] +dynamic = ["version"] dependencies = [ "pefile>=2024.8.26", @@ -34,6 +34,19 @@ dev = [ "pyinstaller-hooks-contrib>=2024.9", ] +test = [ + "volatility3[dev]", + "pytest>=8.3.3,<9", + "yara-x>=0.10.0,<1", +] + +docs = [ + "volatility3[dev]", + "sphinx>=8.0.0,<7", + "sphinx-autodoc-typehints>=2.5.0,<3", + "sphinx-rtd-theme>=3.0.1,<4", +] + [project.urls] homepage = "https://github.com/volatilityfoundation/volatility3/" documentation = "https://volatility3.readthedocs.io/" diff --git a/test/README.md b/test/README.md index dcbe289b0..5891d9508 100644 --- a/test/README.md +++ b/test/README.md @@ -2,14 +2,12 @@ ## Requirements -The Volatility 3 Testing Framework requires the same version of Python as Volatility3 itself. To install the current set of dependencies that the framework requires, use a command like this: +The Volatility 3 Testing Framework requires the same version of Python as Volatility 3 itself. To install the current set of dependencies that the framework requires, use a command like this: ```shell -pip3 install -r requirements-testing.txt +pip3 install -e .[test] ``` -NOTE: `requirements-testing.txt` can be found in this current `test/` directory. - ## Quick Start: Manual Testing 1. To test Volatility 3 on an image, first download one with a command such as: diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt deleted file mode 100644 index 51c8f602c..000000000 --- a/test/requirements-testing.txt +++ /dev/null @@ -1,11 +0,0 @@ -# These packages are required for core functionality. -pefile>=2017.8.1 #foo - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 -yara-x>=0.5.0 - -pytest>=7.0.0 From b739f8d067c024f84087bfdc2443b174cf559462 Mon Sep 17 00:00:00 2001 From: lesander <4174509+lesander@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:05:54 +0100 Subject: [PATCH 046/348] revert 3.8 to 3.9 soft bump --- README.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 78eb7c491..cc33d3cc4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Installing -Volatility 3 requires Python 3.9.0 or later and is published on the [PyPi registry](https://pypi.org/project/volatility3). +Volatility 3 requires Python 3.8.0 or later and is published on the [PyPi registry](https://pypi.org/project/volatility3). ```shell pip install volatility3 diff --git a/pyproject.toml b/pyproject.toml index a966e41b2..fc1ab96cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, ] -requires-python = ">=3.9.0" +requires-python = ">=3.8.0" license = { text = "VSL" } dynamic = ["version"] From 485ef894e113cf68eb1acaef3c679b675639281d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:49:08 +0100 Subject: [PATCH 047/348] remove taints_value overload attr --- .../framework/symbols/linux/extensions/__init__.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 42c1a470d..f9f72c161 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -288,7 +288,7 @@ class module(generic.GenericIntelProcess): """ taints_string = "" for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if taint_flag.module and self.taints_value & taint_flag.shift: + if taint_flag.module and self.taints & taint_flag.shift: taints_string += char return taints_string @@ -310,7 +310,7 @@ class module(generic.GenericIntelProcess): for i, taint_flag in enumerate(self.taint_flags_list): c_true = chr(taint_flag.c_true) c_false = chr(taint_flag.c_false) - if taint_flag.module and (self.taints_value & (1 << i)): + if taint_flag.module and (self.taints & (1 << i)): taints_string += c_true elif taint_flag.module and c_false != " ": taints_string += c_false @@ -378,10 +378,6 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("module -> strtab: Unable to get strtab") - @property - def taints_value(self) -> int: - return self.taints - @property def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) From dd3542b127b751ad083c91be4e8ffd373a1c74f7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:51:48 +0100 Subject: [PATCH 048/348] explicit loop iterator --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f9f72c161..402f8c9c6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -341,10 +341,10 @@ class module(generic.GenericIntelProcess): - module_flags_taint kernel function """ comprehensive_taints = [] - for c in self.get_taints_as_plain_string(): - taint_flag = linux_constants.TAINT_FLAGS.get(c) + for character in self.get_taints_as_plain_string(): + taint_flag = linux_constants.TAINT_FLAGS.get(character) if not taint_flag: - comprehensive_taints.append(f"") + comprehensive_taints.append(f"") elif taint_flag.when_present: comprehensive_taints.append(taint_flag.desc) From 106b3b3dd0849194c4b83b9e254365b0c8f6070c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 16:01:04 +1100 Subject: [PATCH 049/348] Linux: PageCache: Fix inode offset when logging an error --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 81e1f3601..005fc9acc 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -454,7 +454,7 @@ class InodePages(plugins.PluginInterface): if current_fp + len(page_bytes) > inode_size: vollog.error( "Page out of file bounds: inode 0x%x, inode size %d, page index %d", - inode.vol.object, + inode.vol.offset, inode_size, page_idx, ) From 3216700d8b6b0488f4a7431d6c1d940fc5299f24 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 16:05:59 +1100 Subject: [PATCH 050/348] Linux: module object extension: Fix recursion call issue --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d05c34304..bc8e25105 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -294,7 +294,7 @@ class module(generic.GenericIntelProcess): if self.has_member("kallsyms"): return int(self.kallsyms.num_symtab) elif self.has_member("num_symtab"): - return int(self.num_symtab) + return int(self.member("num_symtab")) raise AttributeError( "module -> num_symtab: Unable to determine number of symbols" ) From a2e99b2633a0099e19bbb5692d7919a9e5cd17cd Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 16:13:50 +1100 Subject: [PATCH 051/348] Linux: Object extensions: Fix type annotations for improved clarity and accuracy. --- .../framework/symbols/linux/extensions/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index bc8e25105..41d764f94 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -240,7 +240,7 @@ class module(generic.GenericIntelProcess): for sym in syms: yield sym - def get_symbols_names_and_addresses(self) -> Tuple[str, int]: + def get_symbols_names_and_addresses(self) -> Iterable[Tuple[str, int]]: """Get names and addresses for each symbol of the module Yields: @@ -1098,7 +1098,7 @@ class dentry(objects.StructType): current_dentry = current_dentry.d_parent return None - def get_subdirs(self) -> interfaces.objects.ObjectInterface: + def get_subdirs(self) -> Iterable[interfaces.objects.ObjectInterface]: """Walks dentry subdirs Yields: @@ -2435,7 +2435,7 @@ class inode(objects.StructType): """ return stat.filemode(self.i_mode) - def get_pages(self) -> interfaces.objects.ObjectInterface: + def get_pages(self) -> Iterable[interfaces.objects.ObjectInterface]: """Gets the inode's cached pages Yields: @@ -2643,7 +2643,7 @@ class IDR(objects.StructType): return idr_layer - def _old_kernel_get_entries(self) -> int: + def _old_kernel_get_entries(self) -> Iterable[int]: # Kernels < 4.11 cur = self.cur total = next_id = 0 @@ -2655,7 +2655,7 @@ class IDR(objects.StructType): next_id += 1 - def _new_kernel_get_entries(self) -> int: + def _new_kernel_get_entries(self) -> Iterable[int]: # Kernels >= 4.11 id_storage = linux.IDStorage.choose_id_storage( self._context, kernel_module_name="kernel" @@ -2663,7 +2663,7 @@ class IDR(objects.StructType): for page_addr in id_storage.get_entries(root=self.idr_rt): yield page_addr - def get_entries(self) -> int: + def get_entries(self) -> Iterable[int]: """Walks the IDR and yield a pointer associated with each element. Args: From 3a734fdeb159c8ff31b216b5ef1249d1dd4d3df7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 16:23:33 +1100 Subject: [PATCH 052/348] Linux: Object extensions: Remove redundant log message header; the logger handles this formatting. If needed, we can adjust the logger's format instead --- .../framework/symbols/linux/extensions/__init__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 41d764f94..927f767e2 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -287,7 +287,7 @@ class module(generic.GenericIntelProcess): return self.kallsyms.symtab elif self.has_member("symtab"): return self.symtab - raise AttributeError("module -> symtab: Unable to get symtab") + raise AttributeError("Unable to get symtab") @property def num_symtab(self): @@ -295,9 +295,7 @@ class module(generic.GenericIntelProcess): return int(self.kallsyms.num_symtab) elif self.has_member("num_symtab"): return int(self.member("num_symtab")) - raise AttributeError( - "module -> num_symtab: Unable to determine number of symbols" - ) + raise AttributeError("Unable to determine number of symbols") @property def section_strtab(self): @@ -307,7 +305,7 @@ class module(generic.GenericIntelProcess): # Older kernels elif self.has_member("strtab"): return self.strtab - raise AttributeError("module -> strtab: Unable to get strtab") + raise AttributeError("Unable to get strtab") class task_struct(generic.GenericIntelProcess): From d90019657c26b2f6f723a5225ad99ad6c3fd012b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 18:14:36 +1100 Subject: [PATCH 053/348] Linux: Add kthreads plugin to enumerate kernel thread functions --- .../framework/plugins/linux/kthreads.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 volatility3/framework/plugins/linux/kthreads.py diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py new file mode 100644 index 000000000..639759e49 --- /dev/null +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -0,0 +1,109 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import List + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import linux +from volatility3.framework.constants import architectures +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist, lsmod + +vollog = logging.getLogger(__name__) + + +class Kthreads(plugins.PluginInterface): + """Enumerates kthread functions""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) + ), + ] + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + + modules = lsmod.Lsmod.list_modules(self.context, vmlinux.name) + handlers = linux.LinuxUtilities.generate_kernel_handler_info( + self.context, vmlinux.name, modules + ) + + kthread_type = vmlinux.get_type( + vmlinux.symbol_table_name + constants.BANG + "kthread" + ) + + if not kthread_type.has_member("threadfn"): + raise exceptions.VolatilityException( + "Unsupported kthread implementation. This plugin only works with kernels >= 5.8" + ) + + for task in pslist.PsList.list_tasks( + self.context, vmlinux.name, include_threads=True + ): + if not task.is_kernel_thread: + continue + + if task.has_member("worker_private"): + # kernels >= 5.17 e32cf5dfbe227b355776948b2c9b5691b84d1cbd + ktread_base_pointer = task.worker_private + else: + # 5.8 <= kernels < 5.17 in 52782c92ac85c4e393eb4a903a62e6c24afa633f threadfn + # was added to struct kthread. task.set_child_tid is safe on those versions. + ktread_base_pointer = task.set_child_tid + + if not ktread_base_pointer.is_readable(): + continue + + kthread = ktread_base_pointer.dereference().cast("kthread") + threadfn = kthread.threadfn + if not (threadfn and threadfn.is_readable()): + continue + + task_name = utility.array_to_string(task.comm) + + # kernels >= 5.17 in d6986ce24fc00b0638bd29efe8fb7ba7619ed2aa full_name was added to kthread + thread_name = ( + utility.pointer_to_string(kthread.full_name, count=255) + if kthread.has_member("full_name") + else task_name + ) + module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( + vmlinux, handlers, threadfn + ) + + fields = [ + task.pid, + thread_name, + format_hints.Hex(threadfn), + module_name, + symbol_name, + ] + yield 0, fields + + def run(self): + return renderers.TreeGrid( + [ + ("TID", int), + ("Thread Name", str), + ("Handler Address", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) From 780f9ab6f569af2c4e0095cb529736c2d5ef992d Mon Sep 17 00:00:00 2001 From: eve Date: Tue, 12 Nov 2024 07:22:45 +0000 Subject: [PATCH 054/348] Volshell: add regex_scan --- volatility3/cli/volshell/generic.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index a0477656e..9936244ee 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -14,7 +14,7 @@ from urllib import parse, request from volatility3.cli import text_renderer, volshell from volatility3.framework import exceptions, interfaces, objects, plugins, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.layers import intel, physical, resources +from volatility3.framework.layers import intel, physical, resources, scanners try: import capstone @@ -149,6 +149,7 @@ class Volshell(interfaces.plugins.PluginInterface): (["cc", "create_configurable"], self.create_configurable), (["lf", "load_file"], self.load_file), (["rs", "run_script"], self.run_script), + (["re", "regex_scan"], self.regex_scan), ] def _construct_locals_dict(self) -> Dict[str, Any]: @@ -288,6 +289,21 @@ class Volshell(interfaces.plugins.PluginInterface): remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data, format_string="H") + def regex_scan(self, pattern, count=128, layer_name=None): + """Scans for regex pattern in layer using RegExScanner.""" + if not isinstance(pattern, bytes): + raise TypeError("pattern must be bytes, e.g. re(b'pattern')") + layer_name_to_scan = layer_name or self.current_layer + for offset in self.context.layers[layer_name_to_scan].scan( + scanner=scanners.RegExScanner(pattern), + context=self.context, + ): + remaining_data = self._read_data( + offset, count=count, layer_name=layer_name_to_scan + ) + self._display_data(offset, remaining_data) + print("") + def disassemble(self, offset, count=128, layer_name=None, architecture=None): """Disassembles a number of instructions from the code at offset""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) From 8efe429719f50603e35a170b3eb37e39fc43f5fc Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 18:32:26 +1100 Subject: [PATCH 055/348] Linux: kthreads plugin: Adjust required framework version --- volatility3/framework/plugins/linux/kthreads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 639759e49..263793495 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -19,7 +19,7 @@ vollog = logging.getLogger(__name__) class Kthreads(plugins.PluginInterface): """Enumerates kthread functions""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 11, 0) _version = (1, 0, 0) From 787e15ac2b51987db496f38b6d9fa17e4eb43fe9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 09:50:08 +1100 Subject: [PATCH 056/348] Linux: kthreads plugin: Add missing requirements --- volatility3/framework/plugins/linux/kthreads.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 263793495..2e51b4688 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -34,6 +34,12 @@ class Kthreads(plugins.PluginInterface): requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 3, 0) + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), ] def _generator(self): From 05e8f8ff1075adbb398076a7d91701ba256b8d8a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 12:20:03 +1100 Subject: [PATCH 057/348] intel layer: Move constants to the Intel class --- .../framework/constants/linux/__init__.py | 8 ------- volatility3/framework/layers/intel.py | 24 ++++++++++++------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 303c534ed..7c485d3c3 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -11,14 +11,6 @@ KERNEL_NAME = "__kernel__" """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" -# Translation Layer constants -PAGE_BIT_PRESENT = 0 -PAGE_BIT_PSE = 7 # Page Size Extension: 4 MB (or 2MB) page -PAGE_BIT_PROTNONE = 8 -PAGE_BIT_PAT_LARGE = 12 # 2MB or 1GB pages -PAGE_PRESENT = 1 << PAGE_BIT_PRESENT -PAGE_PROTNONE = 1 << PAGE_BIT_PROTNONE - # include/linux/sched.h PF_KTHREAD = 0x00200000 # I'm a kernel thread diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index e6d20d992..25a98fc21 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -13,7 +13,6 @@ from volatility3 import classproperty from volatility3.framework import exceptions, interfaces, constants from volatility3.framework.configuration import requirements from volatility3.framework.layers import linear -from volatility3.framework.constants import linux as linux_constants vollog = logging.getLogger(__name__) @@ -23,6 +22,16 @@ INTEL_TRANSLATION_DEBUGGING = False class Intel(linear.LinearlyMappedLayer): """Translation Layer for the Intel IA32 memory mapping.""" + _PAGE_BIT_PRESENT = 0 + _PAGE_BIT_PSE = 7 # Page Size Extension: 4 MB (or 2MB) page + _PAGE_BIT_PROTNONE = 8 + _PAGE_BIT_PAT_LARGE = 12 # 2MB or 1GB pages + + _PAGE_PRESENT = 1 << _PAGE_BIT_PRESENT + _PAGE_PSE = 1 << _PAGE_BIT_PSE + _PAGE_PROTNONE = 1 << _PAGE_BIT_PROTNONE + _PAGE_PAT_LARGE = 1 << _PAGE_BIT_PAT_LARGE + _entry_format = " bool: - return ( - self.pte_flags(entry) - & (linux_constants.PAGE_PRESENT | linux_constants.PAGE_PROTNONE) - ) != 0 + return (self.pte_flags(entry) & (self._PAGE_PRESENT | self._PAGE_PROTNONE)) != 0 def _page_is_valid(self, entry: int) -> bool: # Overrides the Intel static method with the Linux-specific implementation @@ -556,7 +562,7 @@ class LinuxMixin(Intel): def pte_needs_invert(self, entry) -> bool: # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted - return not (entry & linux_constants.PAGE_PRESENT) + return not (entry & self._PAGE_PRESENT) def protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" From 36c19aa6d91b561ab57db34eb6c6a2ba7bbe2316 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 12:21:03 +1100 Subject: [PATCH 058/348] core: Bump framework minor version --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index ce803a687..55ef19e4b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 11 # Number of changes that only add to the interface +VERSION_MINOR = 12 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 173e35105a6fc614e0ae329331e8e21e63f52f14 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 12:30:11 +1100 Subject: [PATCH 059/348] LinuxMixin: Make new methods internal --- volatility3/framework/layers/intel.py | 46 ++++++++++++++------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 25a98fc21..dc88c20bf 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -174,13 +174,13 @@ class Intel(linear.LinearlyMappedLayer): f"Page Fault at entry {hex(entry)} in page entry", ) - pfn = self.pte_pfn(entry) + pfn = self._pte_pfn(entry) page_offset = self._mask(offset, position, 0) page = pfn << self.page_shift | page_offset return page, 1 << (position + 1), self._base_layer - def pte_pfn(self, entry: int) -> int: + def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number (PFN) from the page table entry (PTE) entry""" return entry >> self.page_shift @@ -520,11 +520,11 @@ class WindowsIntel32e(WindowsMixin, Intel32e): class LinuxMixin(Intel): @functools.cached_property - def register_mask(self) -> int: + def _register_mask(self) -> int: return (1 << self._bits_per_register) - 1 @functools.cached_property - def physical_mask(self) -> int: + def _physical_mask(self) -> int: # From kernels 4.18 the physical mask is dynamic: See AMD SME, Intel Multi-Key Total # Memory Encryption and CONFIG_DYNAMIC_PHYSICAL_MASK: 94d49eb30e854c84d1319095b5dd0405a7da9362 physical_mask = (1 << self._maxphyaddr) - 1 @@ -536,42 +536,44 @@ class LinuxMixin(Intel): # Note that within the Intel class it's a class method. However, since it uses # complement operations and we are working in Python, it would be more careful to # limit it to the architecture's pointer size. - return ~(self.page_size - 1) & self.register_mask + return ~(self.page_size - 1) & self._register_mask @functools.cached_property - def physical_page_mask(self) -> int: - return self.page_mask & self.physical_mask + def _physical_page_mask(self) -> int: + return self.page_mask & self._physical_mask @functools.cached_property - def pte_pfn_mask(self) -> int: - return self.physical_page_mask + def _pte_pfn_mask(self) -> int: + return self._physical_page_mask @functools.cached_property - def pte_flags_mask(self) -> int: - return ~self.pte_pfn_mask & self.register_mask + def _pte_flags_mask(self) -> int: + return ~self._pte_pfn_mask & self._register_mask - def pte_flags(self, pte) -> int: - return pte & self.pte_flags_mask + def _pte_flags(self, pte) -> int: + return pte & self._pte_flags_mask - def is_pte_present(self, entry: int) -> bool: - return (self.pte_flags(entry) & (self._PAGE_PRESENT | self._PAGE_PROTNONE)) != 0 + def _is_pte_present(self, entry: int) -> bool: + return ( + self._pte_flags(entry) & (self._PAGE_PRESENT | self._PAGE_PROTNONE) + ) != 0 def _page_is_valid(self, entry: int) -> bool: # Overrides the Intel static method with the Linux-specific implementation - return self.is_pte_present(entry) + return self._is_pte_present(entry) - def pte_needs_invert(self, entry) -> bool: + def _pte_needs_invert(self, entry) -> bool: # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted return not (entry & self._PAGE_PRESENT) - def protnone_mask(self, entry: int) -> int: + def _protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" - return ~0 & self.register_mask if self.pte_needs_invert(entry) else 0 + return ~0 & self._register_mask if self._pte_needs_invert(entry) else 0 - def pte_pfn(self, entry: int) -> int: + def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number from the page table entry""" - pfn = entry ^ self.protnone_mask(entry) - return (pfn & self.pte_pfn_mask) >> self.page_shift + pfn = entry ^ self._protnone_mask(entry) + return (pfn & self._pte_pfn_mask) >> self.page_shift class LinuxIntel(LinuxMixin, Intel): From af7a420cdc14689ee6d2ff72d5ac8eefad9e833d Mon Sep 17 00:00:00 2001 From: eve Date: Wed, 13 Nov 2024 08:56:58 +0000 Subject: [PATCH 060/348] Volshell: rename regex_scan short hand to rx so that it does not clash with the built in python re module --- volatility3/cli/volshell/generic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 9936244ee..b196360d5 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -149,7 +149,7 @@ class Volshell(interfaces.plugins.PluginInterface): (["cc", "create_configurable"], self.create_configurable), (["lf", "load_file"], self.load_file), (["rs", "run_script"], self.run_script), - (["re", "regex_scan"], self.regex_scan), + (["rx", "regex_scan"], self.regex_scan), ] def _construct_locals_dict(self) -> Dict[str, Any]: @@ -292,7 +292,7 @@ class Volshell(interfaces.plugins.PluginInterface): def regex_scan(self, pattern, count=128, layer_name=None): """Scans for regex pattern in layer using RegExScanner.""" if not isinstance(pattern, bytes): - raise TypeError("pattern must be bytes, e.g. re(b'pattern')") + raise TypeError("pattern must be bytes, e.g. rx(b'pattern')") layer_name_to_scan = layer_name or self.current_layer for offset in self.context.layers[layer_name_to_scan].scan( scanner=scanners.RegExScanner(pattern), From 357d8a914b4a69d231b4ed9de3aecdcf7179be7f Mon Sep 17 00:00:00 2001 From: eve Date: Thu, 14 Nov 2024 10:55:41 +0000 Subject: [PATCH 061/348] Volshell: Make the default number of bytes returned by db, dw, dd, dq, rx, and dis use the constant DEFAULT_NUM_DISPLAY_BYTES --- volatility3/cli/volshell/generic.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index b196360d5..82c470e1a 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -29,6 +29,8 @@ class Volshell(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) + DEFAULT_NUM_DISPLAY_BYTES = 128 + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__current_layer: Optional[str] = None @@ -269,27 +271,31 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_kernel_name = kernel_name print(f"Current kernel : {self.current_kernel_name}") - def display_bytes(self, offset, count=128, layer_name=None): + def display_bytes(self, offset, count=DEFAULT_NUM_DISPLAY_BYTES, layer_name=None): """Displays byte values and ASCII characters""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data) - def display_quadwords(self, offset, count=128, layer_name=None): + def display_quadwords( + self, offset, count=DEFAULT_NUM_DISPLAY_BYTES, layer_name=None + ): """Displays quad-word values (8 bytes) and corresponding ASCII characters""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data, format_string="Q") - def display_doublewords(self, offset, count=128, layer_name=None): + def display_doublewords( + self, offset, count=DEFAULT_NUM_DISPLAY_BYTES, layer_name=None + ): """Displays double-word values (4 bytes) and corresponding ASCII characters""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data, format_string="I") - def display_words(self, offset, count=128, layer_name=None): + def display_words(self, offset, count=DEFAULT_NUM_DISPLAY_BYTES, layer_name=None): """Displays word values (2 bytes) and corresponding ASCII characters""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data, format_string="H") - def regex_scan(self, pattern, count=128, layer_name=None): + def regex_scan(self, pattern, count=DEFAULT_NUM_DISPLAY_BYTES, layer_name=None): """Scans for regex pattern in layer using RegExScanner.""" if not isinstance(pattern, bytes): raise TypeError("pattern must be bytes, e.g. rx(b'pattern')") @@ -304,7 +310,13 @@ class Volshell(interfaces.plugins.PluginInterface): self._display_data(offset, remaining_data) print("") - def disassemble(self, offset, count=128, layer_name=None, architecture=None): + def disassemble( + self, + offset, + count=DEFAULT_NUM_DISPLAY_BYTES, + layer_name=None, + architecture=None, + ): """Disassembles a number of instructions from the code at offset""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) if not has_capstone: From bee3f001b66fa6a9ba9b1e325cb2fa2e273f8ff2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 15 Nov 2024 11:31:30 +1100 Subject: [PATCH 062/348] linux: netfilter plugin: Enhance docstring to provide a clearer explanation of 'hooked' --- volatility3/framework/plugins/linux/netfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 66075f907..b371d5ad2 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -174,7 +174,7 @@ class AbstractNetfilter(ABC): priority [int]: Priority hook_ops_hook [int]: Hook address module_name [str]: Linux kernel module name - hooked [bool]: hooked? + hooked [bool]: "True" if the network stack has been hijacked """ for netns, net in self.get_net_namespaces(): for proto_idx, proto_name, hook_idx, hook_name in self._proto_hook_loop(): From c909fdd4e76af6f8743a5df9b2ab38c981e68003 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 15 Nov 2024 11:53:05 +1100 Subject: [PATCH 063/348] linux: netfilter plugin: Bump minor version --- volatility3/framework/plugins/linux/netfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index b371d5ad2..73496dfd9 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -675,7 +675,7 @@ class Netfilter(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) _required_linuxutils_version = (2, 1, 0) _required_lsmod_version = (2, 0, 0) From b802b16c62ff5a50ea2dbc659450dc6d1fd5e83e Mon Sep 17 00:00:00 2001 From: eve Date: Fri, 15 Nov 2024 13:03:36 +0000 Subject: [PATCH 064/348] Add first version of regex scanning plugins --- .../framework/plugins/linux/vmaregexscan.py | 127 +++++++++++++++++ volatility3/framework/plugins/regexscan.py | 77 +++++++++++ .../framework/plugins/windows/vadregexscan.py | 128 ++++++++++++++++++ 3 files changed, 332 insertions(+) create mode 100644 volatility3/framework/plugins/linux/vmaregexscan.py create mode 100644 volatility3/framework/plugins/regexscan.py create mode 100644 volatility3/framework/plugins/windows/vadregexscan.py diff --git a/volatility3/framework/plugins/linux/vmaregexscan.py b/volatility3/framework/plugins/linux/vmaregexscan.py new file mode 100644 index 000000000..83df5458f --- /dev/null +++ b/volatility3/framework/plugins/linux/vmaregexscan.py @@ -0,0 +1,127 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import re + +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.layers import scanners +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class VmaRegExScan(plugins.PluginInterface): + """Scans all virtual memory areas for tasks using RegEx.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + MAXSIZE_DEFAULT = 128 + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.StringRequirement( + name="pattern", description="RegEx pattern", optional=False + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size in bytes for displayed context", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), + ] + + def _generator(self, regex_pattern, tasks): + regex_pattern = bytes(regex_pattern, "UTF-8") + vollog.debug(f"RegEx Pattern: {regex_pattern}") + + for task in tasks: + + if not task.mm: + continue + name = utility.array_to_string(task.comm) + + # attempt to create a process layer for each task and skip those + # that cannot (e.g. kernel threads) + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + continue + + # get the proc_layer object from the context + proc_layer = self.context.layers[proc_layer_name] + + # get process sections for scanning + sections = [ + (start, size) for (start, size) in task.get_process_memory_sections() + ] + + for offset in proc_layer.scan( + context=self.context, + scanner=scanners.RegExScanner(regex_pattern), + sections=sections, + progress_callback=self._progress_callback, + ): + result_data = proc_layer.read(offset, self.MAXSIZE_DEFAULT, pad=True) + + # reapply the regex in order to extact just the match + regex_result = re.match(regex_pattern, result_data) + + if regex_result: + # the match is within the results_data (e.g. it fits within MAXSIZE_DEFAULT) + # extract just the match itself + regex_match = regex_result.group(0) + text_result = str(regex_match, encoding="UTF-8", errors="replace") + bytes_result = regex_match + else: + # the match is not with the results_data (e.g. it doesn't fit within MAXSIZE_DEFAULT) + text_result = str(result_data, encoding="UTF-8", errors="replace") + bytes_result = result_data + + user_pid = task.tgid + yield 0, ( + user_pid, + name, + format_hints.Hex(offset), + text_result, + bytes_result, + ) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Offset", format_hints.Hex), + ("Text", str), + ("Hex", bytes), + ], + self._generator( + self.config.get("pattern"), + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ), + ), + ) diff --git a/volatility3/framework/plugins/regexscan.py b/volatility3/framework/plugins/regexscan.py new file mode 100644 index 000000000..d47a16407 --- /dev/null +++ b/volatility3/framework/plugins/regexscan.py @@ -0,0 +1,77 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import re + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints + +vollog = logging.getLogger(__name__) + + +class RegExScan(plugins.PluginInterface): + """Scans kernel memory using RegEx patterns.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + MAXSIZE_DEFAULT = 128 + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.StringRequirement( + name="pattern", description="RegEx pattern", optional=False + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size in bytes for displayed context", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), + ] + + def _generator(self, regex_pattern): + regex_pattern = bytes(regex_pattern, "UTF-8") + vollog.debug(f"RegEx Pattern: {regex_pattern}") + + layer = self.context.layers[self.config["primary"]] + for offset in layer.scan( + context=self.context, scanner=scanners.RegExScanner(regex_pattern) + ): + result_data = layer.read(offset, self.MAXSIZE_DEFAULT, pad=True) + + # reapply the regex in order to extact just the match + regex_result = re.match(regex_pattern, result_data) + + if regex_result: + # the match is within the results_data (e.g. it fits within MAXSIZE_DEFAULT) + # extract just the match itself + regex_match = regex_result.group(0) + text_result = str(regex_match, encoding="UTF-8", errors="replace") + bytes_result = regex_match + else: + # the match is not with the results_data (e.g. it doesn't fit within MAXSIZE_DEFAULT) + text_result = str(result_data, encoding="UTF-8", errors="replace") + bytes_result = result_data + + yield 0, (format_hints.Hex(offset), text_result, bytes_result) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Text", str), + ("Hex", bytes), + ], + self._generator(self.config.get("pattern")), + ) diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py new file mode 100644 index 000000000..742b00ace --- /dev/null +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -0,0 +1,128 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import re + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist, vadyarascan + +vollog = logging.getLogger(__name__) + + +class VadRegExScan(plugins.PluginInterface): + """Scans all virtual memory areas for tasks using RegEx.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + MAXSIZE_DEFAULT = 128 + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.StringRequirement( + name="pattern", description="RegEx pattern", optional=False + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size in bytes for displayed context", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), + ] + + def _generator(self, regex_pattern, procs): + regex_pattern = bytes(regex_pattern, "UTF-8") + vollog.debug(f"RegEx Pattern: {regex_pattern}") + + for proc in procs: + + # attempt to create a process layer for each proc + proc_layer_name = proc.add_process_layer() + if not proc_layer_name: + continue + + # get the proc_layer object from the context + proc_layer = self.context.layers[proc_layer_name] + + # get process sections for scanning + sections = sections = vadyarascan.VadYaraScan.get_vad_maps(proc) + + for offset in proc_layer.scan( + context=self.context, + scanner=scanners.RegExScanner(regex_pattern), + sections=sections, + progress_callback=self._progress_callback, + ): + result_data = proc_layer.read(offset, self.MAXSIZE_DEFAULT, pad=True) + + # reapply the regex in order to extact just the match + regex_result = re.match(regex_pattern, result_data) + + if regex_result: + # the match is within the results_data (e.g. it fits within MAXSIZE_DEFAULT) + # extract just the match itself + regex_match = regex_result.group(0) + text_result = str(regex_match, encoding="UTF-8", errors="replace") + bytes_result = regex_match + else: + # the match is not with the results_data (e.g. it doesn't fit within MAXSIZE_DEFAULT) + text_result = str(result_data, encoding="UTF-8", errors="replace") + bytes_result = result_data + + proc_id = proc.UniqueProcessId + process_name = proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ) + yield 0, ( + proc_id, + process_name, + format_hints.Hex(offset), + text_result, + bytes_result, + ) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] + procs = pslist.PsList.list_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=filter_func, + ) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Offset", format_hints.Hex), + ("Text", str), + ("Hex", bytes), + ], + self._generator(self.config.get("pattern"), procs), + ) From c2058e744b9a0f1e213857f4712ba1d9fdf85964 Mon Sep 17 00:00:00 2001 From: eve Date: Fri, 15 Nov 2024 13:12:03 +0000 Subject: [PATCH 065/348] Fix missing type imports for regex plugins --- volatility3/framework/plugins/linux/vmaregexscan.py | 5 +++-- volatility3/framework/plugins/regexscan.py | 1 + volatility3/framework/plugins/windows/vadregexscan.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmaregexscan.py b/volatility3/framework/plugins/linux/vmaregexscan.py index 83df5458f..77ab29814 100644 --- a/volatility3/framework/plugins/linux/vmaregexscan.py +++ b/volatility3/framework/plugins/linux/vmaregexscan.py @@ -4,8 +4,9 @@ import logging import re +from typing import List -from volatility3.framework import renderers +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import scanners @@ -24,7 +25,7 @@ class VmaRegExScan(plugins.PluginInterface): MAXSIZE_DEFAULT = 128 @classmethod - def get_requirements(cls): + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement( diff --git a/volatility3/framework/plugins/regexscan.py b/volatility3/framework/plugins/regexscan.py index d47a16407..c526b1697 100644 --- a/volatility3/framework/plugins/regexscan.py +++ b/volatility3/framework/plugins/regexscan.py @@ -4,6 +4,7 @@ import logging import re +from typing import List from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py index 742b00ace..141508283 100644 --- a/volatility3/framework/plugins/windows/vadregexscan.py +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -4,6 +4,7 @@ import logging import re +from typing import List from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -23,7 +24,7 @@ class VadRegExScan(plugins.PluginInterface): MAXSIZE_DEFAULT = 128 @classmethod - def get_requirements(cls): + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement( From aa93b57f0dde01b97bc94025447b242afa4c36b2 Mon Sep 17 00:00:00 2001 From: eve Date: Fri, 15 Nov 2024 13:15:40 +0000 Subject: [PATCH 066/348] Fix windows.vadregexscan sections --- volatility3/framework/plugins/windows/vadregexscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py index 141508283..8d17c469c 100644 --- a/volatility3/framework/plugins/windows/vadregexscan.py +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -70,7 +70,7 @@ class VadRegExScan(plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] # get process sections for scanning - sections = sections = vadyarascan.VadYaraScan.get_vad_maps(proc) + sections = vadyarascan.VadYaraScan.get_vad_maps(proc) for offset in proc_layer.scan( context=self.context, From 9f145ddcf0ddb84cf7ae45585ce7e9da0d204a17 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:04:33 +0100 Subject: [PATCH 067/348] pillow dependency --- requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requirements.txt b/requirements.txt index e0d366391..21c8f9a76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,9 @@ leechcorepyc>=2.4.0; sys_platform != 'darwin' # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage gcsfs>=2023.1.0 s3fs>=2023.1.0 + +# This is required by plugins that manipulate pixels and images. +# https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst +# 10.0.0 dropped support for Python3.7 +# 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 +pillow>=10.0.0,<11.0.0 \ No newline at end of file From 36a18f405b8ba7605ca957ca47d540a5b7c520d7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:05:23 +0100 Subject: [PATCH 068/348] fourcc code converter helper --- volatility3/framework/symbols/linux/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..573bc66d8 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -483,6 +483,22 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return kernel + @classmethod + def convert_fourcc_code(cls, code: int) -> str: + """Convert a fourcc integer back to its fourcc string representation. + + Args: + code: the numerical representation of the fourcc + + Returns: + The fourcc code string. + """ + + code_bytes_length = (code.bit_length() + 7) // 8 + return "".join( + [chr((code >> (i * 8)) & 0xFF) for i in range(code_bytes_length)] + ) + class IDStorage(ABC): """Abstraction to support both XArray and RadixTree""" From a7620cd6f659dfa685a445536a240182225a778c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:09:21 +0100 Subject: [PATCH 069/348] linux fbdev subsystem api plugin --- .../framework/plugins/linux/graphics/fbdev.py | 314 ++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 volatility3/framework/plugins/linux/graphics/fbdev.py diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py new file mode 100644 index 000000000..4bde6f62f --- /dev/null +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -0,0 +1,314 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +import io + +# Image manipulation functions are kept in the plugin, +# to prevent a general exit on missing PIL (pillow) dependency. +from PIL import Image +from dataclasses import dataclass +from typing import Type, List, Dict, Tuple +from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.objects import utility +from volatility3.framework.constants import architectures +from volatility3.framework.symbols import linux + +vollog = logging.getLogger(__name__) + + +@dataclass +class Framebuffer: + """Framebuffer object internal representation. This is useful to unify an framebuffer with precalculated + properties and pass it through functions conveniently.""" + + id: str + xres_virtual: int + yres_virtual: int + line_length: int + bpp: int + """Bits Per Pixel""" + size: int + color_fields: Dict[str, Tuple[int, int, int]] + fb_info: interfaces.objects.ObjectInterface + + +class Fbdev(interfaces.plugins.PluginInterface): + """Extract framebuffers from the fbdev graphics subsystem""" + + _version = (1, 0, 0) + _required_framework_version = (2, 11, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.BooleanRequirement( + name="dump", + description="Dump framebuffers", + default=False, + optional=True, + ), + ] + + @classmethod + def parse_fb_pixel_bitfields( + cls, fb_var_screeninfo: interfaces.objects.ObjectInterface + ) -> Dict[str, Tuple[int, int, int]]: + """Organize a framebuffer pixel format into a dictionary. + This is needed to know the position and bitlength of a color inside + a pixel. + + Args: + fb_var_screeninfo: a fb_var_screeninfo kernel object instance + + Returns: + The color fields mappings + + Documentation: + include/uapi/linux/fb.h: + struct fb_bitfield { + __u32 offset; /* beginning of bitfield */ + __u32 length; /* length of bitfield */ + __u32 msb_right; /* != 0 : Most significant bit is right */ + }; + """ + # Naturally order by RGBA + color_mappings = [ + ("R", fb_var_screeninfo.red), + ("G", fb_var_screeninfo.green), + ("B", fb_var_screeninfo.blue), + ("A", fb_var_screeninfo.transp), + ] + color_fields = {} + for color_code, fb_bitfield in color_mappings: + color_fields[color_code] = ( + int(fb_bitfield.offset), + int(fb_bitfield.length), + int(fb_bitfield.msb_right), + ) + return color_fields + + @classmethod + def convert_fb_raw_buffer_to_image( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + fb: Framebuffer, + ) -> Image.Image: + """Convert raw framebuffer pixels to an image. + + Args: + fb: the relevant Framebuffer object + + Returns: + A PIL Image object + + Documentation: + include/uapi/linux/fb.h: + /* Interpretation of offset for color fields: All offsets are from the right, + * inside a "pixel" value, which is exactly 'bits_per_pixel' wide (means: you + * can use the offset as right argument to <<). A pixel afterwards is a bit + * stream and is written to video memory as that unmodified. + """ + kernel = context.modules[kernel_name] + kernel_layer = context.layers[kernel.layer_name] + + raw_pixels = io.BytesIO(kernel_layer.read(fb.fb_info.screen_base, fb.size)) + bytes_per_pixel = fb.bpp // 8 + image = Image.new("RGBA", (fb.xres_virtual, fb.yres_virtual)) + + # This is not designed to be extremely fast (numpy isn't available), + # but convenient and dynamic for any color field layout. + for y in range(fb.yres_virtual): + for x in range(fb.xres_virtual): + raw_pixel = int.from_bytes(raw_pixels.read(bytes_per_pixel), "little") + pixel = [0, 0, 0, 255] + # The framebuffer is expected to have been correctly constructed, + # especially by parse_fb_pixel_bitfields, to get the needed RGBA mappings. + for i, color_code in enumerate(["R", "G", "B", "A"]): + offset, length, msb_right = fb.color_fields[color_code] + if length == 0: + continue + color_value = (raw_pixel >> offset) & (2**length - 1) + if msb_right: + # Reverse bit order + color_value = int( + "{:0{length}b}".format(color_value, length=length)[::-1], 2 + ) + pixel[i] = color_value + image.putpixel((x, y), tuple(pixel)) + + return image + + @classmethod + def dump_fb( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + fb: Framebuffer, + convert_to_image: bool, + image_format: str = "PNG", + ) -> str: + """Dump a Framebuffer raw buffer to disk. + + Args: + fb: the relevant Framebuffer object + convert_to_image: a boolean specifying if the buffer should be converted to an image + image_format: the target PIL image format (defaults to PNG) + + Returns: + The filename of the dumped buffer. + """ + kernel = context.modules[kernel_name] + kernel_layer = context.layers[kernel.layer_name] + base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" + if convert_to_image: + image = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) + output = io.BytesIO() + image.save(output, image_format) + file_handle = open_method(f"{base_filename}.{image_format.lower()}") + file_handle.write(output.getvalue()) + else: + raw_pixels = kernel_layer.read(fb.fb_info.screen_base, fb.size) + file_handle = open_method(f"{base_filename}.raw") + file_handle.write(raw_pixels) + + file_handle.close() + return file_handle.preferred_filename + + @classmethod + def parse_fb_info( + cls, + fb_info: interfaces.objects.ObjectInterface, + ) -> Framebuffer: + """Parse an fb_info struct + Args: + fb_info: an fb_info kernel object live instance + + Returns: + A Framebuffer object + + Documentation: + https://docs.kernel.org/fb/api.html: + - struct fb_fix_screeninfo stores device independent unchangeable information about the frame buffer device and the current format. + Those information can't be directly modified by applications, but can be changed by the driver when an application modifies the format. + - struct fb_var_screeninfo stores device independent changeable information about a frame buffer device, its current format and video mode, + as well as other miscellaneous parameters. + """ + # NotAvailableValue() messes with the filename output on disk + id = utility.array_to_string(fb_info.fix.id) or "N-A" + color_fields = None + + # 0 = color, 1 = grayscale, >1 = FOURCC + if fb_info.var.grayscale in [0, 1]: + color_fields = cls.parse_fb_pixel_bitfields(fb_info.var) + + # There a lot of tricky pixel formats used by drivers and vendors in include/uapi/linux/videodev2.h. + # As Volatility3 is not a video format converter, it is best to play it safe and let the user parse + # the raw data manually (with ffmpeg for example). + elif fb_info.var.grayscale > 1: + fourcc = linux.LinuxUtilities.convert_fourcc_code(fb_info.var.grayscale) + warn_msg = f"""Framebuffer "{id}" uses a FOURCC pixel format "{fourcc}" that isn't natively supported. +You can try using ffmpeg to decode the raw buffer. Example usage: +"ffmpeg -pix_fmts" to list supported formats, then +"ffmpeg -f rawvideo -video_size {fb_info.var.xres_virtual}x{fb_info.var.yres_virtual} -i .raw -pix_fmt output.png".""" + vollog.warning(warn_msg) + + # Prefer using the virtual resolution, instead of the visible one. + # This prevents missing non-visible data stored in the framebuffer. + fb = Framebuffer( + id, + xres_virtual=fb_info.var.xres_virtual, + yres_virtual=fb_info.var.yres_virtual, + line_length=fb_info.fix.line_length, + bpp=fb_info.var.bits_per_pixel, + size=fb_info.var.yres_virtual * fb_info.fix.line_length, + color_fields=color_fields, + fb_info=fb_info, + ) + + return fb + + def _generator(self): + kernel_name = self.config["kernel"] + kernel = self.context.modules[kernel_name] + + if not kernel.has_symbol("num_registered_fb"): + raise exceptions.SymbolError( + "num_registered_fb", + kernel.symbol_table_name, + "The provided symbol does not exist in the symbol table. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.", + ) + + num_registered_fb = kernel.object_from_symbol("num_registered_fb") + if num_registered_fb < 1: + vollog.info("No registered framebuffer in the fbdev API.") + return None + + registered_fb = kernel.object_from_symbol("registered_fb") + fb_info_list = utility.array_of_pointers( + registered_fb, + num_registered_fb, + kernel.symbol_table_name + constants.BANG + "fb_info", + self.context, + ) + + for fb_info in fb_info_list: + fb = self.parse_fb_info(fb_info) + file_output = "Disabled" + if self.config["dump"]: + try: + file_output = self.dump_fb( + self.context, kernel_name, self.open, fb, bool(fb.color_fields) + ) + except exceptions.InvalidAddressException as excp: + vollog.error( + f'Layer {excp.layer_name} failed to read address {hex(excp.invalid_address)} when dumping framebuffer "{fb.id}".' + ) + file_output = "Error" + + try: + fb_device_name = utility.pointer_to_string( + fb.fb_info.dev.kobj.name, 256 + ) + except exceptions.InvalidAddressException: + fb_device_name = NotAvailableValue() + + yield ( + 0, + ( + format_hints.Hex(fb.fb_info.screen_base), + fb_device_name, + fb.id, + fb.size, + f"{fb.xres_virtual}x{fb.yres_virtual}", + fb.bpp, + "RUNNING" if fb.fb_info.state == 0 else "SUSPENDED", + str(file_output), + ), + ) + + def run(self): + columns = [ + ("Address", format_hints.Hex), + ("Device", str), + ("ID", str), + ("Size", int), + ("Virtual resolution", str), + ("BPP", int), + ("State", str), + ("Filename", str), + ] + + return TreeGrid( + columns, + self._generator(), + ) From f192437a944f56ec3b8eae186d61f3049e691e80 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:11:11 +0100 Subject: [PATCH 070/348] typo --- volatility3/framework/plugins/linux/graphics/fbdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 4bde6f62f..60e00d033 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -21,7 +21,7 @@ vollog = logging.getLogger(__name__) @dataclass class Framebuffer: - """Framebuffer object internal representation. This is useful to unify an framebuffer with precalculated + """Framebuffer object internal representation. This is useful to unify a framebuffer with precalculated properties and pass it through functions conveniently.""" id: str From 6d366f16ee84844bec5c49ee94a10655995ea3ed Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 18 Nov 2024 06:38:06 +0000 Subject: [PATCH 071/348] Windows.vadregexscan: update imports --- volatility3/framework/plugins/windows/vadyarascan.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index efcc70d07..fd76bbc48 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -5,7 +5,8 @@ import logging from typing import Iterable, List, Tuple -from volatility3.framework import interfaces, renderers +from volatility3.framework import renderers +from volatility3.framework.interfaces import plugins, configuration, objects from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan @@ -14,14 +15,14 @@ from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) -class VadYaraScan(interfaces.plugins.PluginInterface): +class VadYaraScan(plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) _version = (1, 1, 1) @classmethod - def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + def get_requirements(cls) -> List[configuration.RequirementInterface]: # create a list of requirements for vadyarascan vadyarascan_requirements = [ requirements.ModuleRequirement( @@ -112,7 +113,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): @staticmethod def get_vad_maps( - task: interfaces.objects.ObjectInterface, + task: objects.ObjectInterface, ) -> Iterable[Tuple[int, int]]: """Creates a map of start/end addresses within a virtual address descriptor tree. From 07701fc4cf9385422f9d06b89cf64640e6a76b0d Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 18 Nov 2024 18:32:51 +0000 Subject: [PATCH 072/348] windows.vadregexscan: remove dependency on vadyarascan --- volatility3/framework/plugins/windows/vadregexscan.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py index 8d17c469c..206c9faae 100644 --- a/volatility3/framework/plugins/windows/vadregexscan.py +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -11,7 +11,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist, vadyarascan +from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) @@ -35,9 +35,6 @@ class VadRegExScan(plugins.PluginInterface): requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), - requirements.PluginRequirement( - name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) - ), requirements.ListRequirement( name="pid", description="Filter on specific process IDs", @@ -70,7 +67,11 @@ class VadRegExScan(plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] # get process sections for scanning - sections = vadyarascan.VadYaraScan.get_vad_maps(proc) + sections = [] + for vad in proc.get_vad_root().traverse(): + base = vad.get_start() + if vad.get_size(): + sections.append((base, vad.get_size())) for offset in proc_layer.scan( context=self.context, From e374ca96472279944bd5c0922f64e87f0b30e7e3 Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 18 Nov 2024 21:15:06 +0000 Subject: [PATCH 073/348] Windows.vadregexscan: update imports, revert vadyarascan changes --- volatility3/framework/plugins/windows/vadregexscan.py | 6 +++--- volatility3/framework/plugins/windows/vadyarascan.py | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py index 206c9faae..0d35cd658 100644 --- a/volatility3/framework/plugins/windows/vadregexscan.py +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -6,9 +6,9 @@ import logging import re from typing import List -from volatility3.framework import interfaces, renderers +from volatility3.framework import renderers from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins +from volatility3.framework.interfaces import plugins, configuration from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist @@ -24,7 +24,7 @@ class VadRegExScan(plugins.PluginInterface): MAXSIZE_DEFAULT = 128 @classmethod - def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + def get_requirements(cls) -> List[configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement( diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index fd76bbc48..efcc70d07 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -5,8 +5,7 @@ import logging from typing import Iterable, List, Tuple -from volatility3.framework import renderers -from volatility3.framework.interfaces import plugins, configuration, objects +from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan @@ -15,14 +14,14 @@ from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) -class VadYaraScan(plugins.PluginInterface): +class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) _version = (1, 1, 1) @classmethod - def get_requirements(cls) -> List[configuration.RequirementInterface]: + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # create a list of requirements for vadyarascan vadyarascan_requirements = [ requirements.ModuleRequirement( @@ -113,7 +112,7 @@ class VadYaraScan(plugins.PluginInterface): @staticmethod def get_vad_maps( - task: objects.ObjectInterface, + task: interfaces.objects.ObjectInterface, ) -> Iterable[Tuple[int, int]]: """Creates a map of start/end addresses within a virtual address descriptor tree. From 1ba5b04b7d3ef1b9a8911eae973da10f4d33d7e5 Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 18 Nov 2024 21:20:04 +0000 Subject: [PATCH 074/348] Windows: Remove dep on vadyarascan from svcscan plugin --- volatility3/framework/plugins/windows/svcscan.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 52ed5e759..ca390561f 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -20,7 +20,7 @@ from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.framework.symbols.windows.extensions import services as services_types -from volatility3.plugins.windows import poolscanner, pslist, vadyarascan +from volatility3.plugins.windows import poolscanner, pslist from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) @@ -39,7 +39,7 @@ class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) + _version = (3, 0, 1) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -60,9 +60,6 @@ class SvcScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) ), - requirements.PluginRequirement( - name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) - ), requirements.PluginRequirement( name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) ), @@ -317,10 +314,17 @@ class SvcScan(interfaces.plugins.PluginInterface): layer = context.layers[proc_layer_name] + # get process sections for scanning + sections = [] + for vad in task.get_vad_root().traverse(): + base = vad.get_start() + if vad.get_size(): + sections.append((base, vad.get_size())) + for offset in layer.scan( context=context, scanner=scanners.BytesScanner(needle=service_tag), - sections=vadyarascan.VadYaraScan.get_vad_maps(task), + sections=sections, ): if not is_vista_or_later: service_record = context.object( From 1abb4cc881d052b4f76df1e52bc4d0d9521c8c0c Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 11:13:21 +0100 Subject: [PATCH 075/348] comply with xdg base directory spec by using XDG_CACHE_HOME if its set --- volatility3/framework/constants/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 27fae4ba1..84c9c22ce 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -6,6 +6,7 @@ Stores all the constant values that are generally fixed throughout volatility This includes default scanning block sizes, etc. """ + import enum import os.path import sys @@ -65,7 +66,10 @@ LOGLEVEL_VVV = 7 LOGLEVEL_VVVV = 6 """Logging level for four levels of detail: -vvvvvv""" -CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") + +CACHE_PATH = os.path.join( + os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache"), "volatility3" +) """Default path to store cached data""" SQLITE_CACHE_PERIOD = "-3 days" From b93d5b7c13298ed2c1a24a951569a8e93f444c26 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 11:23:53 +0100 Subject: [PATCH 076/348] update docs --- doc/source/symbol-tables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index b7c26e046..722f9e468 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -9,7 +9,7 @@ How Volatility finds symbol tables All files are stored as JSON data, they can be in pure JSON files as ``.json``, or compressed as ``.json.gz`` or ``.json.xz``. Volatility will automatically decompress them on use. It will also cache their contents (compressed) when used, located -under the user's home directory, in :file:`.cache/volatility3`, along with other useful data. The cache directory currently +under the user's home directory, in :file:`.cache/volatility3` or when `XDG_CACHE_HOME` is set in :file:`${XDG_CACHE_HOME}/volatility3`, along with other useful data. The cache directory currently cannot be altered. Symbol table JSON files live, by default, under the :file:`volatility3/symbols` directory. The symbols directory is From a2b96c0dc0b2989e87e4eb4a9c2a274ef4ed6e7f Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 11:46:59 +0100 Subject: [PATCH 077/348] fix: dont use `/` for compat --- volatility3/framework/constants/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 84c9c22ce..8bdf84730 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -68,7 +68,8 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join( - os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache"), "volatility3" + os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache"), + "volatility3", ) """Default path to store cached data""" From 76ecfc08830e4b7def60520ab423a8d439e8fdbe Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 22 Nov 2024 12:16:14 +0000 Subject: [PATCH 078/348] Core: Add in generic cache-manager chooser function --- volatility3/framework/automagic/linux.py | 17 ++--------------- volatility3/framework/automagic/mac.py | 17 ++--------------- volatility3/framework/automagic/symbol_cache.py | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 52a73f45a..6fe18a4a9 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -27,16 +27,6 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): progress_callback: constants.ProgressCallback = None, ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify linux within this layer.""" - # Version check the SQlite cache - required = (1, 0, 0) - if not requirements.VersionRequirement.matches_required( - required, symbol_cache.SqliteCache.version - ): - vollog.info( - f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}" - ) - return None - # Bail out by default unless we can stack properly layer = context.layers[layer_name] join = interfaces.configuration.path_join @@ -46,12 +36,9 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - identifiers_path = os.path.join( - constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + linux_banners = symbol_cache.load_cache_manager().get_identifier_dictionary( + operating_system="linux" ) - linux_banners = symbol_cache.SqliteCache( - identifiers_path - ).get_identifier_dictionary(operating_system="linux") # If we have no banners, don't bother scanning if not linux_banners: vollog.info( diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index e51753139..7c478b521 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -28,16 +28,6 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): progress_callback: constants.ProgressCallback = None, ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify mac within this layer.""" - # Version check the SQlite cache - required = (1, 0, 0) - if not requirements.VersionRequirement.matches_required( - required, symbol_cache.SqliteCache.version - ): - vollog.info( - f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}" - ) - return None - # Bail out by default unless we can stack properly layer = context.layers[layer_name] new_layer = None @@ -48,12 +38,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - identifiers_path = os.path.join( - constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + mac_banners = symbol_cache.load_cache_manager().get_identifier_dictionary( + operating_system="mac" ) - mac_banners = symbol_cache.SqliteCache( - identifiers_path - ).get_identifier_dictionary(operating_system="mac") # If we have no banners, don't bother scanning if not mac_banners: vollog.info( diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 22f1c94f3..e38771f79 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -492,6 +492,21 @@ class SqliteCache(CacheManagerInterface): return output +def load_cache_manager(cache_file: Optional[str] = None) -> CacheManagerInterface: + """Loads a cache manager based on a specific cache file""" + if cache_file is None: + cache_file = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + # Different implementations of cache + if not os.path.exists(cache_file): + raise ValueError("Non-existant cache file provided") + with open(cache_file, "rb") as fp: + header = fp.read(4) + if header not in [b"SQLi"]: + raise ValueError("Identifier file not in recognized format") + # Currently only one choice, so use that + return SqliteCache(cache_file) + + ### Automagic From 4bc6e1001df8b60586309e56abb4ed7892b98c5d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 22 Nov 2024 19:45:15 +0000 Subject: [PATCH 079/348] Windows: Improve logging of slowscan to show possible PDB entries --- volatility3/framework/automagic/pdbscan.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 7f38a23e1..1d5bf55ea 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -270,6 +270,10 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): progress_callback=progress_callback, ) for kernel in kernels: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1)} with MZ offset at {kernel.get('mz_offset', -1)}", + ) valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) if valid_kernel is not None: break From 1a0d1a238da02db679c0bc5dfa10191abe25b6b7 Mon Sep 17 00:00:00 2001 From: lesander <4174509+lesander@users.noreply.github.com> Date: Sun, 24 Nov 2024 09:28:41 +0000 Subject: [PATCH 080/348] CLI: Partial changes by @lesander contributed in #1343 --- .github/workflows/test.yaml | 4 ++-- volatility3/cli/__init__.py | 5 +++-- volatility3/cli/volshell/__init__.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6358dd45d..73bf342b6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -41,8 +41,8 @@ jobs: - name: Testing... run: | - py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v - py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + pytest ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v + pytest ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v - name: Clean up post-test run: | diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 75b62abf6..5b2aa2838 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -88,7 +88,7 @@ class MuteProgress(PrintedProgress): class CommandLine: """Constructs a command-line interface object for users to run plugins.""" - CLI_NAME = "volatility" + CLI_NAME = os.path.basename(sys.argv[0]) # vol or volatility def __init__(self): self.setup_logging() @@ -364,6 +364,7 @@ class CommandLine: self.CLI_NAME ), action=volargparse.HelpfulSubparserAction, + metavar="PLUGIN", ) for plugin in sorted(plugin_list): plugin_parser = subparser.add_parser( @@ -385,7 +386,7 @@ class CommandLine: argcomplete.autocomplete(parser) args = parser.parse_args() if args.plugin is None: - parser.error("Please select a plugin to run") + parser.error(f"Please select a plugin to run (see '{self.CLI_NAME} --help' for options") vollog.log( constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}" diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 5172c5363..559adc12b 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -49,7 +49,7 @@ class VolShell(cli.CommandLine): python terminal with all the volatility support calls available. """ - CLI_NAME = "volshell" + CLI_NAME = os.path.basename(sys.argv[0]) # volshell def __init__(self): super().__init__() From 523a6e92fc75a972fe5d9a73c6dc65d4322a7f46 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 24 Nov 2024 09:33:30 +0000 Subject: [PATCH 081/348] CLI: Apply black to recent changes --- volatility3/cli/__init__.py | 6 ++++-- volatility3/cli/volshell/__init__.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 5b2aa2838..901f299a8 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -88,7 +88,7 @@ class MuteProgress(PrintedProgress): class CommandLine: """Constructs a command-line interface object for users to run plugins.""" - CLI_NAME = os.path.basename(sys.argv[0]) # vol or volatility + CLI_NAME = os.path.basename(sys.argv[0]) # vol or volatility def __init__(self): self.setup_logging() @@ -386,7 +386,9 @@ class CommandLine: argcomplete.autocomplete(parser) args = parser.parse_args() if args.plugin is None: - parser.error(f"Please select a plugin to run (see '{self.CLI_NAME} --help' for options") + parser.error( + f"Please select a plugin to run (see '{self.CLI_NAME} --help' for options" + ) vollog.log( constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}" diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 559adc12b..e9d3fda08 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -49,7 +49,7 @@ class VolShell(cli.CommandLine): python terminal with all the volatility support calls available. """ - CLI_NAME = os.path.basename(sys.argv[0]) # volshell + CLI_NAME = os.path.basename(sys.argv[0]) # volshell def __init__(self): super().__init__() From 641e008caf34b93b3c12360115066e1e291500ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 27 Nov 2024 15:06:33 +1100 Subject: [PATCH 082/348] Linux: intel: The non-present mapping shouldn't be inverted --- volatility3/framework/layers/intel.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index dc88c20bf..6d0aa6746 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -563,8 +563,9 @@ class LinuxMixin(Intel): return self._is_pte_present(entry) def _pte_needs_invert(self, entry) -> bool: - # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted - return not (entry & self._PAGE_PRESENT) + # Entries that were set to PROT_NONE (PAGE_PRESENT) are inverted + # A clear PTE shouldn't be inverted. See f19f5c4 + return entry and not (entry & self._PAGE_PRESENT) def _protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" From eff2a529c8e2e264553aca0550a5cd9ce3c9c298 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 27 Nov 2024 18:10:14 +1100 Subject: [PATCH 083/348] Linux: intel: Remove the bitwise zero complement to simplify the protnone mask calculation --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 6d0aa6746..d762b41a8 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -569,7 +569,7 @@ class LinuxMixin(Intel): def _protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" - return ~0 & self._register_mask if self._pte_needs_invert(entry) else 0 + return self._register_mask if self._pte_needs_invert(entry) else 0 def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number from the page table entry""" From 7c559c53f4bad23557056fec14a633628e1d5fe2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 12:11:14 +1100 Subject: [PATCH 084/348] test_cases: underscore unused variables --- test/test_volatility.py | 91 ++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 847be88d9..2f3ba3c63 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -61,7 +61,7 @@ def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]) def test_windows_pslist(image, volatility, python): - rc, out, err = runvol_plugin("windows.pslist.PsList", image, volatility, python) + rc, out, _err = runvol_plugin("windows.pslist.PsList", image, volatility, python) out = out.lower() assert out.find(b"system") != -1 assert out.find(b"csrss.exe") != -1 @@ -69,7 +69,7 @@ def test_windows_pslist(image, volatility, python): assert out.count(b"\n") > 10 assert rc == 0 - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"] ) out = out.lower() @@ -79,7 +79,7 @@ def test_windows_pslist(image, volatility, python): def test_windows_psscan(image, volatility, python): - rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) + rc, out, _err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) out = out.lower() assert out.find(b"system") != -1 assert out.find(b"csrss.exe") != -1 @@ -89,21 +89,21 @@ def test_windows_psscan(image, volatility, python): def test_windows_dlllist(image, volatility, python): - rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) + rc, out, _err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 def test_windows_modules(image, volatility, python): - rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) + rc, out, _err = runvol_plugin("windows.modules.Modules", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 def test_windows_hivelist(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.registry.hivelist.HiveList", image, volatility, python ) out = out.lower() @@ -136,7 +136,7 @@ def test_windows_dumpfiles(image, volatility, python): path = tempfile.mkdtemp() - rc, out, err = runvol_plugin( + rc, _out, _err = runvol_plugin( "windows.dumpfiles.DumpFiles", image, volatility, @@ -166,7 +166,7 @@ def test_windows_dumpfiles(image, volatility, python): def test_windows_handles(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"] ) @@ -183,7 +183,7 @@ def test_windows_handles(image, volatility, python): def test_windows_svcscan(image, volatility, python): - rc, out, err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) + rc, out, _err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) assert out.find(b"Microsoft ACPI Driver") != -1 assert out.count(b"\n") > 250 @@ -191,17 +191,19 @@ def test_windows_svcscan(image, volatility, python): def test_windows_thrdscan(image, volatility, python): - rc, out, err = runvol_plugin("windows.thrdscan.ThrdScan", image, volatility, python) + rc, out, _err = runvol_plugin( + "windows.thrdscan.ThrdScan", image, volatility, python + ) # find pid 4 (of system process) which starts with lowest tids assert out.find(b"\t4\t8") != -1 assert out.find(b"\t4\t12") != -1 assert out.find(b"\t4\t16") != -1 - #assert out.find(b"this raieses AssertionError") != -1 + # assert out.find(b"this raieses AssertionError") != -1 assert rc == 0 def test_windows_privileges(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"] ) @@ -213,7 +215,7 @@ def test_windows_privileges(image, volatility, python): def test_windows_getsids(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"] ) @@ -225,7 +227,7 @@ def test_windows_getsids(image, volatility, python): def test_windows_envars(image, volatility, python): - rc, out, err = runvol_plugin("windows.envars.Envars", image, volatility, python) + rc, out, _err = runvol_plugin("windows.envars.Envars", image, volatility, python) assert out.find(b"PATH") != -1 assert out.find(b"PROCESSOR_ARCHITECTURE") != -1 @@ -237,7 +239,7 @@ def test_windows_envars(image, volatility, python): def test_windows_callbacks(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.callbacks.Callbacks", image, volatility, python ) @@ -249,7 +251,7 @@ def test_windows_callbacks(image, volatility, python): def test_windows_vadwalk(image, volatility, python): - rc, out, err = runvol_plugin("windows.vadwalk.VadWalk", image, volatility, python) + rc, out, _err = runvol_plugin("windows.vadwalk.VadWalk", image, volatility, python) assert out.find(b"Vad") != -1 assert out.find(b"VadS") != -1 @@ -260,7 +262,7 @@ def test_windows_vadwalk(image, volatility, python): def test_windows_devicetree(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.devicetree.DeviceTree", image, volatility, python ) @@ -277,7 +279,7 @@ def test_windows_devicetree(image, volatility, python): def test_linux_pslist(image, volatility, python): - rc, out, err = runvol_plugin("linux.pslist.PsList", image, volatility, python) + rc, out, _err = runvol_plugin("linux.pslist.PsList", image, volatility, python) out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) @@ -287,7 +289,9 @@ def test_linux_pslist(image, volatility, python): def test_linux_check_idt(image, volatility, python): - rc, out, err = runvol_plugin("linux.check_idt.Check_idt", image, volatility, python) + rc, out, _err = runvol_plugin( + "linux.check_idt.Check_idt", image, volatility, python + ) out = out.lower() assert out.count(b"__kernel__") >= 10 @@ -296,7 +300,7 @@ def test_linux_check_idt(image, volatility, python): def test_linux_check_syscall(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "linux.check_syscall.Check_syscall", image, volatility, python ) out = out.lower() @@ -308,7 +312,7 @@ def test_linux_check_syscall(image, volatility, python): def test_linux_lsmod(image, volatility, python): - rc, out, err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) + rc, out, _err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 @@ -316,7 +320,7 @@ def test_linux_lsmod(image, volatility, python): def test_linux_lsof(image, volatility, python): - rc, out, err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) + rc, out, _err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) out = out.lower() assert out.count(b"socket:") >= 10 @@ -325,7 +329,7 @@ def test_linux_lsof(image, volatility, python): def test_linux_proc_maps(image, volatility, python): - rc, out, err = runvol_plugin("linux.proc.Maps", image, volatility, python) + rc, out, _err = runvol_plugin("linux.proc.Maps", image, volatility, python) out = out.lower() assert out.count(b"anonymous mapping") >= 10 @@ -334,15 +338,18 @@ def test_linux_proc_maps(image, volatility, python): def test_linux_tty_check(image, volatility, python): - rc, out, err = runvol_plugin("linux.tty_check.tty_check", image, volatility, python) + rc, out, _err = runvol_plugin( + "linux.tty_check.tty_check", image, volatility, python + ) out = out.lower() assert out.find(b"__kernel__") != -1 assert out.count(b"\n") >= 5 assert rc == 0 + def test_linux_sockstat(image, volatility, python): - rc, out, err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) + rc, out, _err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) assert out.count(b"AF_UNIX") >= 354 assert out.count(b"AF_BLUETOOTH") >= 5 @@ -354,7 +361,7 @@ def test_linux_sockstat(image, volatility, python): def test_linux_library_list(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "linux.library_list.LibraryList", image, volatility, python ) @@ -383,7 +390,7 @@ def test_linux_library_list(image, volatility, python): def test_mac_pslist(image, volatility, python): - rc, out, err = runvol_plugin("mac.pslist.PsList", image, volatility, python) + rc, out, _err = runvol_plugin("mac.pslist.PsList", image, volatility, python) out = out.lower() assert (out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1) @@ -392,7 +399,7 @@ def test_mac_pslist(image, volatility, python): def test_mac_check_syscall(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.check_syscall.Check_syscall", image, volatility, python ) out = out.lower() @@ -405,7 +412,7 @@ def test_mac_check_syscall(image, volatility, python): def test_mac_check_sysctl(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.check_sysctl.Check_sysctl", image, volatility, python ) out = out.lower() @@ -416,7 +423,7 @@ def test_mac_check_sysctl(image, volatility, python): def test_mac_check_trap_table(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.check_trap_table.Check_trap_table", image, volatility, python ) out = out.lower() @@ -427,7 +434,7 @@ def test_mac_check_trap_table(image, volatility, python): def test_mac_ifconfig(image, volatility, python): - rc, out, err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) + rc, out, _err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) out = out.lower() assert out.find(b"127.0.0.1") != -1 @@ -437,7 +444,7 @@ def test_mac_ifconfig(image, volatility, python): def test_mac_lsmod(image, volatility, python): - rc, out, err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) + rc, out, _err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) out = out.lower() assert out.find(b"com.apple") != -1 @@ -446,7 +453,7 @@ def test_mac_lsmod(image, volatility, python): def test_mac_lsof(image, volatility, python): - rc, out, err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) + rc, out, _err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) out = out.lower() assert out.count(b"\n") > 50 @@ -454,7 +461,7 @@ def test_mac_lsof(image, volatility, python): def test_mac_malfind(image, volatility, python): - rc, out, err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) + rc, out, _err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) out = out.lower() assert out.count(b"\n") > 20 @@ -462,7 +469,7 @@ def test_mac_malfind(image, volatility, python): def test_mac_mount(image, volatility, python): - rc, out, err = runvol_plugin("mac.mount.Mount", image, volatility, python) + rc, out, _err = runvol_plugin("mac.mount.Mount", image, volatility, python) out = out.lower() assert out.find(b"/dev") != -1 @@ -471,7 +478,7 @@ def test_mac_mount(image, volatility, python): def test_mac_netstat(image, volatility, python): - rc, out, err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) + rc, out, _err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) assert out.find(b"TCP") != -1 assert out.find(b"UDP") != -1 @@ -481,7 +488,7 @@ def test_mac_netstat(image, volatility, python): def test_mac_proc_maps(image, volatility, python): - rc, out, err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) + rc, out, _err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) out = out.lower() assert out.find(b"[heap]") != -1 @@ -490,7 +497,7 @@ def test_mac_proc_maps(image, volatility, python): def test_mac_psaux(image, volatility, python): - rc, out, err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) + rc, out, _err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) out = out.lower() assert out.find(b"executable_path") != -1 @@ -499,7 +506,7 @@ def test_mac_psaux(image, volatility, python): def test_mac_socket_filters(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.socket_filters.Socket_filters", image, volatility, python ) out = out.lower() @@ -509,7 +516,7 @@ def test_mac_socket_filters(image, volatility, python): def test_mac_timers(image, volatility, python): - rc, out, err = runvol_plugin("mac.timers.Timers", image, volatility, python) + rc, out, _err = runvol_plugin("mac.timers.Timers", image, volatility, python) out = out.lower() assert out.count(b"\n") > 6 @@ -517,7 +524,9 @@ def test_mac_timers(image, volatility, python): def test_mac_trustedbsd(image, volatility, python): - rc, out, err = runvol_plugin("mac.trustedbsd.Trustedbsd", image, volatility, python) + rc, out, _err = runvol_plugin( + "mac.trustedbsd.Trustedbsd", image, volatility, python + ) out = out.lower() assert out.count(b"\n") > 10 From 97534f02de61202475c5c3aa9418488d886f744c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 12:19:56 +1100 Subject: [PATCH 085/348] linux: sockstat: It should import and verify the pslist version directly, instead of relying on lsof --- volatility3/framework/plugins/linux/sockstat.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 0ddd3e26d..e5cf48d16 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -12,6 +12,7 @@ from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.framework.symbols import linux from volatility3.plugins.linux import lsof +from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) @@ -21,7 +22,6 @@ class SockHandlers(interfaces.configuration.VersionableInterface): """Handles several socket families extracting the sockets information.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) def __init__(self, vmlinux, task, *args, **kwargs): @@ -438,8 +438,7 @@ class Sockstat(plugins.PluginInterface): """Lists all network connections for all processes.""" _required_framework_version = (2, 0, 0) - - _version = (3, 0, 0) + _version = (3, 0, 1) @classmethod def get_requirements(cls): @@ -455,6 +454,9 @@ class Sockstat(plugins.PluginInterface): requirements.PluginRequirement( name="lsof", plugin=lsof.Lsof, version=(2, 0, 0) ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) ), @@ -591,7 +593,7 @@ class Sockstat(plugins.PluginInterface): tasks: String with a list of tasks and FDs using a socket. It can also have extended information such as socket filters, bpf info, etc. """ - filter_func = lsof.pslist.PsList.create_pid_filter(pids) + filter_func = pslist.PsList.create_pid_filter(pids) socket_generator = self.list_sockets( self.context, symbol_table, filter_func=filter_func ) From fa060646352d41cf6d818e6cb8c40deaa91e7e7f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 13:16:33 +1100 Subject: [PATCH 086/348] linux plugins: Update pslist major version and its 18 dependent plugins --- volatility3/framework/plugins/linux/bash.py | 3 ++- volatility3/framework/plugins/linux/boottime.py | 4 ++-- .../framework/plugins/linux/capabilities.py | 7 +++---- .../framework/plugins/linux/check_creds.py | 5 ++--- volatility3/framework/plugins/linux/elfs.py | 4 ++-- volatility3/framework/plugins/linux/envars.py | 3 ++- volatility3/framework/plugins/linux/kthreads.py | 5 ++--- .../framework/plugins/linux/library_list.py | 5 ++--- volatility3/framework/plugins/linux/lsof.py | 4 ++-- volatility3/framework/plugins/linux/malfind.py | 5 +++-- volatility3/framework/plugins/linux/mountinfo.py | 5 ++--- .../framework/plugins/linux/pidhashtable.py | 9 ++++----- volatility3/framework/plugins/linux/proc.py | 5 +++-- volatility3/framework/plugins/linux/psaux.py | 3 ++- volatility3/framework/plugins/linux/pslist.py | 3 +-- volatility3/framework/plugins/linux/pstree.py | 15 +++++++-------- volatility3/framework/plugins/linux/ptrace.py | 4 ++-- .../framework/plugins/linux/vmaregexscan.py | 5 +++-- .../framework/plugins/linux/vmayarascan.py | 4 ++-- 19 files changed, 48 insertions(+), 50 deletions(-) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index ce4567ca6..77a433a3b 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -22,6 +22,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): """Recovers bash command history from memory.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -32,7 +33,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py index 8f63ee7f8..3df5cb3ac 100644 --- a/volatility3/framework/plugins/linux/boottime.py +++ b/volatility3/framework/plugins/linux/boottime.py @@ -16,7 +16,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) _required_framework_version = (2, 11, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -27,7 +27,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 3, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index bfdb69aba..a8a8fb1fa 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -50,8 +50,7 @@ class Capabilities(plugins.PluginInterface): """Lists process capabilities""" _required_framework_version = (2, 0, 0) - - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -62,7 +61,7 @@ class Capabilities(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pids", @@ -87,7 +86,7 @@ class Capabilities(plugins.PluginInterface): try: kernel_cap_last_cap = vmlinux.object_from_symbol(symbol_name="cap_last_cap") except exceptions.SymbolError: - # It should be a kernel < 3.2 + # It should be a kernel < 3.2 See 73efc0394e148d0e15583e13712637831f926720 return None vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index b7f73c3eb..0857576d5 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -12,8 +12,7 @@ class Check_creds(interfaces.plugins.PluginInterface): """Checks if any processes are sharing credential structures""" _required_framework_version = (2, 0, 0) - - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls): @@ -24,7 +23,7 @@ class Check_creds(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 22e39d127..9f3bd274b 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -25,7 +25,7 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -36,7 +36,7 @@ class Elfs(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 5cbf0f502..22aba6408 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -17,6 +17,7 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -28,7 +29,7 @@ class Envars(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 2e51b4688..b9ced73f3 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -20,8 +20,7 @@ class Kthreads(plugins.PluginInterface): """Enumerates kthread functions""" _required_framework_version = (2, 11, 0) - - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -35,7 +34,7 @@ class Kthreads(plugins.PluginInterface): name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 3, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py index 062ed078e..7ec1f7f7f 100644 --- a/volatility3/framework/plugins/linux/library_list.py +++ b/volatility3/framework/plugins/linux/library_list.py @@ -21,8 +21,7 @@ class LibraryList(interfaces.plugins.PluginInterface): """Enumerate libraries loaded into processes""" _required_framework_version = (2, 0, 0) - - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -33,7 +32,7 @@ class LibraryList(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pids", diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 42b447dfb..802954f43 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -110,7 +110,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -121,7 +121,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 18f3dcd56..0b10e60c6 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -4,7 +4,7 @@ from typing import List import logging -from volatility3.framework import constants, interfaces +from volatility3.framework import interfaces from volatility3.framework import renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -18,6 +18,7 @@ class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -28,7 +29,7 @@ class Malfind(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 2499f009e..65775c4aa 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -36,8 +36,7 @@ class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 2, 0) - - _version = (1, 2, 1) + _version = (1, 2, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -48,7 +47,7 @@ class MountInfo(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index edafe97e0..2d210c233 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -19,8 +19,7 @@ class PIDHashTable(plugins.PluginInterface): """Enumerates processes through the PID hash table""" _required_framework_version = (2, 0, 0) - - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -31,7 +30,7 @@ class PIDHashTable(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) @@ -236,8 +235,8 @@ class PIDHashTable(plugins.PluginInterface): self, decorate_comm: bool = False ) -> interfaces.objects.ObjectInterface: for task in self.get_tasks(): - offset, pid, tid, ppid, name = pslist.PsList.get_task_fields( - task, decorate_comm + offset, pid, tid, ppid, name, _creation_time = ( + pslist.PsList.get_task_fields(task, decorate_comm) ) fields = format_hints.Hex(offset), pid, tid, ppid, name yield 0, fields diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index e7d38b107..00832140a 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,7 +21,8 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) + MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod @@ -34,7 +35,7 @@ class Maps(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index a4a23498f..5467c3b4c 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -15,6 +15,7 @@ class PsAux(plugins.PluginInterface): """Lists processes with their command line arguments""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -26,7 +27,7 @@ class PsAux(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index b05d69c7a..6460462a7 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -18,8 +18,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 0, 0) - - _version = (2, 3, 0) + _version = (3, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index efe5223df..9dc5ea3cc 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -13,6 +13,7 @@ class PsTree(interfaces.plugins.PluginInterface): ID.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -24,7 +25,7 @@ class PsTree(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", @@ -100,13 +101,11 @@ class PsTree(interfaces.plugins.PluginInterface): def yield_processes(pid): task = self._tasks[pid] - row = pslist.PsList.get_task_fields(task, decorate_comm) - # update the first element, the offset, in the row tuple to use format_hints.Hex - # as a simple int is returned from get_task_fields. - row = (format_hints.Hex(row[0]),) + row[1:] - - tid = task.pid - yield (self._levels[tid] - 1, row) + offset, pid, tid, ppid, name, _creation_time = ( + pslist.PsList.get_task_fields(task, decorate_comm) + ) + fields = format_hints.Hex(offset), pid, tid, ppid, name + yield (self._levels[tid] - 1, fields) for child_pid in sorted(self._children.get(tid, [])): yield from yield_processes(child_pid) diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index e467ee644..271c0e75e 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -19,7 +19,7 @@ class Ptrace(plugins.PluginInterface): """Enumerates ptrace's tracer and tracee tasks""" _required_framework_version = (2, 10, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -30,7 +30,7 @@ class Ptrace(plugins.PluginInterface): architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/vmaregexscan.py b/volatility3/framework/plugins/linux/vmaregexscan.py index 77ab29814..4446fc550 100644 --- a/volatility3/framework/plugins/linux/vmaregexscan.py +++ b/volatility3/framework/plugins/linux/vmaregexscan.py @@ -21,7 +21,8 @@ class VmaRegExScan(plugins.PluginInterface): """Scans all virtual memory areas for tasks using RegEx.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) + MAXSIZE_DEFAULT = 128 @classmethod @@ -34,7 +35,7 @@ class VmaRegExScan(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 9fe06b0c8..38d6cbaac 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -15,7 +15,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -28,7 +28,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): optional=True, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) From 63ac0abe642b4ba7fe93edfd2be0d6dd32e0483d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 13:25:08 +1100 Subject: [PATCH 087/348] linux: Add 15 new test cases for plugins dependent on pslist --- test/test_volatility.py | 165 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index 2f3ba3c63..4e948a2bd 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -386,6 +386,171 @@ def test_linux_library_list(image, volatility, python): assert rc == 0 +def test_linux_pstree(image, volatility, python): + rc, out, _err = runvol_plugin("linux.pstree.PsTree", image, volatility, python) + out = out.lower() + + assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_pidhashtable(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.pidhashtable.PIDHashTable", image, volatility, python + ) + out = out.lower() + + assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_bash(image, volatility, python): + rc, out, _err = runvol_plugin("linux.bash.Bash", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_boottime(image, volatility, python): + rc, out, _err = runvol_plugin("linux.boottime.Boottime", image, volatility, python) + out = out.lower() + + assert out.count(b"utc") >= 1 + assert rc == 0 + + +def test_linux_capabilities(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.capabilities.Capabilities", + image, + volatility, + python, + globalargs=["-vvv"], + ) + if rc != 0 and err.count(b"Unsupported kernel capabilities implementation") > 0: + # The linux-sample-1.bin kernel implementation isn't supported. + # However, we can still check that the plugin requirements are met. + return None + + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_check_creds(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_creds.Check_creds", image, volatility, python + ) + out = out.lower() + + # linux-sample-1.bin has no processes sharing credentials. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + + +def test_linux_elfs(image, volatility, python): + rc, out, _err = runvol_plugin("linux.elfs.Elfs", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_envars(image, volatility, python): + rc, out, _err = runvol_plugin("linux.envars.Envars", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_kthreads(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.kthreads.Kthreads", + image, + volatility, + python, + globalargs=["-vvv"], + ) + out = out.lower() + + if rc != 0 and err.count(b"Unsupported kthread implementation") > 0: + # The linux-sample-1.bin kernel implementation isn't supported. + # However, we can still check that the plugin requirements are met. + return None + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_malfind(image, volatility, python): + rc, out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) + out = out.lower() + + # linux-sample-1.bin has no process memory ranges with potential injected code. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + + +def test_linux_mountinfo(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.mountinfo.MountInfo", image, volatility, python + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_psaux(image, volatility, python): + rc, out, _err = runvol_plugin("linux.psaux.PsAux", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 50 + assert rc == 0 + + +def test_linux_ptrace(image, volatility, python): + rc, out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) + out = out.lower() + + # linux-sample-1.bin has no processes being ptreaced. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + + +def test_linux_vmaregexscan(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.vmaregexscan.VmaRegExScan", + image, + volatility, + python, + pluginargs=["--pid", "1", "--pattern", "\\x7fELF"], + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_vmayarascan(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.vmayarascan.VmaYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "1", "--yara-string", "ELF"], + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + # MAC From fb4ab9d4778b61e3a7af7761acac29f3bfe17ffd Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 13:45:26 +1100 Subject: [PATCH 088/348] linux: test cases: remove unused variables --- test/test_volatility.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 4e948a2bd..c0cc11cd3 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -442,10 +442,9 @@ def test_linux_capabilities(image, volatility, python): def test_linux_check_creds(image, volatility, python): - rc, out, _err = runvol_plugin( + rc, _out, _err = runvol_plugin( "linux.check_creds.Check_creds", image, volatility, python ) - out = out.lower() # linux-sample-1.bin has no processes sharing credentials. # This validates that plugin requirements are met and exceptions are not raised. @@ -488,8 +487,7 @@ def test_linux_kthreads(image, volatility, python): def test_linux_malfind(image, volatility, python): - rc, out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) - out = out.lower() + rc, _out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) # linux-sample-1.bin has no process memory ranges with potential injected code. # This validates that plugin requirements are met and exceptions are not raised. @@ -515,8 +513,7 @@ def test_linux_psaux(image, volatility, python): def test_linux_ptrace(image, volatility, python): - rc, out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) - out = out.lower() + rc, _out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) # linux-sample-1.bin has no processes being ptreaced. # This validates that plugin requirements are met and exceptions are not raised. From 9048cd1ab8683edac7d2ccfeb7fdcbee13d447df Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 13:47:03 +1100 Subject: [PATCH 089/348] linux: boottime: Minor, removed unnecessary blank line --- volatility3/framework/plugins/linux/boottime.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py index 3df5cb3ac..56de52883 100644 --- a/volatility3/framework/plugins/linux/boottime.py +++ b/volatility3/framework/plugins/linux/boottime.py @@ -15,7 +15,6 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """Shows the time the system was started""" _required_framework_version = (2, 11, 0) - _version = (1, 0, 1) @classmethod From aaeec80fdf4650be6a9b1f6c25ef5ba4aea03e29 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 13:53:11 +1100 Subject: [PATCH 090/348] linux: library_list testcase: Optimize testing performance by limiting the number of processes to just one. This change will reduce execution time and speed up the test --- test/test_volatility.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index c0cc11cd3..eb365ccf3 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -362,27 +362,19 @@ def test_linux_sockstat(image, volatility, python): def test_linux_library_list(image, volatility, python): rc, out, _err = runvol_plugin( - "linux.library_list.LibraryList", image, volatility, python + "linux.library_list.LibraryList", + image, + volatility, + python, + pluginargs=["--pids", "2363"], ) assert re.search( rb"NetworkManager\s2363\s0x7f52cdda0000\s/lib/x86_64-linux-gnu/libnss_files.so.2", out, ) - assert re.search( - rb"gnome-settings-\s3807\s0x7f7e660b5000\s/lib/x86_64-linux-gnu/libbz2.so.1.0", - out, - ) - assert re.search( - rb"gdu-notificatio\s3878\s0x7f25ce33e000\s/usr/lib/x86_64-linux-gnu/libXau.so.6", - out, - ) - assert re.search( - rb"bash\s8600\s0x7fe78a85f000\s/lib/x86_64-linux-gnu/libnss_files.so.2", - out, - ) - assert out.count(b"\n") >= 2677 + assert out.count(b"\n") > 10 assert rc == 0 From 8ecd7e2ddddb018164d3ef734899b91a93899d09 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 15:17:22 +1100 Subject: [PATCH 091/348] Linux/Mac: Log producer information --- .../framework/automagic/symbol_finder.py | 31 ++++++++++++++++--- volatility3/framework/symbols/intermed.py | 15 ++++++--- volatility3/framework/symbols/metadata.py | 13 +++++++- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 21e594549..55e2ad6f5 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -142,11 +142,11 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): ) for _, banner in banner_list: - vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = self.banners.get(banner, None) - if symbol_files: - isf_path = symbol_files - vollog.debug(f"Using symbol library: {symbol_files}") + vollog.debug(f"Identified banner: {banner!r}") + symbols_file = self.banners.get(banner, None) + if symbols_file: + isf_path = symbols_file + vollog.debug(f"Using symbol library: {symbols_file}") clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join @@ -160,8 +160,29 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): path_join(config_path, requirement.name, "symbol_mask") ] = layer.address_mask + # Keep track of the existing table names so we know which ones were added + old_table_names = set(context.symbol_space._dict) + # Construct the appropriate symbol table requirement.construct(context, config_path) + + new_table_names = context.symbol_space._dict.keys() - old_table_names + # It should add only one symbol table. Ignore the next steps if it doesn't + if len(new_table_names) == 1: + new_table_name = new_table_names.pop() + symbol_table = context.symbol_space._dict[new_table_name] + producer = symbol_table.producer + vollog.debug( + f"producer_name: {producer.name}, producer_version: {producer.version_string}" + ) + for category in symbol_table.metadata._json_data: + vollog.debug(f"{category}:") + for subkey in symbol_table.metadata._json_data[category]: + subkey_item = ", ".join( + f"{key}: '{value}'" for key, value in subkey.items() + ) + vollog.debug(f"\t{subkey_item}") + break else: vollog.debug(f"Symbol library path not found for: {banner}") diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 751f88e39..5f558bf12 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -738,10 +738,17 @@ class Version6Format(Version5Format): @property def metadata(self) -> Optional[interfaces.symbols.MetadataInterface]: """Returns a MetadataInterface object.""" - if self._json_object.get("metadata", {}).get("windows"): - return metadata.WindowsMetadata(self._json_object["metadata"]["windows"]) - if self._json_object.get("metadata", {}).get("linux"): - return metadata.LinuxMetadata(self._json_object["metadata"]["linux"]) + if "metadata" not in self._json_object: + return None + + json_metadata = self._json_object["metadata"] + if "windows" in json_metadata: + return metadata.WindowsMetadata(json_metadata["windows"]) + if "linux" in json_metadata: + return metadata.LinuxMetadata(json_metadata["linux"]) + if "mac" in json_metadata: + return metadata.MacMetadata(json_metadata["mac"]) + return None diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 95f542f07..39ddd6544 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -18,10 +18,17 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): def name(self) -> Optional[str]: return self._json_data.get("name", None) + @property + def version_string(self) -> str: + """Returns the ISF file producer's version as a string. + If no version is present, an empty string is returned. + """ + return self._json_data.get("version", "") + @property def version(self) -> Optional[Tuple[int]]: """Returns the version of the ISF file producer""" - version = self._json_data.get("version", None) + version = self.version_string() if not version: return None if all(x in "0123456789." for x in version): @@ -81,3 +88,7 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): class LinuxMetadata(interfaces.symbols.MetadataInterface): """Class to handle the metadata from a Linux symbol table.""" + + +class MacMetadata(interfaces.symbols.MetadataInterface): + """Class to handle the metadata from a Mac symbol table.""" From 3748cb89b8e2493fd8792315a9b3419d9a26dfb3 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 28 Nov 2024 10:22:45 +0000 Subject: [PATCH 092/348] Windows: Improve debugging output for pdbscan --- volatility3/framework/automagic/pdbscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 1d5bf55ea..3751b383e 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -272,7 +272,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): for kernel in kernels: vollog.log( constants.LOGLEVEL_VVVV, - f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1)} with MZ offset at {kernel.get('mz_offset', -1)}", + f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1):x} with MZ offset at {kernel.get('mz_offset', -1):x}", ) valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) if valid_kernel is not None: From bec6bc659475b7fc016f1b05093e7783ca70d904 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 28 Nov 2024 16:59:52 +0000 Subject: [PATCH 093/348] Fix up vmayarascan and vadyarascan to use yarascan properly --- .../framework/plugins/linux/vmayarascan.py | 58 +++++++++------ .../framework/plugins/windows/vadyarascan.py | 71 ++++++++----------- 2 files changed, 64 insertions(+), 65 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 9fe06b0c8..3d8eb603b 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import logging from typing import Iterable, List, Tuple from volatility3.framework import interfaces, renderers @@ -10,6 +11,8 @@ from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) + class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" @@ -50,6 +53,8 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): # use yarascan to parse the yara options provided and create the rules rules = yarascan.YaraScan.process_yara_options(dict(self.config)) + sanity_check = 1024 * 1024 * 1024 # 1 GB + # filter based on the pid option if provided filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for task in pslist.PsList.list_tasks( @@ -66,29 +71,36 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): # get the proc_layer object from the context proc_layer = self.context.layers[proc_layer_name] - for start, end in self.get_vma_maps(task): - for match in rules.match( - data=proc_layer.read(start, end - start, True) - ): - if yarascan.YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: - yield 0, ( - format_hints.Hex(instance.offset + start), - task.UniqueProcessId, - match.rule, - match_string.identifier, - instance.matched_data, - ) - else: - for offset, name, value in match.strings: - yield 0, ( - format_hints.Hex(offset + start), - task.tgid, - match.rule, - name, - value, - ) + vma_maps = list(self.get_vma_maps(task)) + insane_vma_maps = [ + start for (start, size) in vma_maps if size > sanity_check + ] + for start in insane_vma_maps: + vollog.debug(f"VMA at 0x{start:x} over sanity-check size, not scanning") + + if not vma_maps: + vollog.warning(f"No VMAs were found for task {task.pid}, aborting") + continue + + max_vma_size: int = max( + [size for (start, size) in vma_maps if size <= sanity_check] + ) + scanner = yarascan.YaraScanner(rules=rules) + scanner.chunk_size = max_vma_size + + # scan the process layer with the yarascanner + for offset, rule_name, name, value in proc_layer.scan( + context=self.context, + scanner=scanner, + sections=vma_maps, + ): + yield 0, ( + format_hints.Hex(offset), + task.tgid, + rule_name, + name, + value, + ) @staticmethod def get_vma_maps( diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index efcc70d07..65006bdc2 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -32,6 +32,9 @@ class VadYaraScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), @@ -66,49 +69,33 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] - for start, size in self.get_vad_maps(task): - if size > sanity_check: - vollog.debug( - f"VAD at 0x{start:x} over sanity-check size, not scanning" - ) - continue - data = layer.read(start, size, True) - if not yarascan.YaraScan._yara_x: - for match in rules.match(data=data): - if yarascan.YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: - yield 0, ( - format_hints.Hex(instance.offset + start), - task.UniqueProcessId, - match.rule, - match_string.identifier, - instance.matched_data, - ) - else: - for offset, name, value in match.strings: - yield 0, ( - format_hints.Hex(offset + start), - task.UniqueProcessId, - match.rule, - name, - value, - ) - else: - for match in rules.scan(data).matching_rules: - for match_string in match.patterns: - for instance in match_string.matches: - yield 0, ( - format_hints.Hex(instance.offset + start), - task.UniqueProcessId, - f"{match.namespace}.{match.identifier}", - match_string.identifier, - data[ - instance.offset : instance.offset - + instance.length - ], - ) + vad_maps = list(self.get_vad_maps(task)) + insane_vad_maps = [ + start for (start, size) in vad_maps if size > sanity_check + ] + for start in insane_vad_maps: + vollog.debug(f"VAD at 0x{start:x} over sanity-check size, not scanning") + + max_vad_size: int = max( + [size for (start, size) in vad_maps if size <= sanity_check] + ) + scanner = yarascan.YaraScanner(rules=rules) + scanner.chunk_size = max_vad_size + + # scan the process layer with the yarascanner + for offset, rule_name, name, value in layer.scan( + context=self.context, + scanner=scanner, + sections=vad_maps, + ): + yield 0, ( + format_hints.Hex(offset), + task.UniqueProcessId, + rule_name, + name, + value, + ) @staticmethod def get_vad_maps( From b8023f0c97ae97253ab9b8eae99e1dc4cba1eb79 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 29 Nov 2024 19:05:27 +1100 Subject: [PATCH 094/348] Linux/Mac: Address code review suggestions - Add getters for Linux/Mac ISF sources - Avoid using internal attributes - Use the dict repr instead of walking the dict to simplify code --- .../framework/automagic/symbol_finder.py | 26 ++++++++++--------- volatility3/framework/symbols/metadata.py | 19 +++++++++++--- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 55e2ad6f5..6d689e194 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -161,27 +161,29 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): ] = layer.address_mask # Keep track of the existing table names so we know which ones were added - old_table_names = set(context.symbol_space._dict) + old_table_names = set(context.symbol_space) # Construct the appropriate symbol table requirement.construct(context, config_path) - new_table_names = context.symbol_space._dict.keys() - old_table_names + new_table_names = set(context.symbol_space) - old_table_names # It should add only one symbol table. Ignore the next steps if it doesn't if len(new_table_names) == 1: new_table_name = new_table_names.pop() - symbol_table = context.symbol_space._dict[new_table_name] - producer = symbol_table.producer + symbol_table = context.symbol_space[new_table_name] + producer_metadata = symbol_table.producer vollog.debug( - f"producer_name: {producer.name}, producer_version: {producer.version_string}" + f"producer_name: {producer_metadata.name}, producer_version: {producer_metadata.version_string}" ) - for category in symbol_table.metadata._json_data: - vollog.debug(f"{category}:") - for subkey in symbol_table.metadata._json_data[category]: - subkey_item = ", ".join( - f"{key}: '{value}'" for key, value in subkey.items() - ) - vollog.debug(f"\t{subkey_item}") + + symbol_metadata = symbol_table.metadata + vollog.debug("Types:") + for types_source_dict in symbol_metadata.get_types_sources(): + vollog.debug(f"\t{types_source_dict}") + + vollog.debug("Symbols:") + for symbol_source_dict in symbol_metadata.get_symbols_sources(): + vollog.debug(f"\t{symbol_source_dict}") break else: diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 39ddd6544..02e9cc489 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -4,8 +4,7 @@ import datetime import logging -from typing import Optional, Tuple, Union - +from typing import Optional, Tuple, Union, List, Dict from volatility3.framework import constants, interfaces vollog = logging.getLogger(__name__) @@ -86,9 +85,21 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): return self._json_data.get("pdb", {}).get("age", None) -class LinuxMetadata(interfaces.symbols.MetadataInterface): +class DwarfMetadata(interfaces.symbols.MetadataInterface): + """Base class to handle metadata of DWARF-based ISF sources""" + + def get_types_sources(self) -> List[Optional[Dict]]: + """Returns the types sources metadata""" + return self._json_data.get("types", []) + + def get_symbols_sources(self) -> List[Optional[Dict]]: + """Returns the symbols sources metadata""" + return self._json_data.get("symbols", []) + + +class LinuxMetadata(DwarfMetadata): """Class to handle the metadata from a Linux symbol table.""" -class MacMetadata(interfaces.symbols.MetadataInterface): +class MacMetadata(DwarfMetadata): """Class to handle the metadata from a Mac symbol table.""" From 77778ee6f6cfaa9a9af1d73a225c67a8836727c7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 29 Nov 2024 19:35:18 +1100 Subject: [PATCH 095/348] Linux/Mac: ISF metadata: Rename s/DWARF/POSIX/, as I'm not happy with the generic name. BTF source could potentially generate the same keys --- volatility3/framework/symbols/metadata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 02e9cc489..73ad2cf21 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -85,8 +85,8 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): return self._json_data.get("pdb", {}).get("age", None) -class DwarfMetadata(interfaces.symbols.MetadataInterface): - """Base class to handle metadata of DWARF-based ISF sources""" +class PosixMetadata(interfaces.symbols.MetadataInterface): + """Base class to handle metadata of Posix-based ISF sources""" def get_types_sources(self) -> List[Optional[Dict]]: """Returns the types sources metadata""" @@ -97,9 +97,9 @@ class DwarfMetadata(interfaces.symbols.MetadataInterface): return self._json_data.get("symbols", []) -class LinuxMetadata(DwarfMetadata): +class LinuxMetadata(PosixMetadata): """Class to handle the metadata from a Linux symbol table.""" -class MacMetadata(DwarfMetadata): +class MacMetadata(PosixMetadata): """Class to handle the metadata from a Mac symbol table.""" From 0030129ff8b440f8f3e43870f87d697b3847baa6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 29 Nov 2024 08:43:04 +0000 Subject: [PATCH 096/348] Make suggested fixes to reduce loops and ignore insane sections --- .../framework/plugins/linux/vmayarascan.py | 25 +++++++++-------- .../framework/plugins/windows/vadyarascan.py | 28 ++++++++++++------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 3d8eb603b..89210be69 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -71,20 +71,21 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): # get the proc_layer object from the context proc_layer = self.context.layers[proc_layer_name] - vma_maps = list(self.get_vma_maps(task)) - insane_vma_maps = [ - start for (start, size) in vma_maps if size > sanity_check - ] - for start in insane_vma_maps: - vollog.debug(f"VMA at 0x{start:x} over sanity-check size, not scanning") + max_vma_size = 0 + vma_maps_to_scan = [] + for start, size in self.get_vma_maps(task): + if size > sanity_check: + vollog.debug( + f"VMA at 0x{start:x} over sanity-check size, not scanning" + ) + continue + max_vma_size = max(max_vma_size, size) + vma_maps_to_scan.append((start, size)) - if not vma_maps: - vollog.warning(f"No VMAs were found for task {task.pid}, aborting") + if not vma_maps_to_scan: + vollog.warning(f"No VMAs were found for task {task.tgid}, not scanning") continue - max_vma_size: int = max( - [size for (start, size) in vma_maps if size <= sanity_check] - ) scanner = yarascan.YaraScanner(rules=rules) scanner.chunk_size = max_vma_size @@ -92,7 +93,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): for offset, rule_name, name, value in proc_layer.scan( context=self.context, scanner=scanner, - sections=vma_maps, + sections=vma_maps_to_scan, ): yield 0, ( format_hints.Hex(offset), diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 65006bdc2..a67b8dc0b 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -70,16 +70,24 @@ class VadYaraScan(interfaces.plugins.PluginInterface): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] - vad_maps = list(self.get_vad_maps(task)) - insane_vad_maps = [ - start for (start, size) in vad_maps if size > sanity_check - ] - for start in insane_vad_maps: - vollog.debug(f"VAD at 0x{start:x} over sanity-check size, not scanning") + max_vad_size = 0 + vad_maps_to_scan = [] + + for start, size in self.get_vad_maps(task): + if size > sanity_check: + vollog.debug( + f"VAD at 0x{start:x} over sanity-check size, not scanning" + ) + continue + max_vad_size = max(max_vad_size, size) + vad_maps_to_scan.append((start, size)) + + if not vad_maps_to_scan: + vollog.warning( + f"No VADs were found for task {task.UniqueProcessID}, not scanning" + ) + continue - max_vad_size: int = max( - [size for (start, size) in vad_maps if size <= sanity_check] - ) scanner = yarascan.YaraScanner(rules=rules) scanner.chunk_size = max_vad_size @@ -87,7 +95,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): for offset, rule_name, name, value in layer.scan( context=self.context, scanner=scanner, - sections=vad_maps, + sections=vad_maps_to_scan, ): yield 0, ( format_hints.Hex(offset), From 393db1050f7df14d39c6f71bc0782e5422ed3188 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 29 Nov 2024 08:51:06 +0000 Subject: [PATCH 097/348] Windows: protect again mz_offsets being None --- volatility3/framework/automagic/pdbscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 3751b383e..0b4f6c73a 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -272,7 +272,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): for kernel in kernels: vollog.log( constants.LOGLEVEL_VVVV, - f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1):x} with MZ offset at {kernel.get('mz_offset', -1):x}", + f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1):x} with MZ offset at {(kernel.get('mz_offset', -1) or -1):x}", ) valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) if valid_kernel is not None: From 3df385369cbd2489f8e057b616e98a83b79cf397 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 30 Nov 2024 11:28:09 +0000 Subject: [PATCH 098/348] Ensure the VAD/VMA gets scanned in a single block --- .../framework/plugins/linux/vmayarascan.py | 28 ++++++++++--------- .../framework/plugins/windows/vadyarascan.py | 25 ++++++++--------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 89210be69..8dd64404d 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -36,6 +36,9 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), requirements.ModuleRequirement( name="kernel", description="Linux kernel", @@ -89,19 +92,18 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): scanner = yarascan.YaraScanner(rules=rules) scanner.chunk_size = max_vma_size - # scan the process layer with the yarascanner - for offset, rule_name, name, value in proc_layer.scan( - context=self.context, - scanner=scanner, - sections=vma_maps_to_scan, - ): - yield 0, ( - format_hints.Hex(offset), - task.tgid, - rule_name, - name, - value, - ) + # scan the VMA data (in one contiguous block) with the yarascanner + for start, size in vma_maps_to_scan: + for offset, rule_name, name, value in scanner( + proc_layer.read(start, size, pad=True), start + ): + yield 0, ( + format_hints.Hex(offset), + task.tgid, + rule_name, + name, + value, + ) @staticmethod def get_vma_maps( diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index a67b8dc0b..2e9cc44ea 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -91,19 +91,18 @@ class VadYaraScan(interfaces.plugins.PluginInterface): scanner = yarascan.YaraScanner(rules=rules) scanner.chunk_size = max_vad_size - # scan the process layer with the yarascanner - for offset, rule_name, name, value in layer.scan( - context=self.context, - scanner=scanner, - sections=vad_maps_to_scan, - ): - yield 0, ( - format_hints.Hex(offset), - task.UniqueProcessId, - rule_name, - name, - value, - ) + # scan the VAD data (in one contiguous block) with the yarascanner + for start, size in vad_maps_to_scan: + for offset, rule_name, name, value in scanner( + layer.read(start, size, pad=True), start + ): + yield 0, ( + format_hints.Hex(offset), + task.UniqueProcessId, + rule_name, + name, + value, + ) @staticmethod def get_vad_maps( From 56f6ef0add6d73ecebeb41837a69627f960e8b0d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 30 Nov 2024 11:52:16 +0000 Subject: [PATCH 099/348] Include a test developed by @gcmoreira and @eve-mem --- test/test_volatility.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index 847be88d9..ea9ad8211 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -14,6 +14,7 @@ import tempfile import hashlib import ntpath import json +import contextlib # # HELPER FUNCTIONS @@ -378,6 +379,42 @@ def test_linux_library_list(image, volatility, python): assert out.count(b"\n") >= 2677 assert rc == 0 +def test_linux_vmayarascan_yara_rule(image, volatility, python): + yara_rule_01 = r""" + rule fullvmayarascan + { + strings: + $s1 = "_nss_files_parse_grent" + $s2 = "/lib64/ld-linux-x86-64.so.2" + $s3 = "(bufferend - (char *) 0) % sizeof (char *) == 0" + condition: + all of them + } + """ + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".yar") + try: + with os.fdopen(fd, "w") as f: + f.write(yara_rule_01) + + rc, out, _err = runvol_plugin( + "linux.vmayarascan.VmaYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "8600", "--yara-file", filename], + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + out = out.lower() + assert out.count(b"\n") > 4 + assert rc == 0 + + # MAC From 2dc168628936ffcc334c1de37c024dc3d0fd7ff6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 30 Nov 2024 13:36:14 +0000 Subject: [PATCH 100/348] Windows: Protect the SERICE_RECORD is_valid function a little more The request to .Order could fail depending on where the structure lies in memory. --- .../symbols/windows/extensions/services.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/services.py b/volatility3/framework/symbols/windows/extensions/services.py index e14de761d..0a2194e07 100644 --- a/volatility3/framework/symbols/windows/extensions/services.py +++ b/volatility3/framework/symbols/windows/extensions/services.py @@ -14,13 +14,16 @@ class SERVICE_RECORD(objects.StructType): def is_valid(self) -> bool: """Determine if the structure is valid.""" - if self.Order < 0 or self.Order > 0xFFFF: - return False - try: - _ = self.State.description - _ = self.Start.description - except ValueError: + if self.Order < 0 or self.Order > 0xFFFF: + return False + + try: + _ = self.State.description + _ = self.Start.description + except ValueError: + return False + except exceptions.InvalidAddressException: return False return True From d404747de5ce622ac1907199ed48b4f9905f2bbc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 1 Dec 2024 00:04:46 +0000 Subject: [PATCH 101/348] Tests: Add in vadyarascan tests --- test/test_volatility.py | 58 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index ea9ad8211..371dd281c 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -197,7 +197,7 @@ def test_windows_thrdscan(image, volatility, python): assert out.find(b"\t4\t8") != -1 assert out.find(b"\t4\t12") != -1 assert out.find(b"\t4\t16") != -1 - #assert out.find(b"this raieses AssertionError") != -1 + # assert out.find(b"this raieses AssertionError") != -1 assert rc == 0 @@ -274,6 +274,59 @@ def test_windows_devicetree(image, volatility, python): assert rc == 0 +def test_windows_vadyarascan_yara_rule(image, volatility, python): + yara_rule_01 = r""" + rule fullvadyarascan + { + strings: + $s1 = "!This program cannot be run in DOS mode." + $s2 = "Qw))Pw" + $s3 = "W_wD)Pw" + $s4 = "1Xw+2Xw" + $s5 = "xd`wh``w" + $s6 = "0g`w0g`w8g`w8g`w@g`w@g`wHg`wHg`wPg`wPg`wXg`wXg`w`g`w`g`whg`whg`wpg`wpg`wxg`wxg`w" + condition: + all of them + } + """ + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".yar") + try: + with os.fdopen(fd, "w") as f: + f.write(yara_rule_01) + + rc, out, _err = runvol_plugin( + "windows.vadyarascan.VadYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "4012", "--yara-file", filename], + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + out = out.lower() + assert out.count(b"\n") > 4 + assert rc == 0 + + +def test_windows_vadyarascan(image, volatility, python): + rc, out, _err = runvol_plugin( + "windows.vadyarascan.VadYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "4012", "--yara-string", "MZ"], + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + # LINUX @@ -342,6 +395,7 @@ def test_linux_tty_check(image, volatility, python): assert out.count(b"\n") >= 5 assert rc == 0 + def test_linux_sockstat(image, volatility, python): rc, out, err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) @@ -379,6 +433,7 @@ def test_linux_library_list(image, volatility, python): assert out.count(b"\n") >= 2677 assert rc == 0 + def test_linux_vmayarascan_yara_rule(image, volatility, python): yara_rule_01 = r""" rule fullvmayarascan @@ -415,7 +470,6 @@ def test_linux_vmayarascan_yara_rule(image, volatility, python): assert rc == 0 - # MAC From 3c20e46902a3cc11ed00fcdf122f375ed1ca4b6c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 1 Dec 2024 12:36:35 +1100 Subject: [PATCH 102/348] renderers: Fix HexBytes formatter to apply padding also at the end of the string, ensuring proper output alignment when the pretty renderer justifies each line to the right --- volatility3/cli/text_renderer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 96970d3ce..31307f67e 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -51,8 +51,10 @@ def hex_bytes_as_text(value: bytes, width: int = 16) -> str: # Handle leftovers when the lenght is not mutiple of width if printables: - output += " " * (width - len(printables)) + padding = width - len(printables) + output += " " * (padding) output += printables + output += " " * (padding) return output From c19a54cbd982fc29c7d937d90c830c1dda3bfd03 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 1 Dec 2024 14:08:53 +1100 Subject: [PATCH 103/348] testcases: Minor cleanup: Renaming and reordering functions to align with the Linux/Windows test cases --- test/test_volatility.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 97098d1ca..f7cb23e93 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -315,7 +315,7 @@ def test_windows_vadyarascan_yara_rule(image, volatility, python): assert rc == 0 -def test_windows_vadyarascan(image, volatility, python): +def test_windows_vadyarascan_yara_string(image, volatility, python): rc, out, _err = runvol_plugin( "windows.vadyarascan.VadYaraScan", image, @@ -476,6 +476,7 @@ def test_linux_capabilities(image, volatility, python): python, globalargs=["-vvv"], ) + if rc != 0 and err.count(b"Unsupported kernel capabilities implementation") > 0: # The linux-sample-1.bin kernel implementation isn't supported. # However, we can still check that the plugin requirements are met. @@ -521,13 +522,14 @@ def test_linux_kthreads(image, volatility, python): python, globalargs=["-vvv"], ) - out = out.lower() if rc != 0 and err.count(b"Unsupported kthread implementation") > 0: # The linux-sample-1.bin kernel implementation isn't supported. # However, we can still check that the plugin requirements are met. return None + out = out.lower() + assert out.count(b"\n") > 10 assert rc == 0 @@ -580,20 +582,6 @@ def test_linux_vmaregexscan(image, volatility, python): assert rc == 0 -def test_linux_vmayarascan(image, volatility, python): - rc, out, _err = runvol_plugin( - "linux.vmayarascan.VmaYaraScan", - image, - volatility, - python, - pluginargs=["--pid", "1", "--yara-string", "ELF"], - ) - out = out.lower() - - assert out.count(b"\n") > 10 - assert rc == 0 - - def test_linux_vmayarascan_yara_rule(image, volatility, python): yara_rule_01 = r""" rule fullvmayarascan @@ -630,6 +618,20 @@ def test_linux_vmayarascan_yara_rule(image, volatility, python): assert rc == 0 +def test_linux_vmayarascan_yara_string(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.vmayarascan.VmaYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "1", "--yara-string", "ELF"], + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + # MAC From 535ce3a22a575c14100ca30627f3d2889fd86c9c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 1 Dec 2024 19:34:35 +1100 Subject: [PATCH 104/348] testing: Enable automatic selection of the OS image based on the test and filename prefix, addressing an issue in the development environment. For example, when using VSCode with pytest, test autodiscovery triggers pytest_generate_tests(), adding all images to each test case. This causes issues, as Linux tests end up being executed with Windows and Mac images, and vice versa. --- .github/workflows/test.yaml | 10 ++++++---- test/conftest.py | 33 ++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 084cce295..dfc42499d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,10 +25,13 @@ jobs: - name: Download images run: | + mkdir test_images + cd test_images curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz" gunzip linux-sample-1.bin.gz curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" gunzip win-xp-laptop-2005-06-25.img.gz + cd - - name: Download and Extract symbols run: | @@ -39,13 +42,12 @@ jobs: - name: Testing... run: | - pytest ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v - pytest ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_windows -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_linux -v - name: Clean up post-test run: | - rm -rf *.bin - rm -rf *.img + rm -rf test_images cd volatility3/symbols rm -rf linux rm -rf linux.zip diff --git a/test/conftest.py b/test/conftest.py index 4ad63065b..0115fade9 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -35,16 +35,39 @@ def pytest_addoption(parser): def pytest_generate_tests(metafunc): """Parameterize tests based on image names""" - images = metafunc.config.getoption("image") + images = metafunc.config.getoption("image").copy() for image_dir in metafunc.config.getoption("image_dir"): - images = images + [ - os.path.join(image_dir, dir) for dir in os.listdir(image_dir) + images += [ + os.path.join(image_dir, dir_name) for dir_name in os.listdir(image_dir) ] - # tests with "image" parameter are run against images + # tests with "image" parameter are run against image if "image" in metafunc.fixturenames: + filtered_images = [] + ids = [] + for image in images: + image_base = os.path.basename(image) + test_name = metafunc.definition.originalname + if test_name.startswith("test_windows_") and not image_base.startswith( + "win-" + ): + continue + elif test_name.startswith("test_linux_") and not image_base.startswith( + "linux-" + ): + continue + elif test_name.startswith("test_mac_") and not image_base.startswith( + "mac-" + ): + continue + + filtered_images.append(image) + ids.append(image_base) + metafunc.parametrize( - "image", images, ids=[os.path.basename(image) for image in images] + "image", + filtered_images, + ids=ids, ) From 3ff304ceed2501df6d9291259f9bc6b3c587c303 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 3 Dec 2024 19:58:19 +0000 Subject: [PATCH 105/348] Layers: Fix intel bug introduced in commit 73d4f2f The patch failed to mask the incoming address to the maximum physical address. This allowed non-canonical addresses (potentially within the page table) to be looked up incorrectly. Fixes #1374. Thanks to @the-rectifier for quickly identifying the issue! --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index d762b41a8..c30ae48a8 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -182,7 +182,7 @@ class Intel(linear.LinearlyMappedLayer): def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number (PFN) from the page table entry (PTE) entry""" - return entry >> self.page_shift + return self._mask(entry, self._maxphyaddr - 1, 0) >> self.page_shift def _translate_entry(self, offset: int) -> Tuple[int, int]: """Translates a specific offset based on paging tables. From 5acf8858d95e413d32b7342fe5f388a68597201e Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 5 Dec 2024 18:31:04 +0000 Subject: [PATCH 106/348] =?UTF-8?q?PEP=20488=20=E2=80=93=20Elimination=20o?= =?UTF-8?q?f=20PYO=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python 3.5 implememted PEP 488, eliminating .pyo files. --- volatility3/framework/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 51310bfa2..4c09b4dae 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -164,7 +164,6 @@ def _filter_files(filename: str): return ( filename.endswith(".py") or filename.endswith(".pyc") - or filename.endswith(".pyo") ) and not filename.startswith("__") From f11ef06c27bbaaeb25cb83c3eadfba192531b52b Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 5 Dec 2024 21:30:20 +0000 Subject: [PATCH 107/348] =?UTF-8?q?PEP=20488=20=E2=80=93=20Elimination=20o?= =?UTF-8?q?f=20PYO=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python 3.5 implememted PEP 488, eliminating .pyo files. --- volatility3/framework/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 4c09b4dae..23ea745de 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -162,8 +162,7 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]: def _filter_files(filename: str): """Ensures that a filename traversed is an importable python file""" return ( - filename.endswith(".py") - or filename.endswith(".pyc") + filename.endswith(".py") or filename.endswith(".pyc") ) and not filename.startswith("__") From 846403115a8f14a5b6c47ee6fb53bb20f4d6dc78 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:00:03 +0000 Subject: [PATCH 108/348] Small documention changes --- doc/source/basics.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/basics.rst b/doc/source/basics.rst index 1b8e64780..91a45fbbf 100644 --- a/doc/source/basics.rst +++ b/doc/source/basics.rst @@ -14,7 +14,7 @@ Memory layers ------------- A memory layer is a body of data that can be accessed by requesting data at a specific address. At its lowest level -this data is stored on a phyiscal medium (RAM) and very early computers addresses locations in memory directly. However, +this data is stored on a phyiscal medium (RAM) and very early computers addressed locations in memory directly. However, as the size of memory increased and it became more difficult to manage memory most architectures moved to a "paged" model of memory, where the available memory is cut into specific fixed-sized pages. To help further, programs can ask for any address and the processor will look up their (virtual) address in a map, to find out where the (physical) address that it lives at is, @@ -25,8 +25,8 @@ address `9`). The automagic that runs at the start of every volatility session a kernel virtual layer, which allows for kernel addresses to be looked up and the correct data returned. There can, however, be several maps, and in general there is a different map for each process (although a portion of the operating system's memory is usually mapped to the same location across all processes). The maps may take the same address but point to a different part of -physical memory. It also means that two processes could theoretically share memory, but having an virtual address mapped to the -same physical address as another process. See the worked example below for more information. +physical memory. It also means that two processes could theoretically share memory, both having a virtual address mapped to the +same physical address. See the worked example below for more information. To translate an address on a layer, call :py:meth:`layer.mapping(offset, length, ignore_errors) ` and it will return a list of chunks without overlap, in order, for the requested range. If a portion cannot be mapped, an exception will be thrown unless `ignore_errors` is true. Each @@ -61,7 +61,7 @@ mean they each see something different: 4 -> 2 16 - Free In this example, part of the operating system is visible across all processes (although not all processes can write to the memory, there -is a permissions model for intel addressing which is not discussed further here).) +is a permissions model for Intel addressing which is not discussed further here). In Volatility 3 mappings are represented by a directed graph of layers, whose end nodes are :py:class:`DataLayers ` and whose internal nodes are :py:class:`TranslationLayers `. @@ -69,13 +69,13 @@ In this way, a raw memory image in the LiME file format and a page file can be c memory layer. When requesting addresses from the Intel layer, it will use the Intel memory mapping algorithm, along with the address of the directory table base or page table map, to translate that address into a physical address, which will then either be directed towards the swap layer or the LiME layer. Should it -be directed towards the LiME layer, the LiME file format algorithm will be translate the new address to determine where +be directed towards the LiME layer, the LiME file format algorithm will translate the new address to determine where within the file the data is stored. When the :py:meth:`layer.read() ` method is called, the translation is done automatically and the correct data gathered and combined. .. note:: Volatility 2 had a similar concept, called address spaces, but these could only stack linearly one on top of another. -The list of layers supported by volatility can be determined by running the `frameworkinfo` plugin. +The list of layers supported by Volatility can be determined by running the `frameworkinfo` plugin. Templates and Objects --------------------- From 93a47e811b94c4683db119f81ab4496c9e008736 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 6 Dec 2024 21:52:51 +0000 Subject: [PATCH 109/348] Small documentation changes --- doc/source/basics.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/basics.rst b/doc/source/basics.rst index 91a45fbbf..278ef4d73 100644 --- a/doc/source/basics.rst +++ b/doc/source/basics.rst @@ -167,8 +167,7 @@ There are certain setup tasks that establish the context in a way favorable to a several tasks that are repetitive and also easy to get wrong. These are called :py:class:`Automagic `, since they do things like magically taking a raw memory image and automatically providing the plugin with an appropriate Intel translation layer and an -accurate symbol table without either the plugin or the calling program having to specify all the necessary details. +accurate symbol table without either the plugin or the calling program having to specify all the necessary details. Automagics are a core component which consumers of the library can call or not at their discretion. .. note:: Volatility 2 used to do this as well, but it wasn't a particularly modular mechanism, and was used only for stacking address spaces (rather than identifying profiles), and it couldn't really be disabled/configured easily. - Automagics in Volatility 3 are a core component which consumers of the library can call or not at their discretion. From d29c23e922d22b5ccce0f4bc2ac439acd695a31c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 7 Dec 2024 15:51:35 +0000 Subject: [PATCH 110/348] Windows: Protect against missing _MM_SESSION_SPACE symbol --- volatility3/framework/symbols/windows/extensions/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ecfc2f163..793e506c3 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -825,6 +825,11 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.Session.SessionId at {self.vol.offset:#x}", ) + except exceptions.SymbolError: + vollog.log( + constants.LOGLEVEL_VVV, + "Could not lookup _MM_SESSION_SPACE in symbol table", + ) return renderers.UnreadableValue() From 9ebc0bd90d544faeec98ab4d4dd46723aa0c3ed2 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:20:45 +0000 Subject: [PATCH 111/348] Cosmetic changes to documentation --- doc/source/symbol-tables.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 722f9e468..7f0b4153d 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -25,9 +25,9 @@ as long as the symbol files stay in the same location. Windows symbol tables --------------------- -For Windows systems, Volatility accepts a string made up of the GUID and Age of the required PDB file. It then +For Windows systems, Volatility accepts a string made up of the GUID and age of the required PDB file. It then searches all files under the configured symbol directories under the windows subdirectory. Any that contain metadata -which matches the pdb name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then +which matches the PDB name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then the associated PDB file will be downloaded from Microsoft's Symbol Server and converted into the appropriate JSON format, and will be saved in the correct location. @@ -35,7 +35,7 @@ Windows symbol tables can be manually constructed from an appropriate PDB file. is built into Volatility 3, called :file:`pdbconv.py`. It can be run from the top-level Volatility path, using the following command: -:command:`PYTHONPATH="." python volatility3/framework/symbols/windows/pdbconv.py` +:command:`PYTHONPATH="."; python volatility3/framework/symbols/windows/pdbconv.py` The :envvar:`PYTHONPATH` environment variable is not required if the Volatility library is installed in the system's library path or a virtual environment. @@ -54,8 +54,8 @@ most Volatility plugins. Note that in most linux distributions, the standard ke and the kernel with debugging information is stored in a package that must be acquired separately. A generic table isn't guaranteed to produce accurate results, and would reduce the number of structures -that all plugins could rely on. As such, and because linux kernels with different configurations can produce different structures, -volatility 3 requires that the banners in the JSON file match the banners found in the image *exactly*, not just the version +that all plugins could rely on. As such, and because Linux kernels with different configurations can produce different structures, +Volatility 3 requires that the banners in the JSON file match the banners found in the image *exactly*, not just the version number. This can include elements such as the compilation time and even the version of gcc used for the compilation. The exact match is required to ensure that the results volatility returns are accurate, therefore there is no simple means provided to get the wrong JSON ISF file to easily match. @@ -63,8 +63,8 @@ provided to get the wrong JSON ISF file to easily match. To determine the string for a particular memory image, use the `banners` plugin. Once the specific banner is known, try to locate that exact kernel debugging package for the operating system. Unfortunately each distribution provides its debugging packages under different package names and there are so many that the distribution may not keep all old -versions of the debugging symbols, and therefore **it may not be possible to find the right symbols to analyze a linux -memory image with volatility**. With Macs there are far fewer kernels and only one distribution, making it easier to +versions of the debugging symbols, and therefore **it may not be possible to find the right symbols to analyze a Linux +memory image with Volatility**. With Macs there are far fewer kernels and only one distribution, making it easier to ensure that the right symbols can be found. Once a kernel with debugging symbols/appropriate DWARF file has been located, `dwarf2json `_ will convert it into an @@ -75,7 +75,7 @@ symbol offsets within the DWARF data, which dwarf2json can extract into the JSON The banners available for volatility to use can be found using the `isfinfo` plugin, but this will potentially take a long time to run depending on the number of JSON files available. This will list all the JSON (ISF) files that -volatility3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON +Volatility 3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON file, the banners must match exactly (down to the compilation date). .. note:: From 97698cc5edc4a4d699e452353307c624ccb58a88 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:29:58 +0000 Subject: [PATCH 112/348] Cosmetic changes to documentation --- doc/source/vol2to3.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/source/vol2to3.rst b/doc/source/vol2to3.rst index e768df0c2..2520562a6 100644 --- a/doc/source/vol2to3.rst +++ b/doc/source/vol2to3.rst @@ -27,7 +27,7 @@ The object model has changed as well, objects now inherit directly from their Py object is actually a Python integer (and has all the associated methods, and can be used wherever a normal int could). In Volatility 2, a complex proxy object was constructed which tried to emulate all the methods of the host object, but ultimately it was a different type and could not be used in the same places (critically, it could make the ordering of -operations important, since a + b might not work, but b + a might work fine). +operations important, since x + y might not work, but y + x might work fine). Volatility 3 has also had significant speed improvements, where Volatility 2 was designed to allow access to live memory images and situations in which the underlying data could change during the run of the plugin, in Volatility 3 the data @@ -56,15 +56,14 @@ Volatility 2 were strictly limited to a stack, one on top of one other. In Vola Automagic --------- -In Volatility 2, we often tried to make this simpler for both users and developers. This resulted in something was -referred to as automagic, in that it was magic that happened automatically. We've now codified that more, so that the +In Volatility 2, we often tried to make this simpler for both users and developers. This resulted in something referred to as automagic, in that it was magic that happened automatically. We've now codified that more, so that the automagic processes are clearly defined and can be enabled or disabled as necessary for any particular run. We also included a stacker automagic to emulate the most common feature of Volatility 2, automatically stacking address spaces (now translation layers) on top of each other. -By default the automagic chosen to be run are determined based on the plugin requested, so that linux plugins get linux -specific automagic and windows plugins get windows specific automagic. This should reduce unnecessarily searching for -linux kernels in a windows image, for example. At the moment this is not user configurableS. +By default the automagic chosen to be run are determined based on the plugin requested, so that Linux plugins get Linux +specific automagic and Windows plugins get Windows specific automagic. This should reduce unnecessarily searching for +Linux kernels in a Windows image, for example. At the moment this is not user configurable. Searching and Scanning ---------------------- From 04d25544428c4a890c7cc88ebe4f80908a101464 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:36:18 +0000 Subject: [PATCH 113/348] Cosmetic changes to documentation --- doc/source/volshell.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 5a4b21ade..3c4f4ce5d 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -144,12 +144,12 @@ We can provide arguments via the `dpo` method call: 356 4 smss.exe 0x8c0bccf8d040 3 - N/A False 2021-03-13 17:25:33.000000 N/A Disabled ... -Here's we've provided the kernel name that was requested by the volshell plugin itself (the generic volshell does not +Here we've provided the kernel name that was requested by the volshell plugin itself (the generic volshell does not load a kernel module, and instead only has a TranslationLayerRequirement). A different module could be created and provided instead. The context used by the `dpo` method is always `context`. -Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by +Instead of printing the results directly to screen, they can be gathered into a TreeGrid objects for direct access by using the `generate_treegrid` or `gt` command. :: From e165c78ba70de9961749f1d347fac2bcf6bca13d Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:37:39 +0000 Subject: [PATCH 114/348] Cosmetic changes to documentation --- doc/source/vol2to3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/vol2to3.rst b/doc/source/vol2to3.rst index 2520562a6..9b5a739f8 100644 --- a/doc/source/vol2to3.rst +++ b/doc/source/vol2to3.rst @@ -36,11 +36,11 @@ This was because live memory analysis was barely ever used, and this feature cou re-read many times over for no benefit (particularly since each re-read could result in many additional image reads from following page table translations). -Finally, in order to provide Volatility specific information without impact on the ability for structures to have members +Further, in order to provide Volatility specific information without impact on the ability for structures to have members with arbitrary names, all the metadata about the object (such as its layer or offset) have been moved to a read-only :py:meth:`~volatility3.framework.interfaces.objects.ObjectInterface.vol` dictionary. -Further the distinction between a :py:class:`~volatility3.framework.interfaces.objects.Template` (the thing that +Finally, the distinction between a :py:class:`~volatility3.framework.interfaces.objects.Template` (the thing that constructs an object) and the :py:class:`Object ` itself has been made more explicit. In Volatility 2, some information (such as size) could only be determined from a constructed object, leading to instantiating a template on an empty buffer, just to determine the size. In Volatility 3, templates contain From 45f9064623cedf0ac0ceb9b95a5f2729904ba12b Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 7 Dec 2024 17:11:49 +0000 Subject: [PATCH 115/348] Cosmetic changes to documentation --- doc/source/symbol-tables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 7f0b4153d..59c1febcc 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -35,7 +35,7 @@ Windows symbol tables can be manually constructed from an appropriate PDB file. is built into Volatility 3, called :file:`pdbconv.py`. It can be run from the top-level Volatility path, using the following command: -:command:`PYTHONPATH="."; python volatility3/framework/symbols/windows/pdbconv.py` +:command:`PYTHONPATH="." python volatility3/framework/symbols/windows/pdbconv.py` The :envvar:`PYTHONPATH` environment variable is not required if the Volatility library is installed in the system's library path or a virtual environment. From b86e8397188900740e3fce848b6ac3abddf1389c Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 9 Dec 2024 18:30:19 +0000 Subject: [PATCH 116/348] Interfaces: change allow list for filenames to ensure they work safely on windows. Fixes issue #1387 --- volatility3/framework/interfaces/plugins.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 697e4cdc3..74902636e 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -59,14 +59,14 @@ class FileHandlerInterface(io.RawIOBase): @staticmethod def sanitize_filename(filename: str) -> str: - """Sanititizes the filename to ensure only a specific whitelist of characters is allowed through""" - allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]{}!$%^:#~?<>,|" + """Sanititizes the filename to ensure only a specific allow list of characters is allowed through""" + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]{}!$%^#~," result = "" for char in filename: if char in allowed: result += char else: - result += "?" + result += "_" # change unwanted chars to an underscore return result def __enter__(self): From 582feccf938a75571d455552c3a3e51c026d3a66 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 9 Dec 2024 12:56:07 -0600 Subject: [PATCH 117/348] Address feedback --- .../framework/plugins/windows/mftscan.py | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 014516b7c..feea78ece 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,7 +5,7 @@ import contextlib import datetime import logging -from typing import Generator, Iterable, Dict, Tuple +from typing import Generator, Iterable, Dict, Tuple, Callable from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -42,7 +42,15 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): context: interfaces.context.ContextInterface, config_path: str, primary_layer_name: str, - attr_callback, + attr_callback: Callable[ + [ + Dict[int, Tuple[str, int, int]], + interfaces.objects.ObjectInterface, + interfaces.objects.ObjectInterface, + str, + ], + Generator, + ], ) -> interfaces.objects.ObjectInterface: try: primary = context.layers[primary_layer_name] @@ -121,7 +129,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) @staticmethod - def parse_mft_records(record_map, mft_record, attr, symbol_table): + def parse_mft_records( + record_map: Dict[int, Tuple[str, int, int]], + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table_name: str, + ): # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record try: @@ -131,7 +144,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Standard Information Attribute if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": - si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + si_object = ( + symbol_table_name + constants.BANG + "STANDARD_INFORMATION_ENTRY" + ) attr_data = attr.Attr_Data.cast(si_object) yield 0, ( format_hints.Hex(attr_data.vol.offset), @@ -150,7 +165,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # File Name Attribute elif attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() @@ -201,9 +216,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): else: # past the first $DATA record, attempt to get the ADS name # NotAvailableValue = > 1st Data, but name was not parsable - ads_name = attr.get_resident_filename() - if not ads_name: - ads_name = renderers.NotAvailableValue() + ads_name = attr.get_resident_filename() or renderers.NotAvailableValue() content = attr.get_resident_filecontent() if content: @@ -227,7 +240,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, - symbol_table, + symbol_table_name: str, return_first_record: bool, ) -> Generator[Iterable, None, None]: """ @@ -240,7 +253,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # file name, DATA count, offset record_map[mft_record.vol.offset] = [renderers.NotAvailableValue(), 0, None] if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" attr_data = attr.Attr_Data.cast(fn_object) rec_name = attr_data.get_full_name() record_map[mft_record.vol.offset][0] = rec_name @@ -337,10 +350,10 @@ class ADS(interfaces.plugins.PluginInterface): record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, - symbol_table, + symbol_table_name: str, ): return MFTScan.parse_data_records( - record_map, mft_record, attr, symbol_table, False + record_map, mft_record, attr, symbol_table_name, False ) def _generator(self): @@ -406,10 +419,10 @@ class ResidentData(interfaces.plugins.PluginInterface): record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, - symbol_table, + symbol_table_name: str, ): return MFTScan.parse_data_records( - record_map, mft_record, attr, symbol_table, True + record_map, mft_record, attr, symbol_table_name, True ) def _generator(self): From e66a3e929b3c1253255f762db5677efd6c0510ec Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Thu, 12 Sep 2024 18:51:47 -0500 Subject: [PATCH 118/348] Add detection of direct and indirect system calls --- .../plugins/windows/direct_system_calls.py | 450 ++++++++++++++++++ .../plugins/windows/indirect_system_calls.py | 124 +++++ 2 files changed, 574 insertions(+) create mode 100644 volatility3/framework/plugins/windows/direct_system_calls.py create mode 100644 volatility3/framework/plugins/windows/indirect_system_calls.py diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py new file mode 100644 index 000000000..409f24750 --- /dev/null +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -0,0 +1,450 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import capstone +from collections import namedtuple +from typing import List, Tuple, Optional, Generator, Callable + +from volatility3.framework.objects import utility +from volatility3.framework import interfaces, renderers, symbols, exceptions +from volatility3.framework.configuration import requirements +from volatility3.plugins import yarascan +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + + +# Full details on the techniques used in these plugins to detect EDR-evading malware +# can be found in our 20 page whitepaper submitted to DEFCON along with the presentation +# https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + +syscall_finder_type = namedtuple( + "syscall_finder_type", + [ + "get_syscall_target_address", + "wants_syscall_inst", + "rule_str", + "invalid_ops", + "termination_ops", + ], +) + +syscall_finder_type.__doc__ = """ +This type to used to specify how malicious system call invocations should be found. + +`get_syscall_target_address` is optionally used to extract the address containing the malicious 'syscall' instruction +`wants_syscall_inst` whether or not this method expects the 'syscall' instrunction directly within the malicious code block +`rule` the opcode string to search for the malicious syscall instructions +`invalid_ops` instructions that only appear in invalid code blocks. Stops processing of the code block when encountered. +`termination_ops` instructions that are expected to be present in the code block and that stop processing +""" + + +class DirectSystemCalls(interfaces.plugins.PluginInterface): + """Detects the Direct System Call technique used to bypass EDRs""" + + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + valid_syscall_handlers = ("ntdll.dll", "win32u.dll") + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.syscall_finder = syscall_finder_type( + # for direct system calls, we find the `syscall` instruction directly, so we already know the address + None, + # yes, we want the syscall instruction present as it is what this technique looks for + True, + # regex to find "\x0f\x05" (syscall) followed later by "\xc3" (ret) + # we allow spacing in between to break naive anti-analysis forms (e.g., TarTarus Gate) + # Standard techniques, such as HellsGate, look like: + # mov r10, rcx + # mov eax, + # syscall + # ret + "/\\x0f\\x05[^\\xc3]{,24}\\xc3/", + # any of these will not be in a workable, malicious direct system call block + ["jmp", "call", "leave", "int3"], + # the expected form is to end with a "ret" back to the calling code + ["ret"], + ) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # create a list of requirements for vadyarascan + vadyarascan_requirements = [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) + ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) + ), + ] + + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() + + # return the combined requirements + return yarascan_requirements + vadyarascan_requirements + + @staticmethod + def _is_syscall_block( + disasm_func: Callable, + syscall_finder: syscall_finder_type, + data: bytes, + address: int, + ) -> Optional[Tuple[str, capstone._cs_insn]]: + """ + Determines if the bytes starting at `data` represent a valid syscall instrunction invocation block + + To maliciously invoke the system call instruction, malware must do each of the following: + + 1) update RAX to the system call number + 2) update R10 to the first parameter + 3) hit the 'termination' instrunction set in `syscall_finder_type` + + We also track whether the 'syscall' instruction was encountered while parsing + + This function is reusable for every technique we found and studied during the DEFCON research timeframe + + Args: + disasm_func: capstone disassembly function gathered from `get_disasm_function` + syscall_finder: the method and constraints on the malicious system call blocks that the calling plugin knows how to find + data: the bytes from memory to search for malicious syscall invocations + address: the address from where `data` came from in the particular process + Returns: + Optional[Tuple[str, capstone._cs_insn]]: For valid blocks, the disassembled bytes in string from and the last (termination) instruction + """ + found_movr10 = False + found_movreax = False + found_syscall = False + found_end = False + end_inst = None + + disasm_bytes = "" + + for inst in disasm_func(data, address): + disasm_bytes += f"{inst.address:#x}: {inst.mnemonic} {inst.op_str}; " + + # an instruction of all 0x00 opcodes + if inst.opcode.count(0) == len(inst.opcode): + break + + op = inst.mnemonic + + # invalid op, bail + if op in syscall_finder.invalid_ops: + break + + # found the end instruction wanted by the caller + elif op in syscall_finder.termination_ops: + found_end = True + end_inst = inst + break + + # track this no matter what to make code more re-usable + elif op == "syscall": + found_syscall = True + + # if we hit a 'syscall' but RAX or R10 haven't been touched + # then we are in an invalid path, so bail + if not syscall_finder.wants_syscall_inst or ( + not (found_movr10 and found_movreax) + ): + break + + else: + # attempt to see if any other instruction type wrote to registers + try: + _, regs_written = inst.regs_access() + except capstone.CsError: + continue + + if regs_written: + for r in regs_written: + # track writes to eax/rax or R10 + reg = inst.reg_name(r) + if reg in ["eax", "rax"]: + found_movreax = True + + elif reg == "r10": + found_movr10 = True + + # if any of these are missing, the block is invalid regardless of + # the technique we are trying to detect now or in the future + if not (found_movr10 and found_movreax and found_end): + return None + + # if the finder requires a 'syscall' instruction then bail now if we didn't find one + if syscall_finder.wants_syscall_inst and not found_syscall: + return None + + return disasm_bytes, end_inst + + @staticmethod + def get_disasm_function(architecture: str) -> Callable: + """ + Returns the disassembly handler for the given architecture + .detail is used to get full instruction information + + Args: + architecture: the name of the architecture for the process being disassembled + Returns: + The disasm function from capstone for the given architecture + """ + disasm_types = { + "intel": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32), + "intel64": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64), + } + + disasm_type = disasm_types[architecture] + disasm_type.detail = True + return disasm_type.disasm + + @classmethod + def _is_valid_syscall( + cls, + syscall_finder: syscall_finder_type, + proc_layer: interfaces.layers.DataLayerInterface, + architecture: str, + vads: List[Tuple[int, int, str]], + address: int, + ) -> Optional[Tuple[int, str]]: + """ + Args: + syscall_finder: + proc_layer: the memory layer of the process being scanned + architecture: the name of the architecture for the process being disassembled + vads: the ranges of this process under 10MB + address: the starting address to check for malicious syscall code blocks + + Returns: + Optional[Tuple[int, str]]: For valid code blocks, the starting address of the block and the disassembly string + """ + # the number bytes behind the yara rule hit to scan + behind = 32 + + address = address - behind + + try: + data = proc_layer.read(address, behind * 2) + except exceptions.InvalidAddressException: + return None + + disasm_func = cls.get_disasm_function(architecture) + + # since Intel does not have fixed-size instructions, we have to scan + # each byte offset and re-disassemble the remaining block + for offset in range(behind): + # if this looks like a system call back (r10, rax, ret/jmp) + syscall_info = cls._is_syscall_block( + disasm_func, syscall_finder, data[offset:], address + offset + ) + if syscall_info: + disasm_bytes, end_inst = syscall_info + + # if we can recover (and require) a target address for this malware technique + if syscall_finder.get_syscall_target_address: + target_address = syscall_finder.get_syscall_target_address( + proc_layer, end_inst + ) + + # could not determine the address -> invalid basic block + if not target_address: + continue + + # we only care about calls to system call DLLs + path = cls._get_range_path(vads, target_address) + if not isinstance(path, str) or not path.lower().endswith( + cls.valid_syscall_handlers + ): + continue + + # return the address and disassembly string if all checks pass + return address + offset, disasm_bytes + + return None + + @staticmethod + def _get_vad_maps( + task: interfaces.objects.ObjectInterface, + ) -> List[Tuple[int, int, str]]: + """Creates a map of start/end addresses within a virtual address + descriptor tree. + + Args: + task: The EPROCESS object of which to traverse the vad tree + + Returns: + An iterable of tuples containing start and end addresses for each descriptor + """ + vads: List[Tuple[int, int, str]] = [] + + # scan regions under 10MB + scan_max = 10 * 1000 * 1000 + + vad_root = task.get_vad_root() + + for vad in vad_root.traverse(): + if vad.get_size() < scan_max: + vads.append((vad.get_start(), vad.get_size(), vad.get_file_name())) + + return vads + + @staticmethod + def _get_range_path(ranges: List[Tuple[int, int, str]], address: int) -> Optional[str]: + """ + Returns the path for the range holding `address`, if found + + Args: + ranges: VADs collected from `_get_vad_maps` + address: the address to find + Returns: + The path holding the address, if any + """ + for start, size, path in ranges: + if start <= address < start + size: + return path + + return None + + @classmethod + def _get_tasks_to_scan( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table_name: str, + ) -> Generator[ + Tuple[interfaces.objects.ObjectInterface, str, str, str], None, None + ]: + """ + Gathers active processes with the extra information needed + to detect malicious syscall instructions + + Returns: + Generator of the process object, name, memory layer, and architecture + """ + + # gather active processes + filter_func = pslist.PsList.create_active_process_filter() + + is_32bit_arch = not symbols.symbol_table_is_64bit(context, symbol_table_name) + + for proc in pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, + filter_func=filter_func, + ): + proc_name = utility.array_to_string(proc.ImageFileName) + + # skip Defender + if proc_name in ["MsMpEng.exe"]: + continue + + try: + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + if is_32bit_arch or proc.get_is_wow64(): + architecture = "intel" + else: + architecture = "intel64" + + yield proc, proc_name, proc_layer_name, architecture + + @classmethod + def _get_rule_hits( + cls, + context: interfaces.objects.ObjectInterface, + proc_layer: interfaces.layers.DataLayerInterface, + vads: List[Tuple[int, int, str]], + pattern: str, + ) -> Generator[Tuple[int, Optional[str]], None, None]: + """ + Runs the given opcode rule through Yara and returns the address and file path of hits + + Args: + context: + proc_layer: the layer to scan + vads: the ranges inside of the process being scanned + pattern: the opcodes rule from the plugin to detect a particular EDR-bypass technique + + Returns: + Generator of the address and file path of hits + """ + sections = [(vad[0], vad[1]) for vad in vads] + + rule = yarascan.YaraScanner.get_rule(pattern) + + for hit in proc_layer.scan( + context=context, + scanner=yarascan.YaraScanner(rules=rule), + sections=sections, + ): + address = hit[0] + + path = cls._get_range_path(vads, address) + + # ignore hits in the system call DLLs + if isinstance(path, str) and path.lower().endswith( + cls.valid_syscall_handlers + ): + continue + + yield address, path + + def _generator(self) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: + kernel = self.context.modules[self.config["kernel"]] + + for proc, proc_name, proc_layer_name, architecture in self._get_tasks_to_scan( + self.context, kernel.layer_name, kernel.symbol_table_name + ): + proc_layer = self.context.layers[proc_layer_name] + + vads = self._get_vad_maps(proc) + + # for each valid process, look for malicious syscall invocations + for address, vad_path in self._get_rule_hits( + self.context, proc_layer, vads, self.syscall_finder.rule_str + ): + syscall_info = self._is_valid_syscall( + self.syscall_finder, proc_layer, architecture, vads, address + ) + if not syscall_info: + continue + + address, disasm_bytes = syscall_info + + yield 0, ( + proc_name, + proc.UniqueProcessId, + vad_path, + format_hints.Hex(address), + disasm_bytes, + ) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("Range", str), + ("Address", format_hints.Hex), + ("Disasm", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py new file mode 100644 index 000000000..417c1b46e --- /dev/null +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -0,0 +1,124 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import struct +import logging +from typing import List, Optional + +import capstone + +from volatility3.framework import interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.plugins import yarascan +from volatility3.plugins.windows import pslist, direct_system_calls + +vollog = logging.getLogger(__name__) + + +class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.syscall_finder = direct_system_calls.syscall_finder_type( + # gets the target address of a indirect jmp + self._indirect_syscall_block_target, + # we are looking for indirect system calls, so we don't want 'syscall' instructions in our code block + False, + # jmp [address]; ret + "/\\xff\\x25[^\\xc3]{,24}\\xc3/", + # any of these mean we aren't in a malicious indirect call + ["call", "leave", "int3", "ret"], + # stop at jmp, this should reference the system call instruction + ["jmp"], + ) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # create a list of requirements for vadyarascan + vadyarascan_requirements = [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) + ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="direct_system_calls", + plugin=direct_system_calls.DirectSystemCalls, + version=(1, 0, 0), + ), + ] + + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() + + # return the combined requirements + return yarascan_requirements + vadyarascan_requirements + + @staticmethod + def _indirect_syscall_block_target( + proc_layer: interfaces.layers.DataLayerInterface, inst: capstone._cs_insn + ) -> Optional[int]: + """ + This function determines the address of a jmp in the following form: + + jmp [address] + + To determine this, we must: + 1) Pull the 4 byte relative offset of 'address' inside the instruction + 2) Compute the full address of this relative offset + 3) Read from the address as it is being dereferenced + 4) Ensure the target address points to a 'syscall' instruction + + Args: + proc_layer: the layer of the potential syscall block + inst: the terminating instruction of the syscall block check + Returns: + The target address of the jump if it can be computed + """ + + try: + jmp_address_str = proc_layer.read(inst.address, 6) + except exceptions.InvalidAddressException: + return None + + # Should be an jmp... + if jmp_address_str[0:2] != b"\xff\x25": + return None + + # get the address of the 'jmp [address]' instrunction + relative_offset = struct.unpack(" Date: Thu, 12 Sep 2024 18:58:16 -0500 Subject: [PATCH 119/348] Fix formatting problem between black versions --- .../framework/plugins/windows/direct_system_calls.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 409f24750..1edc01671 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -304,7 +304,9 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return vads @staticmethod - def _get_range_path(ranges: List[Tuple[int, int, str]], address: int) -> Optional[str]: + def _get_range_path( + ranges: List[Tuple[int, int, str]], address: int + ) -> Optional[str]: """ Returns the path for the range holding `address`, if found @@ -407,7 +409,9 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): yield address, path - def _generator(self) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: + def _generator( + self, + ) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: kernel = self.context.modules[self.config["kernel"]] for proc, proc_name, proc_layer_name, architecture in self._get_tasks_to_scan( From fce2125a8e4d2742720cc90f9d80e1e2fed9f4c8 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 15:03:51 -0500 Subject: [PATCH 120/348] Make VAD API public as intended --- .../plugins/windows/direct_system_calls.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 1edc01671..1999563fd 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -49,6 +49,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): _required_framework_version = (2, 4, 0) _version = (1, 0, 0) + # DLLs that are expected to host system call invocations valid_syscall_handlers = ("ntdll.dll", "win32u.dll") def __init__(self, *args, **kwargs): @@ -266,7 +267,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): continue # we only care about calls to system call DLLs - path = cls._get_range_path(vads, target_address) + path = cls.get_range_path(vads, target_address) if not isinstance(path, str) or not path.lower().endswith( cls.valid_syscall_handlers ): @@ -278,7 +279,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return None @staticmethod - def _get_vad_maps( + def get_vad_maps( task: interfaces.objects.ObjectInterface, ) -> List[Tuple[int, int, str]]: """Creates a map of start/end addresses within a virtual address @@ -304,14 +305,14 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return vads @staticmethod - def _get_range_path( + def get_range_path( ranges: List[Tuple[int, int, str]], address: int ) -> Optional[str]: """ Returns the path for the range holding `address`, if found Args: - ranges: VADs collected from `_get_vad_maps` + ranges: VADs collected from `get_vad_maps` address: the address to find Returns: The path holding the address, if any @@ -399,7 +400,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): ): address = hit[0] - path = cls._get_range_path(vads, address) + path = cls.get_range_path(vads, address) # ignore hits in the system call DLLs if isinstance(path, str) and path.lower().endswith( @@ -419,7 +420,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): ): proc_layer = self.context.layers[proc_layer_name] - vads = self._get_vad_maps(proc) + vads = self.get_vad_maps(proc) # for each valid process, look for malicious syscall invocations for address, vad_path in self._get_rule_hits( From 262c7f1aa7e7113508d067afc0e45219643ab943 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 15:22:50 -0500 Subject: [PATCH 121/348] Make VAD API public as intended --- volatility3/framework/plugins/windows/direct_system_calls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 1999563fd..151776e75 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -324,7 +324,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return None @classmethod - def _get_tasks_to_scan( + def get_tasks_to_scan( cls, context: interfaces.context.ContextInterface, layer_name: str, @@ -415,7 +415,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): ) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: kernel = self.context.modules[self.config["kernel"]] - for proc, proc_name, proc_layer_name, architecture in self._get_tasks_to_scan( + for proc, proc_name, proc_layer_name, architecture in self.get_tasks_to_scan( self.context, kernel.layer_name, kernel.symbol_table_name ): proc_layer = self.context.layers[proc_layer_name] From e7fca5a83f6607a72183d560d4646121d85291e2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 16 Sep 2024 09:38:15 -0500 Subject: [PATCH 122/348] Update year --- volatility3/framework/plugins/windows/direct_system_calls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 151776e75..eac18e8d4 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # From af65d7e32daf778488928960e176d8774a2ed1b1 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 18 Oct 2024 11:33:38 -0500 Subject: [PATCH 123/348] Address feedback --- .../plugins/windows/direct_system_calls.py | 16 ++++++++++++++-- .../plugins/windows/indirect_system_calls.py | 10 +++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index eac18e8d4..ec8cd811b 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -3,7 +3,7 @@ # import logging -import capstone + from collections import namedtuple from typing import List, Tuple, Optional, Generator, Callable @@ -16,6 +16,12 @@ from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) +try: + import capstone + + has_capstone = True +except ImportError: + has_capstone = False # Full details on the techniques used in these plugins to detect EDR-evading malware # can be found in our 20 page whitepaper submitted to DEFCON along with the presentation @@ -114,7 +120,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): 1) update RAX to the system call number 2) update R10 to the first parameter - 3) hit the 'termination' instrunction set in `syscall_finder_type` + 3) hit the 'termination' instruction set in `syscall_finder_type` We also track whether the 'syscall' instruction was encountered while parsing @@ -413,6 +419,12 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): def _generator( self, ) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: + if not has_capstone: + vollog.warning( + "capstone is not installed. This plugin requires capstone to operate." + ) + return + kernel = self.context.modules[self.config["kernel"]] for proc, proc_name, proc_layer_name, architecture in self.get_tasks_to_scan( diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index 417c1b46e..b6494645e 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -6,8 +6,6 @@ import struct import logging from typing import List, Optional -import capstone - from volatility3.framework import interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.plugins import yarascan @@ -15,6 +13,12 @@ from volatility3.plugins.windows import pslist, direct_system_calls vollog = logging.getLogger(__name__) +# The generator of DirectSystemCalls will bail with a warning if capstone is not installed +try: + import capstone +except ImportError: + pass + class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): _required_framework_version = (2, 4, 0) @@ -98,7 +102,7 @@ class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): if jmp_address_str[0:2] != b"\xff\x25": return None - # get the address of the 'jmp [address]' instrunction + # get the address of the 'jmp [address]' instruction relative_offset = struct.unpack(" Date: Mon, 9 Dec 2024 13:19:06 -0600 Subject: [PATCH 124/348] Add capstone to test system requirements. Allow lazy type checks --- pyproject.toml | 1 + volatility3/framework/plugins/windows/direct_system_calls.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fc1ab96cb..9b4b8d485 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dev = [ test = [ "volatility3[dev]", "pytest>=8.3.3,<9", + "capstone>=5.0.3,<6", "yara-x>=0.10.0,<1", ] diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index ec8cd811b..51035b2bc 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -112,7 +112,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): syscall_finder: syscall_finder_type, data: bytes, address: int, - ) -> Optional[Tuple[str, capstone._cs_insn]]: + ) -> Optional[Tuple[str, "capstone._cs_insn"]]: """ Determines if the bytes starting at `data` represent a valid syscall instrunction invocation block From a07ee5a0d53b553b854a8f2ff697ddf7922255a2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 9 Dec 2024 17:49:22 -0600 Subject: [PATCH 125/348] fix(Windows: Handles): Unreliable SAR value on 24H2 Handles are not being decoded in 24H2+ samples. This is because the `Handles._decode_pointer` method grabs the SAR shift value from the disassemble function, but in these samples this value (`0x11`) is incorrect. Adding a fallback to the default SAR value of `0x10` if the obtained pointer is not valid in the kernel address space resolves the issue. --- volatility3/framework/plugins/windows/handles.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 3e5a2fd82..e5cbbf4ca 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -21,11 +21,14 @@ except ImportError: has_capstone = False +DEFAULT_SAR_VALUE = 0x10 # to be used only when decoding fails + + class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -118,6 +121,10 @@ class Handles(interfaces.plugins.PluginInterface): ) offset = self._decode_pointer(handle_table_entry.LowValue, magic) + if not self.context.layers[virtual].is_valid(offset): + offset = self._decode_pointer( + handle_table_entry.LowValue, DEFAULT_SAR_VALUE + ) else: if handle_table_entry.InfoTable == 0: return None @@ -142,7 +149,6 @@ class Handles(interfaces.plugins.PluginInterface): pointers in the _HANDLE_TABLE_ENTRY which allows us to find the associated _OBJECT_HEADER. """ - DEFAULT_SAR_VALUE = 0x10 # to be used only when decoding fails if self._sar_value is None: if not has_capstone: From 1cde9ae06ee855e084fe5631eb37f2ffa979ef5c Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 09:46:28 +0000 Subject: [PATCH 126/348] Slightly modify volshell.rst --- doc/source/volshell.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 3c4f4ce5d..c95456dda 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -36,7 +36,7 @@ operating system mode for volshell, and the current layer available for use. (primary) >>> -Volshell itself in essentially a plugin, but an interactive one. As such, most values are accessed through `self` +Volshell itself is essentially a plugin, but an interactive one. As such, most values are accessed through `self` although there is also a `context` object whenever a context must be provided. The prompt for the tool will indicate the name of the current layer (which can be accessed as `self.current_layer` @@ -92,7 +92,7 @@ It can also be provided with an object and will interpret the data for each in t 0x2e8 : UniqueProcessId symbol_table_name1!pointer 4 ... -These values can be accessed directory as attributes +These values can be accessed directly as attributes :: @@ -180,7 +180,7 @@ used: layer = cc(mynewlayer.MyNewLayer, on_top_of = 'primary', other_parameter = 'important') with open('output.dmp', 'wb') as fp: - for i in range(0, 1073741824, 0x1000): + for i in range(0, 0x4000000, 0x1000): data = layer.read(i, 0x1000, pad = True) fp.write(data) From fdd49d0921a8ca22ea388573dec33ba93a1ef485 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 10:11:26 +0000 Subject: [PATCH 127/348] Slightly modify documentation --- doc/source/glossary.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 66dabfafe..c4a93f908 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -145,9 +145,9 @@ Struct, Structure Symbol This is used in many different contexts, as a short term for many things. Within Volatility, a symbol is a - construct that usually encompasses a specific type :ref:`type` at a specific :ref:`offset`, + construct that usually encompasses a specific :ref:`type` at a specific :ref:`offset`, representing a particular instance of that type within the memory of a compiled and running program. An example - would be the location in memory of a list of active tcp endpoints maintained by the networking stack + would be the location in memory of a list of active TCP endpoints maintained by the networking stack within an operating system. T From 4bccf116292e6269f0ecc306b8dba0973af55697 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:31:16 +0000 Subject: [PATCH 128/348] Remove redundant part of if statement Also reorder imports. --- volatility3/cli/volshell/generic.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 82c470e1a..534546dcd 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -11,11 +11,6 @@ import sys from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union from urllib import parse, request -from volatility3.cli import text_renderer, volshell -from volatility3.framework import exceptions, interfaces, objects, plugins, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.layers import intel, physical, resources, scanners - try: import capstone @@ -23,6 +18,11 @@ try: except ImportError: has_capstone = False +from volatility3.cli import text_renderer, volshell +from volatility3.framework import exceptions, interfaces, objects, plugins, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import intel, physical, resources, scanners + class Volshell(interfaces.plugins.PluginInterface): """Shell environment to directly interact with a memory image.""" @@ -553,12 +553,11 @@ class Volshell(interfaces.plugins.PluginInterface): if argname in kwargs: del kwargs[argname] - for keyword in kwargs: - val = kwargs[keyword] + for keyword, val in kwargs.items(): if not isinstance( val, interfaces.configuration.BasicTypes ) and not isinstance(val, list): - if not isinstance(val, list) or all( + if all( isinstance(x, interfaces.configuration.BasicTypes) for x in val ): raise TypeError( From faa6cab797da8719305f49e1e824448a159509eb Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:16:42 +0000 Subject: [PATCH 129/348] Remove redundant part of if statement Also reorder imports. --- volatility3/cli/volshell/generic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 534546dcd..b1a61fcff 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -555,8 +555,8 @@ class Volshell(interfaces.plugins.PluginInterface): for keyword, val in kwargs.items(): if not isinstance( - val, interfaces.configuration.BasicTypes - ) and not isinstance(val, list): + val, (interfaces.configuration.BasicTypes, list) + ): if all( isinstance(x, interfaces.configuration.BasicTypes) for x in val ): From 0b2f4fdeb772ccb097aaf310ff3169ae37279f13 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:26:39 +0000 Subject: [PATCH 130/348] Remove redundant part of if statement Also reorder imports. --- volatility3/cli/volshell/generic.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index b1a61fcff..08132608b 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -554,12 +554,8 @@ class Volshell(interfaces.plugins.PluginInterface): del kwargs[argname] for keyword, val in kwargs.items(): - if not isinstance( - val, (interfaces.configuration.BasicTypes, list) - ): - if all( - isinstance(x, interfaces.configuration.BasicTypes) for x in val - ): + if not isinstance(val, (interfaces.configuration.BasicTypes, list)): + if all(isinstance(x, interfaces.configuration.BasicTypes) for x in val): raise TypeError( "Configurable values must be simple types (int, bool, str, bytes)" ) From 39cb75accae827436d8d923592f1d62b34cf9ede Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:02:48 +0000 Subject: [PATCH 131/348] Slightly modify documentation Include regex_scan, new functionality of volshell. --- doc/source/volshell.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index c95456dda..73b000763 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -187,8 +187,22 @@ used: As this demonstrates, all of the python is accessible, as are the volshell built in functions (such as `cc` which creates a constructable, like a layer or a symbol table). +User Convenience +---------------- + +There are functions available that make often-done tasks easiers, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is mentioned when volshell starts. + Loading files -------------- +^^^^^^^^^^^^^ Files can be loaded as physical layers using the `load_file` or `lf` command, which takes a filename or a URI. This will be added to `context.layers` and can be accessed by the name returned by `lf`. + +Regex +^^^^^ + +It is easy to scan for some bytes or a pattern using `regex_scan` or `rx`. + +An optional size can be given for the displayed results as with the other fuctions (db, dw, dd, dq, etc). + +You can of course specify a different layer name as well. From fc33fd912787d0420cd9c4c5effbfba0ea89a933 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:14:59 +0000 Subject: [PATCH 132/348] Slightly modify documentation Include regex_scan, new functionality of volshell. --- doc/source/volshell.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 73b000763..b46647dc3 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -190,7 +190,7 @@ creates a constructable, like a layer or a symbol table). User Convenience ---------------- -There are functions available that make often-done tasks easiers, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is mentioned when volshell starts. +There are functions available that make often-done tasks easier, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is advertised when volshell starts. Loading files ^^^^^^^^^^^^^ From b6717d80d9ac5c820cf84803b53c9aee1e115464 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 10 Dec 2024 23:55:44 +0000 Subject: [PATCH 133/348] Windows: Fix up minor typo and CodeQL warning --- volatility3/framework/plugins/windows/direct_system_calls.py | 2 +- volatility3/framework/plugins/windows/indirect_system_calls.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 51035b2bc..b0c162f46 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -39,7 +39,7 @@ syscall_finder_type = namedtuple( ) syscall_finder_type.__doc__ = """ -This type to used to specify how malicious system call invocations should be found. +This type is used to specify how malicious system call invocations should be found. `get_syscall_target_address` is optionally used to extract the address containing the malicious 'syscall' instruction `wants_syscall_inst` whether or not this method expects the 'syscall' instrunction directly within the malicious code block diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index b6494645e..1a5eb317f 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -13,10 +13,10 @@ from volatility3.plugins.windows import pslist, direct_system_calls vollog = logging.getLogger(__name__) -# The generator of DirectSystemCalls will bail with a warning if capstone is not installed try: import capstone except ImportError: + # The generator of DirectSystemCalls will bail with a warning if capstone is not installed pass From 3d260f3829e5f7bd4611db726358169498bb6ae8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:42:20 +0000 Subject: [PATCH 134/348] Slightly modify documentation Include regex_scan, new functionality of volshell. --- doc/source/volshell.rst | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index b46647dc3..47ea2e905 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -203,6 +203,45 @@ Regex It is easy to scan for some bytes or a pattern using `regex_scan` or `rx`. +:: + + (layer_name) >>> rx(rb"(Linux version|Darwin Kernel Version) [0-9]+\.[0-9]+\.[0-9]+") + 0x880001400070 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0x880001400080 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0x880001400090 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0x8800014000a0 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0x8800014000b0 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0x8800014000c0 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0x8800014000d0 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0x8800014000e0 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0x880001769027 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0x880001769037 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0x880001769047 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0x880001769057 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0x880001769067 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0x880001769077 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0x880001769087 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0x880001769097 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0xffff81400070 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0xffff81400080 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0xffff81400090 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0xffff814000a0 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0xffff814000b0 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0xffff814000c0 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0xffff814000d0 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0xffff814000e0 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0xffff81769027 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0xffff81769037 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0xffff81769047 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0xffff81769057 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0xffff81769067 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0xffff81769077 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0xffff81769087 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0xffff81769097 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + An optional size can be given for the displayed results as with the other fuctions (db, dw, dd, dq, etc). -You can of course specify a different layer name as well. +You can, of course, specify a different layer name as well. From 9d0cd4b4c985a568083e20cb4ff13164e6d60963 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Dec 2024 11:17:15 +1100 Subject: [PATCH 135/348] Linux: PageCache: Update inode plugin to conform to framework dumping convention --- .../framework/plugins/linux/pagecache.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 005fc9acc..6d2607ada 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -389,7 +389,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -412,9 +412,10 @@ class InodePages(plugins.PluginInterface): description="Inode address", optional=True, ), - requirements.StringRequirement( + requirements.BooleanRequirement( name="dump", - description="Output file path", + description="Extract inode content", + default=False, optional=True, ), ] @@ -436,7 +437,7 @@ class InodePages(plugins.PluginInterface): """ if not inode.is_reg: vollog.error("The inode is not a regular file") - return + return None # By using truncate/seek, provided the filesystem supports it, a sparse file will be # created, saving both disk space and I/O time. @@ -471,7 +472,7 @@ class InodePages(plugins.PluginInterface): if self.config["inode"] and self.config["find"]: vollog.error("Cannot use --inode and --find simultaneously") - return + return None if self.config["find"]: inodes_iter = Files.get_inodes( @@ -487,15 +488,15 @@ class InodePages(plugins.PluginInterface): inode = vmlinux.object("inode", self.config["inode"], absolute=True) else: vollog.error("You must use either --inode or --find") - return + return None if not inode.is_valid(): vollog.error("Invalid inode at 0x%x", inode.vol.offset) - return + return None if not inode.is_reg: vollog.error("The inode is not a regular file") - return + return None inode_size = inode.i_size for page_obj in inode.get_pages(): @@ -520,8 +521,13 @@ class InodePages(plugins.PluginInterface): if self.config["dump"]: filename = self.config["dump"] - vollog.info("[*] Writing inode at 0x%x to '%s'", inode.vol.offset, filename) - self.write_inode_content_to_file(inode, filename, self.open, vmlinux_layer) + open_method = self.open + inode_address = inode.vol.offset + filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") + vollog.info("[*] Writing inode at 0x%x to '%s'", inode_address, filename) + self.write_inode_content_to_file( + inode, filename, open_method, vmlinux_layer + ) def run(self): headers = [ From 58a9c3d6dae0759e0dbf590d53e14d7553f30a28 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 07:20:30 +0000 Subject: [PATCH 136/348] Slightly modify documentation Include regex_scan, new functionality of volshell. Add Intermediate Symbol File (ISF) to glossary. --- doc/source/glossary.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index c4a93f908..a9460b1a2 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -23,7 +23,7 @@ Alignment .. _Array: Array - This represents a list of items, which can be access by an index, which is zero-based (meaning the first + This represents a list of items, which can be accessed by an index, which is zero-based (meaning the first element has index 0). Items in arrays are almost always the same size (it is not a generic list, as in python) even if they are :ref:`pointers` to different sized objects. @@ -43,7 +43,14 @@ Dereference .. _Domain: Domain - This the grouping for input values for a mapping or mathematical function. + The set of input values for a mapping or mathematical function. + +I +- +.. _Intermediate Symbol File (ISF): + +Intermediate Symbol File (ISF) + They contain kernel structures and specific offsets formatted as JSON. For macOS and Linux analysis, the kernel needs to be added as an ISF file to the volatility 3 symbols directory. For Windows, the required ISF file can often be generated from PDB files automatically downloaded from Microsoft servers, and therefore does not require manual intervention. M - @@ -55,7 +62,7 @@ Map, mapping attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). For further information, please see - `Function (mathematics) in wikipedia https://en.wikipedia.org/wiki/Function_(mathematics)` + `https://en.wikipedia.org/wiki/Function_(mathematics)`. .. _Member: @@ -69,7 +76,7 @@ O .. _Object: Object - This has a specific meaning within computer programming (as in Object Oriented Programming), but within the world + This has a specific meaning within computer programming (as in object-oriented programming), but within the world of Volatility it is used to refer to a type that has been associated with a chunk of data, or a specific instance of a type. See also :ref:`Type`. From 6ffef285f4c1ba39816d76726f00086029abdedc Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:36:51 +0000 Subject: [PATCH 137/348] Tweak the getting started linux tutorial --- doc/source/getting-started-linux-tutorial.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst index d4b40d053..a1aad235d 100644 --- a/doc/source/getting-started-linux-tutorial.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -27,7 +27,7 @@ To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol Listing plugins --------------- -The following is a sample of the linux plugins available for volatility3, it is not complete and more more plugins may +The following is a sample of the linux plugins available for volatility3, it is not complete and more plugins may be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. For plugin requests, please create an issue with a description of the requested plugin. @@ -40,7 +40,7 @@ For plugin requests, please create an issue with a description of the requested linux.check_creds.Check_creds linux.check_idt.Check_idt -.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of linux plugins. +.. note:: Here the the command is piped to grep and head to provide the start of the list of linux plugins. Using plugins @@ -80,9 +80,9 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. +If an ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. -.. tip:: Use the banner text which is most repeated to search from ISF Server. +.. tip:: Use the banner text which is most repeated to search on the ISF Server. linux.pslist ~~~~~~~~~~~~ @@ -157,7 +157,7 @@ linux.pstree ***** 1548 1266 gsd-keyboard ***** 1550 1266 gsd-media-keys -``linux.pstree`` helps us to display the parent child relationships between processes. +``linux.pstree`` helps us to display the parent-child relationships between processes. linux.bash ~~~~~~~~~~ From e31e13f471006f7dcff11f8b7f601b45a9cdf471 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:44:01 +0000 Subject: [PATCH 138/348] Tweak the getting started mac tutorial --- doc/source/getting-started-mac-tutorial.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index 42e58c0d5..61af7089b 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -37,7 +37,7 @@ For plugin requests, please create an issue with a description of the requested mac.check_sysctl.Check_sysctl mac.check_trap_table.Check_trap_table -.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of macOS plugins. +.. note:: Here the the command is piped to grep and head to provide the start of the list of macOS plugins. Using plugins @@ -78,7 +78,7 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-mac-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. +If an ISF file cannot be found then, follow the instructions on :ref:`getting-started-mac-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. mac.pslist ~~~~~~~~~~ @@ -125,7 +125,7 @@ mac.pstree 337 1 system_installd * 455 337 update_dyld_shar -``mac.pstree`` helps us to display the parent child relationships between processes. +``mac.pstree`` helps us to display the parent-child relationships between processes. mac.ifconfig ~~~~~~~~~~~~ @@ -150,4 +150,4 @@ mac.ifconfig utun0 False utun0 fe80:5::2a95:bb15:87e3:977c False -we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. +We can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. From d77d696e2b017fa7dc8b2355efa9cdde88470d6e Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:49:25 +0000 Subject: [PATCH 139/348] Tweak the getting started windows tutorial --- doc/source/getting-started-windows-tutorial.rst | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/doc/source/getting-started-windows-tutorial.rst b/doc/source/getting-started-windows-tutorial.rst index c89b065f5..979cf1d96 100644 --- a/doc/source/getting-started-windows-tutorial.rst +++ b/doc/source/getting-started-windows-tutorial.rst @@ -15,19 +15,19 @@ Memory can be acquired using a number of tools, below are some examples but othe Listing Plugins --------------- -The following is a sample of the windows plugins available for volatility3, it is not complete and more more plugins may +The following is a sample of the windows plugins available for volatility3, it is not complete and more plugins may be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. For plugin requests, please create an issue with a description of the requested plugin. .. code-block:: shell-session - $ python3 vol.py --help | grep windows | head -n 5 + $ python3 vol.py --help | grep windows | head -n 4 windows.bigpools.BigPools windows.cmdline.CmdLine windows.crashinfo.Crashinfo windows.dlllist.DllList -.. note:: Here the the command is piped to grep and head in-order to provide the start of a list of the available windows plugins. +.. note:: Here the the command is piped to grep and head to provide the start of a list of the available windows plugins. Using plugins ------------- @@ -95,9 +95,9 @@ windows.pstree ** 616 504 svchost.exe 0xfa8002b86ab0 13 314 0 False 2022-02-07 16:32:16.000000 N/A ** 624 504 svchost.exe 0xfa8002410630 10 350 0 False 2022-02-07 16:30:14.000000 N/A -``windows.pstree`` helps to display the parent child relationships between processes. +``windows.pstree`` helps to display the parent-child relationships between processes. -.. note:: Here the the command is piped to head in-order to provide smaller output, here listing only the first 20. +.. note:: Here the the command is piped to head to provide smaller output, here listing only the first 20. windows.hashdump ~~~~~~~~~~~~~~~~ @@ -116,9 +116,3 @@ windows.hashdump Dennis 1003 aad3b435b51404eeaad3b435b51404ee cf96684bbc7877920adaa9663698bf54 ``windows.hashdump`` helps to list the hashes of the users in the system. - - - - - - From c740a6c77017834329f0361866536509938bc0c4 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:14:35 +0000 Subject: [PATCH 140/348] Modify using as a library documentation Tiny changes. --- doc/source/using-as-a-library.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index 4acf35f98..55b77e90a 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -3,7 +3,7 @@ Using Volatility 3 as a Library This portion of the documentation discusses how to access the Volatility 3 framework from an external application. -The general process of using volatility as a library is to as follows: +The general process of using volatility as a library is as follows: 1. :ref:`create_context` 2. (Optional) :ref:`available_plugins` @@ -21,7 +21,7 @@ Creating a context First we make sure the volatility framework works the way we expect it (and is the version we expect). The versioning used is semantic versioning, meaning any version with the same major number and a higher or equal minor number will satisfy the requirement. An example is below since the CLI doesn't need any of the features -from versions 1.1 or 1.2: +from version 1.1: :: @@ -86,7 +86,7 @@ List requirements are a list of simple types (integers, booleans, floats and str options, multiple requirements needs all their subrequirements fulfilled and the other types require the names of valid translation layers or symbol tables within the context, respectively. Luckily, each of these requirements can tell you whether they've been fulfilled or not later in the process. For now, they can be used to ask the user to -fill in any parameters they made need to. Some requirements are optional, others are not. +fill in any parameters they may need to. Some requirements are optional, others are not. The plugin is essentially a multiple requirement. It should also be noted that automagic classes can have requirements (as can translation layers). @@ -100,7 +100,7 @@ Once you know what requirements the plugin will need, you can populate them with The configuration is essentially a hierarchical tree of values, much like the windows registry. Each plugin is instantiated at a particular branch within the hierarchy and will look for its configuration options under that hierarchy (if it holds any configurable items, it will likely instantiate those at a point -underneaths its own branch). To set the hierarchy, you'll need to know where the configurables will be constructed. +underneath its own branch). To set the hierarchy, you'll need to know where the configurables will be constructed. For this example, we'll assume plugins' base_config_path is set as `plugins`, and that automagics are configured under the `automagic` tree. We'll see later how to ensure this matches up with the plugins and automagic when they're @@ -139,7 +139,7 @@ A suitable list of automagics for a particular plugin (based on operating system This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just the automagics which apply to the operating system. Each automagic can exclude itself from being used for specific -operating systems, so that an automagic designed for linux is not used for windows or mac plugins. +operating systems, so that for example an automagic designed for linux is not used for windows or mac plugins. These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that the plugin will be constructed on ('plugins' by default) and a progress_callback. This is a callable which takes @@ -157,8 +157,8 @@ Any exceptions that occur during the execution of the automagic will be returned Run the plugin -------------- -Firstly, we should check whether the plugin will be able to run (ie, whether the configuration options it needs -have been successfully set). We do this as follow (where plugin_config_path is the base_config_path (which defaults +Firstly, we should check whether the plugin will be able to run (i.e., whether the configuration options it needs +have been successfully set). We do this as follows, where plugin_config_path is the base_config_path (which defaults to `plugins` and then the name of the class itself): :: @@ -166,7 +166,7 @@ to `plugins` and then the name of the class itself): unsatisfied = plugin.unsatisfied(context, plugin_config_path) If unsatisfied is an empty list, then the plugin has been given everything it requires. If not, it will be a -Dictionary of the hierarchy paths and their associated requirements that weren't satisfied. +dictionary of the hierarchy paths and their associated requirements that weren't satisfied. The plugin can then be instantiated with the context (containing the plugin's configuration) and the path that the plugin can find its configuration at. This configuration path only needs to be a unique value to identify where the From b235ed05b7f916de864916db98e7f2385a52bb07 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:50:23 +0000 Subject: [PATCH 141/348] Update the CLI manual documentation --- doc/source/vol-cli.rst | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 7b91e815d..9fb48e67a 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -58,7 +58,7 @@ Options EXTEND. Extensions must be of the form **configuration.item.name=value** -p PLUGIN_DIRS, --plugin-dirs PLUGIN_DIRS - Specified a semi-colon separated list of paths that contain directories + Specified as a semi-colon separated list of paths that contain directories where plugins may be found. These paths are searched before the default paths when loading python files for plugins. This can therefore be used to override built-in plugins. NOTE: All python code within this directory @@ -67,12 +67,12 @@ Options -s SYMBOL_DIRS, --symbol-dirs SYMBOL_DIRS SYMBOL_DIRS is a semi-colon separated list of paths that contain symbol files or symbol zip packs. Symbols must be within a particular directory - structure if they depending on the operating system of the symbols, + structure if they depend on the operating system of the symbols, whilst symbol packs must be in the root of the directory and named after - the after the operating system to which they apply. + the operating system to which they apply. -v, --verbose - A flag which can be used multiple times, each time increasing the level of + A flag which can be used multiple times (up to four), each time increasing the level of detail in the logs produced. -l LOG, --log LOG @@ -87,7 +87,7 @@ Options -q, --quiet When present, this flag mutes the progress feedback for operations. This can be beneficial when piping the output directly to a file or another - tool. This also removes the + tool. -r RENDERER, --renderer RENDERER Specifies the output format in which to display results. The default is @@ -120,9 +120,7 @@ Options Change the default path used to store the cache. --offline - Do not search online for additional JSON files. - Run offline mode (defaults to false) and for - remote windows symbol tables, linux/mac banner repositories. + Run offline mode (defaults to false). Do not search online for additional JSON files, remote windows symbol tables, nor linux/mac banner repositories. --single-location SINGLE_LOCATION This specifies a URL which will be downloaded if necessary, and built @@ -152,7 +150,7 @@ but can be overridden by creating a JSON file (`%APPDATA%/volatility3/vol.json` systems, or `~/.config/volatility3/vol.json` or `volshell.json` for all others). The format of this file is a JSON dictionary, containing the options above and their value. -It should be noted that the ordering is (`<` means is overridden by): +It should be noted that the ordering is (`x < y` means `x` is overridden by `y`): `in-built default value < config file value < command line parameter` From 04517ca79768c16cb8afc323a5625567fe87d225 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 12 Dec 2024 23:24:40 -0600 Subject: [PATCH 142/348] Windows: Handle missing _MM_SESSION_SPACE As of Windows 11 24H2, the `_MM_SESSION_SPACE` type no longer appears in the kernel PDB. Instead, the `_EPROCESS.Session` member refers to a new type, `_PSP_SESSION_SPACE`, which does not have a type definition. However, experimentation has shown that this new structure is functionally identical to the old structure - the `ProcessList` and `SessionId` members still appear to be at their old offsets. In order to account for this when analyzing these newer Windows versions, this catches the `SymbolError` and instantiates an `unsigned long` at the offset (8) where the `SessionId` member would normally be defined within an `_MM_SESSION_SPACE` structure. --- .../framework/plugins/windows/modules.py | 34 ++++++++++++++---- .../symbols/windows/extensions/__init__.py | 35 +++++++++++++------ 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index ba45834d5..5ff252074 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -165,13 +165,33 @@ class Modules(interfaces.plugins.PluginInterface): # create the session space object in the process' own layer. # not all processes have a valid session pointer. - session_space = context.object( - symbol_table + constants.BANG + "_MM_SESSION_SPACE", - layer_name=layer_name, - offset=proc.Session, - ) + try: + session_space = context.object( + symbol_table + constants.BANG + "_MM_SESSION_SPACE", + layer_name=layer_name, + offset=proc.Session, + ) + session_id = session_space.SessionId - if session_space.SessionId in seen_ids: + except exceptions.SymbolError: + # In Windows 11 24H2, the _MM_SESSION_SPACE type was + # replaced with _PSP_SESSION_SPACE, and the kernel PDB + # doesn't contain information about its members (otherwise, + # we would just fall back to the new type). However, it + # appears to be, for our purposes, functionally identical + # to the _MM_SESSION_SPACE. Because _MM_SESSION_SPACE + # stores its session ID at offset 8 as an unsigned long, we + # create an unsigned long at that offset and use that + # instead. + session_id = int( + context.object( + layer_name=layer_name, + object_type=symbol_table + constants.BANG + "unsigned long", + offset=proc.Session + 8, + ) + ) + + if session_id in seen_ids: continue except exceptions.InvalidAddressException: @@ -184,7 +204,7 @@ class Modules(interfaces.plugins.PluginInterface): continue # save the layer if we haven't seen the session yet - seen_ids.append(session_space.SessionId) + seen_ids.append(session_id) yield proc_layer_name @classmethod diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 793e506c3..78f59fc0c 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -813,23 +813,36 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): offset=kvo, native_layer_name=self.vol.native_layer_name, ) - session = ntkrnlmp.object( - object_type="_MM_SESSION_SPACE", offset=self.Session, absolute=True - ) - - if session.has_member("SessionId"): - return session.SessionId + try: + session = ntkrnlmp.object( + object_type="_MM_SESSION_SPACE", + offset=self.Session, + absolute=True, + ) + if session.has_member("SessionId"): + return session.SessionId + except exceptions.SymbolError: + # In Windows 11 24H2, the _MM_SESSION_SPACE type was + # replaced with _PSP_SESSION_SPACE, and the kernel PDB + # doesn't contain information about its members (otherwise, + # we would just fall back to the new type). However, it + # appears to be, for our purposes, functionally identical + # to the _MM_SESSION_SPACE. Because _MM_SESSION_SPACE + # stores its session ID at offset 8 as an unsigned long, we + # create an unsigned long at that offset and use that + # instead. + session_id = ntkrnlmp.object( + object_type="unsigned long", + offset=self.Session + 8, + absolute=True, + ) + return int(session_id) except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.Session.SessionId at {self.vol.offset:#x}", ) - except exceptions.SymbolError: - vollog.log( - constants.LOGLEVEL_VVV, - "Could not lookup _MM_SESSION_SPACE in symbol table", - ) return renderers.UnreadableValue() From e8b318552839e25264ebade5c0cc58d3fae27b12 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 13 Dec 2024 00:09:02 -0600 Subject: [PATCH 143/348] Windows: Handles - New pointer calculation method After researching this structure (`_HANDLE_TABLE_ENTRY`), it appears to be stable as far back as Windows 8. It's also a union, with an `ObjectPointerBits` member at the same offset as `LowValue` but within a specific bit range (bit length 44, bit position 20). Taking this value and shifting it left by four produces the correct pointer. This four-bit shift is due to 16-byte alignment of object header structures, and is what we would expect to see with 44-bit pointers in Windows. See https://www.alex-ionescu.com/behind-windows-x64s-44-bit-memory-addressing-limit/ --- .../framework/plugins/windows/handles.py | 122 +----------------- 1 file changed, 4 insertions(+), 118 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index e5cbbf4ca..977a8c804 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -3,9 +3,9 @@ # import logging -from typing import List, Optional, Dict +from typing import Dict, List, Optional -from volatility3.framework import constants, exceptions, renderers, interfaces, symbols +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints @@ -13,16 +13,6 @@ from volatility3.plugins.windows import pslist, psscan vollog = logging.getLogger(__name__) -try: - import capstone - - has_capstone = True -except ImportError: - has_capstone = False - - -DEFAULT_SAR_VALUE = 0x10 # to be used only when decoding fails - class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" @@ -32,7 +22,6 @@ class Handles(interfaces.plugins.PluginInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._sar_value = None self._type_map = None self._cookie = None self._level_mask = 7 @@ -65,21 +54,6 @@ class Handles(interfaces.plugins.PluginInterface): ), ] - def _decode_pointer(self, value, magic): - """Windows encodes pointers to objects and decodes them on the fly - before using them. - - This function mimics the decoding routine so we can generate the - proper pointer values as well. - """ - - value = value & 0xFFFFFFFFFFFFFFF8 - value = value >> magic - # if (value & (1 << 47)): - # value = value | 0xFFFF000000000000 - - return value - def _get_item(self, handle_table_entry, handle_value): """Given a handle table entry (_HANDLE_TABLE_ENTRY) structure from a process' handle table, determine where the corresponding object's @@ -103,28 +77,11 @@ class Handles(interfaces.plugins.PluginInterface): ) if is_64bit: - if handle_table_entry.LowValue == 0: + if handle_table_entry.ObjectPointerBits == 0: return None - magic = self.find_sar_value() + offset = handle_table_entry.ObjectPointerBits << 4 - # is this the right thing to raise here? - if magic is None: - if has_capstone: - raise AttributeError( - "Unable to find the SAR value for decoding handle table pointers" - ) - else: - raise exceptions.MissingModuleException( - "capstone", - "Requires capstone to find the SAR value for decoding handle table pointers", - ) - - offset = self._decode_pointer(handle_table_entry.LowValue, magic) - if not self.context.layers[virtual].is_valid(offset): - offset = self._decode_pointer( - handle_table_entry.LowValue, DEFAULT_SAR_VALUE - ) else: if handle_table_entry.InfoTable == 0: return None @@ -142,77 +99,6 @@ class Handles(interfaces.plugins.PluginInterface): object_header.HandleValue = handle_value return object_header - def find_sar_value(self): - """Locate ObpCaptureHandleInformationEx if it exists in the sample. - - Once found, parse it for the SAR value that we need to decode - pointers in the _HANDLE_TABLE_ENTRY which allows us to find the - associated _OBJECT_HEADER. - """ - - if self._sar_value is None: - if not has_capstone: - vollog.debug( - "capstone module is missing, unable to create disassembly of ObpCaptureHandleInformationEx" - ) - return None - kernel = self.context.modules[self.config["kernel"]] - - virtual_layer_name = kernel.layer_name - kvo = self.context.layers[virtual_layer_name].config[ - "kernel_virtual_offset" - ] - ntkrnlmp = self.context.module( - kernel.symbol_table_name, layer_name=virtual_layer_name, offset=kvo - ) - - try: - func_addr = ntkrnlmp.get_symbol("ObpCaptureHandleInformationEx").address - except exceptions.SymbolError: - vollog.debug("Unable to locate ObpCaptureHandleInformationEx symbol") - return None - - try: - func_addr_to_read = kvo + func_addr - num_bytes_to_read = 0x200 - vollog.debug( - f"ObpCaptureHandleInformationEx symbol located at {hex(func_addr_to_read)}" - ) - data = self.context.layers.read( - virtual_layer_name, func_addr_to_read, num_bytes_to_read - ) - except exceptions.InvalidAddressException: - vollog.warning( - f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of {hex(DEFAULT_SAR_VALUE)}" - ) - self._sar_value = DEFAULT_SAR_VALUE - return self._sar_value - - md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) - - instruction_count = 0 - for address, size, mnemonic, op_str in md.disasm_lite( - data, kvo + func_addr - ): - # print("{} {} {} {}".format(address, size, mnemonic, op_str)) - instruction_count += 1 - if mnemonic.startswith("sar"): - # if we don't want to parse op strings, we can disasm the - # single sar instruction again, but we use disasm_lite for speed - self._sar_value = int(op_str.split(",")[1].strip(), 16) - vollog.debug( - f"SAR located at {hex(address)} with value of {hex(self._sar_value)}" - ) - break - - if self._sar_value is None: - vollog.warning( - f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of {hex(DEFAULT_SAR_VALUE)}" - ) - self._sar_value = DEFAULT_SAR_VALUE - - return self._sar_value - @classmethod def get_type_map( cls, From 31492f4ab80dcc3c5ca38c2c4754ccfafc46a994 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 14 Dec 2024 15:50:50 +0000 Subject: [PATCH 144/348] Rectify maximum repetition of verbose flag From four to six (-vvvvvv). --- doc/source/vol-cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 9fb48e67a..43ca33f04 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -72,7 +72,7 @@ Options the operating system to which they apply. -v, --verbose - A flag which can be used multiple times (up to four), each time increasing the level of + A flag which can be used multiple times (up to six, -vvvvvv), each time increasing the level of detail in the logs produced. -l LOG, --log LOG From 5086be30b2c153bf168704022cff793190e1a750 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Sun, 15 Dec 2024 14:04:15 +0800 Subject: [PATCH 145/348] Refactor: move version None check to top --- volatility3/framework/configuration/requirements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 86e1aac52..f130f9544 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -529,6 +529,8 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): component: Type[interfaces.configuration.VersionableInterface] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: + if version is None: + raise TypeError("Version cannot be None") if description is None: description = f"Version {'.'.join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( @@ -537,8 +539,6 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): if component is None: raise TypeError("Component cannot be None") self._component: Type[interfaces.configuration.VersionableInterface] = component - if version is None: - raise TypeError("Version cannot be None") self._version = version def unsatisfied( From c8c39837abdf489c8316aa51ebb4f2634321d77c Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 15 Dec 2024 19:26:22 +0000 Subject: [PATCH 146/348] Tiny change text_renderer.py --- volatility3/cli/text_renderer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 31307f67e..408a562d8 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -49,12 +49,12 @@ def hex_bytes_as_text(value: bytes, width: int = 16) -> str: output += "\n" printables = "" - # Handle leftovers when the lenght is not mutiple of width + # Handle leftovers when the length is not mutiple of width if printables: padding = width - len(printables) - output += " " * (padding) + output += " " * padding output += printables - output += " " * (padding) + output += " " * padding return output @@ -132,7 +132,7 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str: for i in disasm_types[disasm.architecture].disasm( disasm.data, disasm.offset ): - output += f"\n0x{i.address:x}:\t{i.mnemonic}\t{i.op_str}" + output += f"\n{i.address:#x}:\t{i.mnemonic}\t{i.op_str}" return output return QuickTextRenderer._type_renderers[bytes](disasm.data) @@ -342,7 +342,7 @@ class PrettyTextRenderer(CLIRenderer): column_separator = " | " tree_indent_column = "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(20) + random.choices(string.ascii_uppercase + string.digits, k=20) ) max_column_widths = dict( [(column.name, len(column.name)) for column in grid.columns] From bb1ff69e426ad688ea116bcdab2f1a9c77a182e6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 16 Dec 2024 19:25:24 +1100 Subject: [PATCH 147/348] linux: dentry: Fix dentry type support for kernels pre-3.19 --- .../framework/symbols/linux/extensions/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 927f767e2..829622154 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1107,9 +1107,16 @@ class dentry(objects.StructType): walk_member = "d_sib" list_head_member = self.d_children elif self.has_member("d_child") and self.has_member("d_subdirs"): - # 2.5.0 <= kernels < 6.8 + # 3.19.0 <= kernels < 6.8 walk_member = "d_child" list_head_member = self.d_subdirs + elif self.has_member("d_u") and self.has_member("d_subdirs"): + # kernels < 3.19 + + # Actually, 'd_u.d_child' but to_list() doesn't support something like that. + # Since, it's an union, everything is at the same offset than 'd_u'. + walk_member = "d_u" + list_head_member = self.d_subdirs else: raise exceptions.VolatilityException("Unsupported dentry type") From 3b0f0915c7fd24512d12603ab989a53f6ac68928 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 16 Dec 2024 19:36:54 +1100 Subject: [PATCH 148/348] linux: page_cache: add testcase for page_cache.files plugin --- test/test_volatility.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index f7cb23e93..b5910e1c8 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -632,6 +632,26 @@ def test_linux_vmayarascan_yara_string(image, volatility, python): assert rc == 0 +def test_linux_page_cache_files(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.pagecache.Files", + image, + volatility, + python, + pluginargs=["--find", "/etc/passwd"], + ) + out = out.lower() + + assert out.count(b"\n") > 4 + + # inode_num inode_addr ... file_path + assert re.search( + rb"146829\s0x88001ab5c270.*?/etc/passwd", + out, + ) + assert rc == 0 + + # MAC From 02b11b44a28634c230b0676bf4feafe37fac6dff Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:59:58 +0000 Subject: [PATCH 149/348] Remove use of int function after math.ceil Return type of math.ceil is already an int. --- volatility3/framework/layers/intel.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index c30ae48a8..846f246dc 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -73,8 +73,8 @@ class Intel(linear.LinearlyMappedLayer): ) # These can vary depending on the type of space - self._index_shift = int( - math.ceil(math.log2(struct.calcsize(self._entry_format))) + self._index_shift = math.ceil( + math.log2(struct.calcsize(self._entry_format)) ) @classproperty @@ -125,7 +125,6 @@ class Intel(linear.LinearlyMappedLayer): high_mask = (1 << (high_bit + 1)) - 1 low_mask = (1 << low_bit) - 1 mask = high_mask ^ low_mask - # print(high_bit, low_bit, bin(mask), bin(value)) return value & mask @staticmethod @@ -147,7 +146,7 @@ class Intel(linear.LinearlyMappedLayer): return self._mask(addr, self._maxvirtaddr, 0) + self._canonical_prefix def decanonicalize(self, addr: int) -> int: - """Removes canonicalization to ensure an adress fits within the correct range if it has been canonicalized + """Removes canonicalization to ensure an address fits within the correct range if it has been canonicalized This will produce an address outside the range if the canonicalization is incorrect """ From b37923c183bec9a6381d5d592b405a9fcf51887f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 16 Dec 2024 17:08:37 +0000 Subject: [PATCH 150/348] Remove use of int function after math.ceil Return type of math.ceil is already an int. --- volatility3/framework/layers/intel.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 846f246dc..7918ebed4 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -73,9 +73,7 @@ class Intel(linear.LinearlyMappedLayer): ) # These can vary depending on the type of space - self._index_shift = math.ceil( - math.log2(struct.calcsize(self._entry_format)) - ) + self._index_shift = math.ceil(math.log2(struct.calcsize(self._entry_format))) @classproperty @functools.lru_cache() From 267c5a60c3b99da48cb0ead9c9d1492b857ea340 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 17 Dec 2024 10:05:38 +1100 Subject: [PATCH 151/348] Linux: PageCache: Remove unused variable --- volatility3/framework/plugins/linux/pagecache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 6d2607ada..46b24b27a 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -520,7 +520,6 @@ class InodePages(plugins.PluginInterface): yield 0, fields if self.config["dump"]: - filename = self.config["dump"] open_method = self.open inode_address = inode.vol.offset filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") From 7299f925dcd8a7ae1a00b47cf4846fc683c8e5ac Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 16 Dec 2024 17:58:53 -0600 Subject: [PATCH 152/348] Windows: Handles - major version bump Bumps the major version in plugin + dependences after removal of a publicly exposed instance method. --- volatility3/framework/plugins/windows/callbacks.py | 2 +- volatility3/framework/plugins/windows/dumpfiles.py | 2 +- volatility3/framework/plugins/windows/handles.py | 2 +- volatility3/framework/plugins/windows/poolscanner.py | 2 +- volatility3/framework/plugins/windows/psxview.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 562846def..414a8814a 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -48,7 +48,7 @@ class Callbacks(interfaces.plugins.PluginInterface): name="driverirp", plugin=driverirp.DriverIrp, version=(1, 0, 0) ), requirements.PluginRequirement( - name="handles", plugin=handles.Handles, version=(1, 0, 0) + name="handles", plugin=handles.Handles, version=(2, 0, 0) ), ] diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 33d2d0d41..bc554c0bf 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -69,7 +69,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): name="pslist", component=pslist.PsList, version=(2, 0, 0) ), requirements.VersionRequirement( - name="handles", component=handles.Handles, version=(1, 0, 0) + name="handles", component=handles.Handles, version=(2, 0, 0) ), ] diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 977a8c804..a3067b09f 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -18,7 +18,7 @@ class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 3) + _version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 1f70cfb8c..8c56d202d 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -139,7 +139,7 @@ class PoolScanner(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="handles", plugin=handles.Handles, version=(1, 0, 0) + name="handles", plugin=handles.Handles, version=(2, 0, 0) ), ] diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index a8d185a2c..6c845bf81 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -62,7 +62,7 @@ class PsXView(plugins.PluginInterface): name="thrdscan", component=thrdscan.ThrdScan, version=(1, 0, 0) ), requirements.VersionRequirement( - name="handles", component=handles.Handles, version=(1, 0, 0) + name="handles", component=handles.Handles, version=(2, 0, 0) ), requirements.BooleanRequirement( name="physical-offsets", From fd9d3ec04c967c5e1a16d7735cf7f064d2b82a47 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 16 Dec 2024 18:17:42 -0600 Subject: [PATCH 153/348] Windows: Typing - Remove type casts, add signature Removes the needless `int` casts, and adds the return type to the `get_session_id` method signature. --- volatility3/framework/plugins/windows/modules.py | 16 +++++++--------- .../symbols/windows/extensions/__init__.py | 4 ++-- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 5ff252074..b9d754328 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -2,14 +2,14 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import List, Iterable, Generator +from typing import Generator, Iterable, List -from volatility3.framework import exceptions, interfaces, constants, renderers +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, pedump +from volatility3.plugins.windows import pedump, pslist vollog = logging.getLogger(__name__) @@ -183,12 +183,10 @@ class Modules(interfaces.plugins.PluginInterface): # stores its session ID at offset 8 as an unsigned long, we # create an unsigned long at that offset and use that # instead. - session_id = int( - context.object( - layer_name=layer_name, - object_type=symbol_table + constants.BANG + "unsigned long", - offset=proc.Session + 8, - ) + session_id = context.object( + layer_name=layer_name, + object_type=symbol_table + constants.BANG + "unsigned long", + offset=proc.Session + 8, ) if session_id in seen_ids: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 78f59fc0c..12f84ca90 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -797,7 +797,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return renderers.UnreadableValue() - def get_session_id(self): + def get_session_id(self) -> Union[int, interfaces.renderers.BaseAbsentValue]: try: if self.has_member("Session"): if self.Session == 0: @@ -836,7 +836,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): offset=self.Session + 8, absolute=True, ) - return int(session_id) + return session_id except exceptions.InvalidAddressException: vollog.log( From 02f17af8a632fb70d1152c66ff3847b2fb921033 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 17 Dec 2024 12:09:29 +1100 Subject: [PATCH 154/348] linux: fix task parent pid in several plugins. It also adds a method to get the correct one in a unified way from the task object extension --- .../framework/plugins/linux/capabilities.py | 4 ++-- volatility3/framework/plugins/linux/envars.py | 12 ++---------- volatility3/framework/plugins/linux/psaux.py | 10 ++-------- volatility3/framework/plugins/linux/pslist.py | 4 ++-- volatility3/framework/plugins/linux/psscan.py | 7 ++----- volatility3/framework/plugins/linux/pstree.py | 6 +++--- .../framework/symbols/linux/extensions/__init__.py | 14 ++++++++++++++ 7 files changed, 27 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index a8a8fb1fa..a06ee4c1b 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -50,7 +50,7 @@ class Capabilities(plugins.PluginInterface): """Lists process capabilities""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -136,7 +136,7 @@ class Capabilities(plugins.PluginInterface): comm=utility.array_to_string(task.comm), pid=int(task.pid), tgid=int(task.tgid), - ppid=int(task.parent.pid), + ppid=int(task.get_parent_pid()), euid=int(task.cred.euid), ) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 22aba6408..a3eb21cf5 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -17,7 +17,7 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls): @@ -48,15 +48,7 @@ class Envars(plugins.PluginInterface): # get process name as string name = utility.array_to_string(task.comm) - - # try and get task parent - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." - ) - ppid = 0 + ppid = task.get_parent_pid() # kernel threads never have an mm as they do not have userland mappings try: diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 5467c3b4c..5a4d75c70 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -15,7 +15,7 @@ class PsAux(plugins.PluginInterface): """Lists processes with their command line arguments""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls): @@ -98,14 +98,8 @@ class PsAux(plugins.PluginInterface): # walk the process list and report the arguments for task in tasks: pid = task.pid - - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - ppid = 0 - + ppid = task.get_parent_pid() name = utility.array_to_string(task.comm) - args = self._get_command_line_args(task, name) yield (0, (pid, ppid, name, args)) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 6460462a7..edfc0688c 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -18,7 +18,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) + _version = (3, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -95,7 +95,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ pid = task.tgid tid = task.pid - ppid = task.parent.tgid if task.parent else 0 + ppid = task.get_parent_pid() name = utility.array_to_string(task.comm) start_time = task.get_create_time() if decorate_comm: diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 40784a647..55e3778ab 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -28,7 +28,7 @@ class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -52,10 +52,7 @@ class PsScan(interfaces.plugins.PluginInterface): """ pid = task.tgid tid = task.pid - ppid = 0 - - if task.parent.is_readable(): - ppid = task.parent.tgid + ppid = task.get_parent_pid() name = utility.array_to_string(task.comm) exit_state = DescExitStateEnum(task.exit_state).name diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 9dc5ea3cc..7ea9df3d6 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -13,7 +13,7 @@ class PsTree(interfaces.plugins.PluginInterface): ID.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls): @@ -56,9 +56,9 @@ class PsTree(interfaces.plugins.PluginInterface): seen = set([pid]) level = 0 proc = self._tasks.get(pid) - while proc and proc.parent and proc.parent.pid not in seen: + while proc and proc.get_parent_pid() not in seen: if proc.is_thread_group_leader: - parent_pid = proc.parent.pid + parent_pid = proc.get_parent_pid() else: parent_pid = proc.tgid diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 927f767e2..f27306e67 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -635,6 +635,20 @@ class task_struct(generic.GenericIntelProcess): # root time namespace, not within the task's own time namespace return boottime + task_start_time_timedelta + def get_parent_pid(self) -> int: + """Returns the parent process ID (PPID) + + This method replicates the Linux kernel's `getppid` syscall behavior. + Avoid using `task.parent`; instead, use this function for accurate results. + """ + + if self.real_parent and self.real_parent.is_readable(): + ppid = self.real_parent.pid + else: + ppid = 0 + + return ppid + class fs_struct(objects.StructType): def get_root_dentry(self): From 74ff42a12d2665d05e64c7c71beb2bec5f4c9333 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 17 Dec 2024 13:28:57 +1100 Subject: [PATCH 155/348] Fix ProducerMetadata class bug introduced in #1369 --- volatility3/framework/symbols/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 73ad2cf21..7e069e518 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -27,7 +27,7 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): @property def version(self) -> Optional[Tuple[int]]: """Returns the version of the ISF file producer""" - version = self.version_string() + version = self.version_string if not version: return None if all(x in "0123456789." for x in version): From 0255151ef508f5c4a1e75c71dfafa99218e15b05 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Tue, 17 Dec 2024 12:15:46 +0800 Subject: [PATCH 156/348] Fix: Error early if no inodes are found in linux.pagecache.InodePages plugin --- volatility3/framework/plugins/linux/pagecache.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 46b24b27a..7dbf074b3 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -483,6 +483,9 @@ class InodePages(plugins.PluginInterface): if inode_in.path == self.config["find"]: inode = inode_in.inode break # Only the first match + else: + vollog.error("Unable to find inode with path %s", self.config["find"]) + return None elif self.config["inode"]: inode = vmlinux.object("inode", self.config["inode"], absolute=True) From 054f0496c123ec074a134ab9e75416fdef15e1c4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 17:55:52 +0000 Subject: [PATCH 157/348] Windows: Cannot use capstone typing information if capstone didn'tr import --- volatility3/framework/plugins/windows/indirect_system_calls.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index 1a5eb317f..c4f3f6d28 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -73,8 +73,7 @@ class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): @staticmethod def _indirect_syscall_block_target( - proc_layer: interfaces.layers.DataLayerInterface, inst: capstone._cs_insn - ) -> Optional[int]: + proc_layer: interfaces.layers.DataLayerInterface, inst) -> Optional[int]: """ This function determines the address of a jmp in the following form: From 246d19c0fadbb986e4ec4c019505bed4d32b6359 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:14:29 +0000 Subject: [PATCH 158/348] Windows: Fix black issue --- volatility3/framework/plugins/windows/indirect_system_calls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index c4f3f6d28..f09851b30 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -73,7 +73,8 @@ class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): @staticmethod def _indirect_syscall_block_target( - proc_layer: interfaces.layers.DataLayerInterface, inst) -> Optional[int]: + proc_layer: interfaces.layers.DataLayerInterface, inst + ) -> Optional[int]: """ This function determines the address of a jmp in the following form: From c45beb3ebe7feaec42567eb8ef3dd665e15db3ae Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:12:19 +0000 Subject: [PATCH 159/348] Automagic: Fixes #1417 --- volatility3/framework/automagic/symbol_cache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index e38771f79..2c9883c7d 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -299,6 +299,13 @@ class SqliteCache(CacheManagerInterface): This also updates remote locations based on a cache timeout. """ + if progress_callback is None: + + def dummy_progress(*args, **kargs) -> None: + return None + + progress_callback = dummy_progress + on_disk_locations = set( [ filename From 9ef90c2091a5b838817fc9251251b74f34dccf7e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 17 Dec 2024 13:14:33 -0600 Subject: [PATCH 160/348] Windows: PeDump - use contextmanager Several tools, including pyright and PyCharm, report that `file_handle` may be an unbound local. Regardless of whether or not this is likely to happen in practice, it makes sense to just use a `ContextManager` here anyway, since `FileHandlerInterface` implements it. --- .../framework/plugins/windows/pedump.py | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 858d0615a..5b4bb07d7 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -64,30 +64,27 @@ class PEDump(interfaces.plugins.PluginInterface): """ Returns the filename of the dump file or None """ - try: - file_handle = open_method(file_name) + with open_method(file_name) as file_handle: + try: + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=base, - layer_name=layer_name, - ) + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + IOError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - IOError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") - return None - finally: - file_handle.close() - - return file_handle.preferred_filename + return file_handle.preferred_filename @classmethod def dump_ldr_entry( From 24e1904376aa501e5ec4b240f2cbe6fb23267ebe Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:22:35 +0000 Subject: [PATCH 161/348] Modify using as a library documentation Tiny changes. --- doc/source/using-as-a-library.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index 55b77e90a..144cae644 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -21,7 +21,7 @@ Creating a context First we make sure the volatility framework works the way we expect it (and is the version we expect). The versioning used is semantic versioning, meaning any version with the same major number and a higher or equal minor number will satisfy the requirement. An example is below since the CLI doesn't need any of the features -from version 1.1: +from version 1.1 or later: :: @@ -139,7 +139,7 @@ A suitable list of automagics for a particular plugin (based on operating system This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just the automagics which apply to the operating system. Each automagic can exclude itself from being used for specific -operating systems, so that for example an automagic designed for linux is not used for windows or mac plugins. +operating systems, such that an automagic designed for linux is not used for windows or mac plugins. These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that the plugin will be constructed on ('plugins' by default) and a progress_callback. This is a callable which takes @@ -166,7 +166,7 @@ to `plugins` and then the name of the class itself): unsatisfied = plugin.unsatisfied(context, plugin_config_path) If unsatisfied is an empty list, then the plugin has been given everything it requires. If not, it will be a -dictionary of the hierarchy paths and their associated requirements that weren't satisfied. +dict of the hierarchy paths and their associated requirements that weren't satisfied. The plugin can then be instantiated with the context (containing the plugin's configuration) and the path that the plugin can find its configuration at. This configuration path only needs to be a unique value to identify where the From b04ca88d0754bfc2e79fbe70c9280eba344d8375 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 17 Dec 2024 13:28:37 -0600 Subject: [PATCH 162/348] Windows Extensions: Fixes type-hint on list_timers This method is incorrectly type-hinted as returning a `Tuple` when it should be returning an instance of the `KTIMER` extension class. This leaves the version number as is since it only updates the type-hint, but let me know if that's incorrect and we need to bump it. --- volatility3/framework/plugins/windows/timers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index d49c28784..fad25df72 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -14,7 +14,7 @@ from volatility3.framework import ( ) from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols.windows import versions +from volatility3.framework.symbols.windows import versions, extensions from volatility3.plugins.windows import ssdt, kpcrs vollog = logging.getLogger(__name__) @@ -49,7 +49,7 @@ class Timers(interfaces.plugins.PluginInterface): kernel_module_name: str, layer_name: str, symbol_table: str, - ) -> Iterable[Tuple[str, int, str]]: + ) -> Iterable[extensions.KTIMER]: """Lists all kernel timers. Args: From c134dcd64306e3c5f355f97d4e117d34c3e76f33 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 17 Dec 2024 13:55:34 -0600 Subject: [PATCH 163/348] Windows Timers: Bump patch version --- volatility3/framework/plugins/windows/timers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index fad25df72..8bd7c8eb4 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -24,7 +24,7 @@ class Timers(interfaces.plugins.PluginInterface): """Print kernel timers and associated module DPCs""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From d76bf96f36a60a4c17aa51fa9033b80836b10812 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:25:10 +0000 Subject: [PATCH 164/348] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/framework/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 23ea745de..61ad787a1 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -64,8 +64,8 @@ def require_interface_version(*args) -> None: if args[1] > interface_version()[1]: raise RuntimeError( "Framework interface version {} is an older revision than the required version {}".format( - ".".join([str(x) for x in interface_version()[0:2]]), - ".".join([str(x) for x in args[0:2]]), + ".".join(str(x) for x in interface_version()[0:2]), + ".".join(str(x) for x in args[0:2]), ) ) From 60ca99710728fc923420b8e7c64b8b8dfa60e1a2 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:40:54 +0000 Subject: [PATCH 165/348] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 55ef19e4b..8c4856f0b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -5,7 +5,7 @@ VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( - ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + ".".join(str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]) + VERSION_SUFFIX ) """The canonical version of the volatility3 package""" From c8e2347a0f1577f349cbc212821e7b3122cc60e1 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:47:36 +0000 Subject: [PATCH 166/348] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/framework/plugins/windows/getservicesids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index 9b20ed2d0..12fc44682 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -26,7 +26,7 @@ def createservicesid(svc) -> str: ## The use of struct here is OK. It doesn't make much sense ## to leverage obj.Object inside this loop. dec.append(struct.unpack(" Date: Tue, 17 Dec 2024 20:48:42 +0000 Subject: [PATCH 167/348] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/framework/plugins/linux/check_creds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 0857576d5..4916b67d2 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -55,7 +55,7 @@ class Check_creds(interfaces.plugins.PluginInterface): for cred_addr, pids in creds.items(): if len(pids) > 1: - pid_str = ", ".join([str(pid) for pid in pids]) + pid_str = ", ".join(str(pid) for pid in pids) fields = [ format_hints.Hex(cred_addr), From eeb7cf71317bb5a195172d756d44ec11916b84a9 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:50:38 +0000 Subject: [PATCH 168/348] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/cli/volshell/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 82c470e1a..d02d054ee 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -203,7 +203,7 @@ class Volshell(interfaces.plugins.PluginInterface): connector = " " if chunk_size < 2: connector = "" - ascii_data = connector.join([self._ascii_bytes(x) for x in valid_data]) + ascii_data = connector.join(self._ascii_bytes(x) for x in valid_data) print(hex(offset), " ", hex_data, " ", ascii_data) offset += 16 From ae1079f923a4b5f36aa394805c26805e0b31408e Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:52:29 +0000 Subject: [PATCH 169/348] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index f130f9544..a52d7ff27 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -532,7 +532,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): if version is None: raise TypeError("Version cannot be None") if description is None: - description = f"Version {'.'.join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" + description = f"Version {'.'.join(str(x) for x in version)} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) From 8e9b719a1641a8c72fcbae61c313061fb60864f0 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 13:23:11 +0100 Subject: [PATCH 170/348] use ruff for linting and enforce linting via ci --- .github/workflows/ruff.yaml | 15 +++++++++++++++ pyproject.toml | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 .github/workflows/ruff.yaml diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml new file mode 100644 index 000000000..77e3aa864 --- /dev/null +++ b/.github/workflows/ruff.yaml @@ -0,0 +1,15 @@ +--- +name: Ruff + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/ruff-action@v1 + with: + args: check + src: "." diff --git a/pyproject.toml b/pyproject.toml index 9b4b8d485..7035f7a15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,24 @@ show_traceback = true [tool.mypy.overrides] ignore_missing_imports = true +[tool.ruff] +line-length = 88 +target-version = "py38" + +[tool.ruff.lint] +select = [ + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade +] + +ignore = [ + "E501", # ignore due to conflict with formatter +] + [build-system] requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" From ad7daafc508da70e140ccb4babb3c53b9a2085d7 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 16:45:18 +0100 Subject: [PATCH 171/348] linted with ruff (`ruff check . --fix`) --- development/compare-vol.py | 2 +- development/pdbparse-to-json.py | 5 +- development/schema_validate.py | 4 +- volatility3/cli/__init__.py | 22 +++----- volatility3/cli/text_filter.py | 2 +- volatility3/cli/text_renderer.py | 2 +- volatility3/cli/volargparse.py | 2 +- volatility3/cli/volshell/__init__.py | 6 +-- volatility3/cli/volshell/generic.py | 1 - volatility3/framework/__init__.py | 10 ++-- volatility3/framework/automagic/linux.py | 4 +- volatility3/framework/automagic/mac.py | 2 +- volatility3/framework/automagic/pdbscan.py | 4 +- .../framework/automagic/symbol_cache.py | 9 +--- .../framework/automagic/symbol_finder.py | 2 +- .../framework/configuration/requirements.py | 4 +- volatility3/framework/contexts/__init__.py | 2 +- .../framework/interfaces/configuration.py | 10 ++-- volatility3/framework/interfaces/layers.py | 7 +-- volatility3/framework/interfaces/plugins.py | 2 +- volatility3/framework/interfaces/renderers.py | 12 +++-- volatility3/framework/interfaces/symbols.py | 3 +- volatility3/framework/layers/crash.py | 9 ++-- volatility3/framework/layers/intel.py | 19 +++---- volatility3/framework/layers/leechcore.py | 2 +- volatility3/framework/layers/qemu.py | 2 +- volatility3/framework/layers/registry.py | 8 +-- volatility3/framework/layers/resources.py | 2 +- .../framework/layers/scanners/multiregexp.py | 2 +- volatility3/framework/objects/utility.py | 4 +- volatility3/framework/plugins/layerwriter.py | 2 +- .../framework/plugins/linux/check_idt.py | 2 +- .../framework/plugins/linux/check_syscall.py | 2 +- .../framework/plugins/linux/pagecache.py | 2 +- volatility3/framework/plugins/linux/proc.py | 4 +- volatility3/framework/plugins/linux/psscan.py | 2 +- .../framework/plugins/mac/proc_maps.py | 4 +- volatility3/framework/plugins/timeliner.py | 4 +- .../framework/plugins/windows/cmdline.py | 4 +- .../framework/plugins/windows/consoles.py | 22 ++------ .../framework/plugins/windows/dlllist.py | 15 ++---- .../framework/plugins/windows/dumpfiles.py | 8 +-- .../framework/plugins/windows/envars.py | 6 +-- .../plugins/windows/getservicesids.py | 2 +- .../framework/plugins/windows/getsids.py | 6 +-- .../plugins/windows/hollowprocesses.py | 37 +++++-------- volatility3/framework/plugins/windows/iat.py | 8 +-- .../framework/plugins/windows/malfind.py | 8 +-- .../framework/plugins/windows/memmap.py | 10 +--- .../framework/plugins/windows/modules.py | 4 +- .../framework/plugins/windows/netscan.py | 31 ++--------- .../framework/plugins/windows/netstat.py | 8 +-- .../framework/plugins/windows/pe_symbols.py | 2 +- .../framework/plugins/windows/pedump.py | 54 +++++++++---------- .../framework/plugins/windows/privileges.py | 2 +- .../framework/plugins/windows/psxview.py | 1 - .../plugins/windows/registry/hivelist.py | 26 +++------ .../plugins/windows/registry/userassist.py | 6 +-- .../plugins/windows/skeleton_key_check.py | 8 +-- .../framework/plugins/windows/strings.py | 4 +- .../framework/plugins/windows/svclist.py | 4 +- .../framework/plugins/windows/svcscan.py | 14 ++--- .../framework/plugins/windows/thrdscan.py | 2 +- .../framework/plugins/windows/threads.py | 2 +- .../framework/plugins/windows/timers.py | 2 +- .../plugins/windows/unhooked_system_calls.py | 2 +- .../framework/plugins/windows/vadinfo.py | 4 +- .../framework/plugins/windows/verinfo.py | 4 +- volatility3/framework/renderers/__init__.py | 8 +-- volatility3/framework/symbols/intermed.py | 2 +- .../framework/symbols/linux/__init__.py | 2 +- .../symbols/linux/extensions/__init__.py | 4 +- .../symbols/windows/extensions/__init__.py | 8 +-- .../symbols/windows/extensions/consoles.py | 2 +- .../symbols/windows/extensions/mbr.py | 7 +-- .../symbols/windows/extensions/network.py | 8 ++- .../symbols/windows/extensions/pe.py | 4 +- .../symbols/windows/extensions/pool.py | 6 +-- .../symbols/windows/extensions/registry.py | 4 +- .../framework/symbols/windows/pdbconv.py | 6 +-- .../framework/symbols/windows/pdbutil.py | 6 +-- .../plugins/windows/registry/certificates.py | 2 +- volatility3/plugins/windows/statistics.py | 4 +- volatility3/schemas/__init__.py | 6 +-- 84 files changed, 191 insertions(+), 381 deletions(-) diff --git a/development/compare-vol.py b/development/compare-vol.py index d0d834038..1074c5d9c 100644 --- a/development/compare-vol.py +++ b/development/compare-vol.py @@ -101,7 +101,7 @@ class Volatility2Test(VolatilityTest): print(f" Tested volatility2 imageinfo with image {image.filepath}: {end_time - start_time}") with open(os.path.join(self.output_directory, f'vol2_imageinfo_{image_hash}_stdout'), "wb") as f: f.write(vol2_completed.stdout) - image.vol2_profile = re.search(b"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout)[1] + image.vol2_profile = re.search(rb"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout)[1] class RekallTest(VolatilityTest): diff --git a/development/pdbparse-to-json.py b/development/pdbparse-to-json.py index 819e44e15..6eb265227 100644 --- a/development/pdbparse-to-json.py +++ b/development/pdbparse-to-json.py @@ -145,8 +145,7 @@ class PDBConvertor: """Generates the metadata necessary for this object""" dbg = self._pdb.STREAM_DBI last_bytes = str(binascii.hexlify(self._pdb.STREAM_PDB.GUID.Data4), 'ascii')[-16:] - guidstr = u'{:08x}{:04x}{:04x}{}'.format(self._pdb.STREAM_PDB.GUID.Data1, self._pdb.STREAM_PDB.GUID.Data2, - self._pdb.STREAM_PDB.GUID.Data3, last_bytes) + guidstr = f'{self._pdb.STREAM_PDB.GUID.Data1:08x}{self._pdb.STREAM_PDB.GUID.Data2:04x}{self._pdb.STREAM_PDB.GUID.Data3:04x}{last_bytes}' pdb_data = { "GUID": guidstr.upper(), "age": self._pdb.STREAM_PDB.Age, @@ -195,7 +194,7 @@ class PDBConvertor: try: sects = self._pdb.STREAM_SECT_HDR_ORIG.sections omap = self._pdb.STREAM_OMAP_FROM_SRC - except AttributeError as e: + except AttributeError: # In this case there is no OMAP, so we use the given section # headers and use the identity function for omap.remap sects = self._pdb.STREAM_SECT_HDR.sections diff --git a/development/schema_validate.py b/development/schema_validate.py index 0908e934f..031039e38 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -28,7 +28,7 @@ if __name__ == '__main__': schema = None if args.schema: - with open(os.path.abspath(args.schema), 'r') as s: + with open(os.path.abspath(args.schema)) as s: schema = json.load(s) failures = [] @@ -36,7 +36,7 @@ if __name__ == '__main__': try: if os.path.exists(filename): print(f"[?] Validating file: {filename}") - with open(filename, 'r') as t: + with open(filename) as t: test = json.load(t) if args.schema: diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 901f299a8..cf1335443 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -57,7 +57,7 @@ formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) -class PrintedProgress(object): +class PrintedProgress: """A progress handler that prints the progress value and the description onto the command line.""" @@ -126,9 +126,7 @@ class CommandLine: "--help", action="help", default=argparse.SUPPRESS, - help="Show this help message and exit, for specific plugin options use '{} --help'".format( - parser.prog - ), + help=f"Show this help message and exit, for specific plugin options use '{parser.prog} --help'", ) parser.add_argument( "-c", @@ -360,9 +358,7 @@ class CommandLine: subparser = parser.add_subparsers( title="Plugins", dest="plugin", - description="For plugin specific options, run '{} --help'".format( - self.CLI_NAME - ), + description=f"For plugin specific options, run '{self.CLI_NAME} --help'", action=volargparse.HelpfulSubparserAction, metavar="PLUGIN", ) @@ -416,7 +412,7 @@ class CommandLine: # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: - with open(args.config, "r") as f: + with open(args.config) as f: json_val = json.load(f) ctx.config.splice( plugin_config_path, @@ -722,9 +718,7 @@ class CommandLine: if isinstance(requirement, requirements.ListRequirement): if not isinstance(value, list): raise TypeError( - "Configuration for ListRequirement was not a list: {}".format( - requirement.name - ) + f"Configuration for ListRequirement was not a list: {requirement.name}" ) value = [requirement.element_type(x) for x in value] if not inspect.isclass(configurables_list[configurable]): @@ -797,7 +791,7 @@ class CommandLine: fd, self._name = tempfile.mkstemp( suffix=".vol3", prefix="tmp_", dir=output_dir ) - self._file = io.open(fd, mode="w+b") + self._file = open(fd, mode="w+b") CLIFileHandler.__init__(self, filename) for item in dir(self._file): if not item.startswith("_") and item not in ( @@ -870,9 +864,7 @@ class CommandLine: requirement, interfaces.configuration.RequirementInterface ): raise TypeError( - "Plugin contains requirements that are not RequirementInterfaces: {}".format( - configurable.__name__ - ) + f"Plugin contains requirements that are not RequirementInterfaces: {configurable.__name__}" ) if isinstance(requirement, interfaces.configuration.SimpleTypeRequirement): additional["type"] = requirement.instance_type diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 3d69934e9..955d647f5 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -76,7 +76,7 @@ class ColumnFilter: if self.regex: return re.search(self.pattern, f"{item}") return self.pattern in f"{item}" - except IOError: + except OSError: return False def found(self, row: List[Any]) -> bool: diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 408a562d8..937ba4ef4 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -467,7 +467,7 @@ class JsonRenderer(CLIRenderer): def output_result(self, outfd, result): """Outputs the JSON data to a file in a particular format""" - outfd.write("{}\n".format(json.dumps(result, indent=2, sort_keys=True))) + outfd.write(f"{json.dumps(result, indent=2, sort_keys=True)}\n") def render(self, grid: interfaces.renderers.TreeGrid): outfd = sys.stdout diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index fd61ddce0..dce9cafa6 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -5,7 +5,7 @@ import argparse import gettext import re -from typing import List, Optional, Sequence, Any, Union +from typing import Optional, Sequence, Any, Union # This effectively overrides/monkeypatches the core argparse module to provide more helpful output around choices diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index e9d3fda08..0affe5d59 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -282,9 +282,7 @@ class VolShell(cli.CommandLine): for plugin in volshell_plugin_list: subparser = parser.add_argument_group( title=plugin.capitalize(), - description="Configuration options based on {} options".format( - plugin.capitalize() - ), + description=f"Configuration options based on {plugin.capitalize()} options", ) self.populate_requirements_argparse(subparser, volshell_plugin_list[plugin]) configurables_list[plugin] = volshell_plugin_list[plugin] @@ -331,7 +329,7 @@ class VolShell(cli.CommandLine): # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: - with open(args.config, "r") as f: + with open(args.config) as f: json_val = json.load(f) ctx.config.splice( plugin_config_path, diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 82c470e1a..65040bb2a 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -585,7 +585,6 @@ class NullFileHandler(io.BytesIO, interfaces.plugins.FileHandlerInterface): def writelines(self, lines: Iterable[bytes]): """Dummy method""" - pass def write(self, b: bytes): """Dummy method""" diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 23ea745de..e0f7c778a 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -56,9 +56,7 @@ def require_interface_version(*args) -> None: if len(args): if args[0] != interface_version()[0]: raise RuntimeError( - "Framework interface version {} is incompatible with required version {}".format( - interface_version()[0], args[0] - ) + f"Framework interface version {interface_version()[0]} is incompatible with required version {args[0]}" ) if len(args) > 1: if args[1] > interface_version()[1]: @@ -70,7 +68,7 @@ def require_interface_version(*args) -> None: ) -class NonInheritable(object): +class NonInheritable: def __init__(self, value: Any, cls: Type) -> None: self.default_value = value self.cls = cls @@ -187,9 +185,7 @@ def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str traceback.TracebackException.from_exception(e).format(chain=True) ) ) - vollog.debug( - "Failed to import module {} based on file: {}".format(module, path) - ) + vollog.debug(f"Failed to import module {module} based on file: {path}") failures.append(module) if not ignore_errors: raise diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index ef54a0aa5..6b58577a3 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -173,9 +173,7 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if aslr_shift & 0xFFF != 0 or kaslr_shift & 0xFFF != 0: continue vollog.debug( - "Linux ASLR shift values determined: physical {:0x} virtual {:0x}".format( - kaslr_shift, aslr_shift - ) + f"Linux ASLR shift values determined: physical {kaslr_shift:0x} virtual {aslr_shift:0x}" ) return kaslr_shift, aslr_shift diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 7c478b521..89dd5a187 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -184,7 +184,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = [int(x) for x in banner[22:].split(b".")[0:2]] + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) tmp_aslr_shift = offset - cls.virtual_to_physical_address( version_json_address diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 0b4f6c73a..729c48063 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -215,9 +215,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): return (virtual_layer_name, kvo, kernel) else: vollog.debug( - "Potential kernel_virtual_offset did not map to expected location: {}".format( - hex(kvo) - ) + f"Potential kernel_virtual_offset did not map to expected location: {hex(kvo)}" ) except exceptions.InvalidAddressException: vollog.debug( diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index e38771f79..9fad506ae 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -106,7 +106,6 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): def add_identifier(self, location: str, operating_system: str, identifier: str): """Adds an identifier to the store""" - pass def find_location( self, identifier: bytes, operating_system: Optional[str] @@ -120,18 +119,15 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: The location of the symbols file that matches the identifier """ - pass def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" - pass def update(self): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. """ - pass def get_identifier_dictionary( self, operating_system: Optional[str] = None, local_only: bool = False @@ -145,15 +141,12 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A dictionary of identifiers mapped to a location """ - pass def get_identifier(self, location: str) -> Optional[bytes]: """Returns an identifier based on a specific location or None""" - pass def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" - pass def get_location_statistics( self, location: str @@ -572,6 +565,6 @@ class RemoteIdentifierFormat: try: subrbf = RemoteIdentifierFormat(location) yield from subrbf.process(identifiers, operating_system) - except IOError: + except OSError: vollog.debug(f"Remote file not found: {location}") return identifiers diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 6d689e194..1d30f3f51 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -4,7 +4,7 @@ import logging import os -from typing import Any, Callable, Iterable, List, Optional, Tuple +from typing import Callable, List, Optional, Tuple from volatility3.framework import constants, interfaces, layers from volatility3.framework.automagic import symbol_cache diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index f130f9544..828c89daa 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -664,9 +664,7 @@ class ModuleRequirement( if value is not None: vollog.log( constants.LOGLEVEL_V, - "TypeError - Module Requirement only accepts string labels: {}".format( - repr(value) - ), + f"TypeError - Module Requirement only accepts string labels: {repr(value)}", ) return {config_path: self} diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 6961d9328..5111b168a 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -356,7 +356,7 @@ class SizedModule(Module): return size or 0 @property # type: ignore # FIXME: mypy #5107 - @functools.lru_cache() + @functools.lru_cache def hash(self) -> str: """Hashes the module for equality checks. diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index da0a4556c..cbbf7e342 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -94,7 +94,7 @@ class HierarchicalDict(collections.abc.Mapping): raise TypeError(f"Separator must be a one character string: {separator}") self._separator = separator self._data: Dict[str, ConfigSimpleType] = {} - self._subdict: Dict[str, "HierarchicalDict"] = {} + self._subdict: Dict[str, HierarchicalDict] = {} if isinstance(initial_dict, str): initial_dict = json.loads(initial_dict) if isinstance(initial_dict, dict): @@ -182,9 +182,7 @@ class HierarchicalDict(collections.abc.Mapping): else: if not isinstance(value, HierarchicalDict): raise TypeError( - "HierarchicalDicts can only store HierarchicalDicts within their structure: {}".format( - type(value) - ) + f"HierarchicalDicts can only store HierarchicalDicts within their structure: {type(value)}" ) self._subdict[key] = value @@ -498,9 +496,7 @@ class SimpleTypeRequirement(RequirementInterface): if not isinstance(value, self.instance_type): vollog.log( constants.LOGLEVEL_V, - "TypeError - {} requirements only accept {} type: {}".format( - self.name, self.instance_type.__name__, repr(value) - ), + f"TypeError - {self.name} requirements only accept {self.instance_type.__name__} type: {repr(value)}", ) return {config_path: self} return {} diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 78687d8d5..56798aca9 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -188,7 +188,6 @@ class DataLayerInterface( the object unreadable (exceptions will be thrown using a DataLayer after destruction) """ - pass @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -361,9 +360,7 @@ class DataLayerInterface( data += self.context.layers[layer_name].read(address, chunk_size) except exceptions.InvalidAddressException: vollog.debug( - "Invalid address in layer {} found scanning {} at address {:x}".format( - layer_name, self.name, address - ) + f"Invalid address in layer {layer_name} found scanning {self.name} at address {address:x}" ) if len(data) > scanner.chunk_size + scanner.overlap: @@ -721,7 +718,7 @@ class LayerContainer(collections.abc.Mapping): raise NotImplementedError("Cycle checking has not yet been implemented") -class DummyProgress(object): +class DummyProgress: """A class to emulate Multiprocessing/threading Value objects.""" def __init__(self) -> None: diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 74902636e..f763815a6 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -46,7 +46,7 @@ class FileHandlerInterface(io.RawIOBase): def preferred_filename(self, filename: str): """Sets the preferred filename""" if self.closed: - raise IOError("FileHandler name cannot be changed once closed") + raise OSError("FileHandler name cannot be changed once closed") if not isinstance(filename, str): raise TypeError("FileHandler preferred filenames must be strings") if os.path.sep in filename: diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index b13de1834..7105274c0 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -26,7 +26,11 @@ from typing import ( Union, ) -Column = NamedTuple("Column", [("name", str), ("type", Any)]) + +class Column(NamedTuple): + name: str + type: Any + RenderOption = Any @@ -98,11 +102,11 @@ class TreeNode(abc.Sequence, metaclass=ABCMeta): """ -class BaseAbsentValue(object): +class BaseAbsentValue: """Class that represents values which are not present for some reason.""" -class Disassembly(object): +class Disassembly: """A class to indicate that the bytes provided should be disassembled (based on the architecture)""" @@ -137,7 +141,7 @@ ColumnsType = List[Tuple[str, BaseTypes]] VisitorSignature = Callable[[TreeNode, _Type], _Type] -class TreeGrid(object, metaclass=ABCMeta): +class TreeGrid(metaclass=ABCMeta): """Class providing the interface for a TreeGrid (which contains TreeNodes) The structure of a TreeGrid is designed to maintain the structure of the tree in a single object. diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index b645f5cd1..ead91fb4d 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -250,7 +250,6 @@ class BaseSymbolTableInterface: def clear_symbol_cache(self) -> None: """Clears the symbol cache of this symbol table.""" - pass class SymbolSpaceInterface(collections.abc.Mapping): @@ -378,7 +377,7 @@ class NativeTableInterface(BaseSymbolTableInterface): return [] -class MetadataInterface(object): +class MetadataInterface: """Interface for accessing metadata stored within a symbol table.""" def __init__(self, json_data: Dict) -> None: diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 3cfc0a25b..a5b25d178 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -1,7 +1,6 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import contextlib import logging import struct from typing import Tuple, Optional @@ -138,7 +137,7 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): ulong_bitmap_array = summary_header.get_buffer_long() # outer_index points to a 32 bits array inside a list of arrays, # each bit indicating a page mapping state - for outer_index in range(0, ulong_bitmap_array.vol.count): + for outer_index in range(ulong_bitmap_array.vol.count): ulong_bitmap = ulong_bitmap_array[outer_index] # All pages in this 32 bits array are mapped (speedup iteration process) if ulong_bitmap == 0xFFFFFFFF: @@ -166,7 +165,7 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): seg_first_bit = None # Some pages in this 32 bits array are mapped and some aren't else: - for inner_bit_position in range(0, 32): + for inner_bit_position in range(32): current_bit = outer_index * 32 + inner_bit_position page_mapped = ulong_bitmap & (1 << inner_bit_position) if page_mapped: @@ -220,9 +219,7 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): for idx, (start_position, mapped_offset, length, _) in enumerate(segments): vollog.log( constants.LOGLEVEL_VVVV, - "Segment {}: Position {:#x} Offset {:#x} Length {:#x}".format( - idx, start_position, mapped_offset, length - ), + f"Segment {idx}: Position {start_position:#x} Offset {mapped_offset:#x} Length {length:#x}", ) self._segments = segments diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 7918ebed4..7c2c72ac1 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -76,13 +76,13 @@ class Intel(linear.LinearlyMappedLayer): self._index_shift = math.ceil(math.log2(struct.calcsize(self._entry_format))) @classproperty - @functools.lru_cache() + @functools.lru_cache def page_shift(cls) -> int: """Page shift for the intel memory layers.""" return cls._page_size_in_bits @classproperty - @functools.lru_cache() + @functools.lru_cache def page_size(cls) -> int: """Page size for the intel memory layers. @@ -91,25 +91,25 @@ class Intel(linear.LinearlyMappedLayer): return 1 << cls._page_size_in_bits @classproperty - @functools.lru_cache() + @functools.lru_cache def page_mask(cls) -> int: """Page mask for the intel memory layers.""" return ~(cls.page_size - 1) @classproperty - @functools.lru_cache() + @functools.lru_cache def bits_per_register(cls) -> int: """Returns the bits_per_register to determine the range of an IntelTranslationLayer.""" return cls._bits_per_register @classproperty - @functools.lru_cache() + @functools.lru_cache def minimum_address(cls) -> int: return 0 @classproperty - @functools.lru_cache() + @functools.lru_cache def maximum_address(cls) -> int: return (1 << cls._maxvirtaddr) - 1 @@ -251,12 +251,7 @@ class Intel(linear.LinearlyMappedLayer): if INTEL_TRANSLATION_DEBUGGING: vollog.log( constants.LOGLEVEL_VVVV, - "Entry {} at index {} gives data {} as {}".format( - hex(entry), - hex(index), - hex(struct.unpack(self._entry_format, entry_data)[0]), - name, - ), + f"Entry {hex(entry)} at index {hex(index)} gives data {hex(struct.unpack(self._entry_format, entry_data)[0])} as {name}", ) # Read out the new entry from memory diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index 542fd6ca2..eeede1673 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -48,7 +48,7 @@ if HAS_LEECHCORE: try: self._handle = leechcorepyc.LeechCore(self._device) except TypeError: - raise IOError(f"Unable to open LeechCore device {self._device}") + raise OSError(f"Unable to open LeechCore device {self._device}") return self._handle def fileno(self): diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index ff483291c..a8127e954 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -236,7 +236,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if self._architecture is None: vollog.log( constants.LOGLEVEL_VV, - f"QEVM architecture could not be determined", + "QEVM architecture could not be determined", ) # Once all segments have been read, determine the PCI hole if any diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 609832886..cc364ad50 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -156,9 +156,7 @@ class RegistryHive(linear.LinearlyMappedLayer): else: # It doesn't matter that we use KeyNode, we're just after the first two bytes vollog.debug( - "Unknown Signature {} (0x{:x}) at offset {}".format( - signature, cell.u.KeyNode.Signature, cell_offset - ) + f"Unknown Signature {signature} (0x{cell.u.KeyNode.Signature:x}) at offset {cell_offset}" ) return cell @@ -178,9 +176,7 @@ class RegistryHive(linear.LinearlyMappedLayer): if not root_node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"): raise RegistryFormatException( self.name, - "Encountered {} instead of _CM_KEY_NODE".format( - root_node.vol.type_name - ), + f"Encountered {root_node.vol.type_name} instead of _CM_KEY_NODE", ) node_key = [root_node] if key.endswith("\\"): diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 2dba7caa8..c7a7fee67 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -57,7 +57,7 @@ def cascadeCloseFile(new_fp: IO[bytes], original_fp: IO[bytes]) -> IO[bytes]: return new_fp -class ResourceAccessor(object): +class ResourceAccessor: """Object for opening URLs as files (downloading locally first if necessary)""" diff --git a/volatility3/framework/layers/scanners/multiregexp.py b/volatility3/framework/layers/scanners/multiregexp.py index be3581f05..9831a9d8e 100644 --- a/volatility3/framework/layers/scanners/multiregexp.py +++ b/volatility3/framework/layers/scanners/multiregexp.py @@ -6,7 +6,7 @@ import re from typing import Generator, List, Tuple -class MultiRegexp(object): +class MultiRegexp: """Algorithm for multi-string matching.""" def __init__(self) -> None: diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index 8aa527cdb..b241ed56a 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -22,8 +22,8 @@ def bswap_32(value: int) -> int: def bswap_64(value: int) -> int: - low = bswap_32((value >> 32)) - high = bswap_32((value & 0xFFFFFFFF)) + low = bswap_32(value >> 32) + high = bswap_32(value & 0xFFFFFFFF) return ((high << 32) | low) & 0xFFFFFFFFFFFFFFFF diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 24149a390..10e7a7a72 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -119,7 +119,7 @@ class LayerWriter(plugins.PluginInterface): # Update the filename, which may have changed if a file # with the same name already existed. output_name = file_handle.preferred_filename - except IOError as excp: + except OSError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) yield 0, (f"Layer has been written to {output_name}",) diff --git a/volatility3/framework/plugins/linux/check_idt.py b/volatility3/framework/plugins/linux/check_idt.py index cc3a08933..07582e2c1 100644 --- a/volatility3/framework/plugins/linux/check_idt.py +++ b/volatility3/framework/plugins/linux/check_idt.py @@ -53,7 +53,7 @@ class Check_idt(interfaces.plugins.PluginInterface): address_mask = self.context.layers[vmlinux.layer_name].address_mask # hw handlers + system call - check_idxs = list(range(0, 20)) + [128] + check_idxs = list(range(20)) + [128] if is_32bit: if vmlinux.has_type("gate_struct"): diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index b6634d612..3537a9fa1 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -103,7 +103,7 @@ class Check_syscall(plugins.PluginInterface): try: func_addr = vmlinux.get_symbol(syscall_entry_func).address - except exceptions.SymbolError as e: + except exceptions.SymbolError: # if we can't find the disassemble function then bail and rely on a different method return 0 diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 7dbf074b3..382268515 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -462,7 +462,7 @@ class InodePages(plugins.PluginInterface): f.seek(current_fp) f.write(page_bytes) - except IOError as e: + except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) def _generator(self): diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 00832140a..065f239a9 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -125,9 +125,7 @@ class Maps(plugins.PluginInterface): proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None vm_size = vm_end - vm_start diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 40784a647..0cca4704f 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -133,7 +133,7 @@ class PsScan(interfaces.plugins.PluginInterface): ) elif len(kernel_layer.dependencies) == 0: vollog.error( - f"Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." + "Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." ) raise exceptions.LayerException( kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index fe5179dfa..5c002e472 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -115,9 +115,7 @@ class Maps(interfaces.plugins.PluginInterface): proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None vm_size = vm_end - vm_start diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index c754e43ef..ba729f898 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -143,9 +143,7 @@ class Timeliner(interfaces.plugins.PluginInterface): times = self.timeline.get((plugin_name, item), {}) if times.get(timestamp_type, None) is not None: vollog.debug( - "Multiple timestamps for the same plugin/file combination found: {} {}".format( - plugin_name, item - ) + f"Multiple timestamps for the same plugin/file combination found: {plugin_name} {item}" ) times[timestamp_type] = timestamp self.timeline[(plugin_name, item)] = times diff --git a/volatility3/framework/plugins/windows/cmdline.py b/volatility3/framework/plugins/windows/cmdline.py index 8cfb5576c..9bd9eda0e 100644 --- a/volatility3/framework/plugins/windows/cmdline.py +++ b/volatility3/framework/plugins/windows/cmdline.py @@ -84,9 +84,7 @@ class CmdLine(interfaces.plugins.PluginInterface): result_text = f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)" except exceptions.InvalidAddressException as exp: - result_text = "Process {}: Required memory at {:#x} is not valid (incomplete layer {}?)".format( - proc_id, exp.invalid_address, exp.layer_name - ) + result_text = f"Process {proc_id}: Required memory at {exp.invalid_address:#x} is not valid (incomplete layer {exp.layer_name}?)" yield (0, (proc.UniqueProcessId, process_name, result_text)) diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index ad1c9d4bd..a448989c0 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -95,9 +95,7 @@ class Consoles(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) @classmethod @@ -176,12 +174,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) vollog.debug( - "Determined OS Version: {}.{} {}.{}".format( - kuser.NtMajorVersion, - kuser.NtMinorVersion, - vers.MajorVersion, - vers.MinorVersion, - ) + f"Determined OS Version: {kuser.NtMajorVersion}.{kuser.NtMinorVersion} {vers.MajorVersion}.{vers.MinorVersion}" ) if nt_major_version == 10 and arch == "x64": @@ -260,9 +253,7 @@ class Consoles(interfaces.plugins.PluginInterface): if ver: conhost_mod_version = ver[3] vollog.debug( - "Determined conhost.exe's FileVersion: {}".format( - conhost_mod_version - ) + f"Determined conhost.exe's FileVersion: {conhost_mod_version}" ) else: vollog.debug("Could not determine conhost.exe's FileVersion.") @@ -311,12 +302,7 @@ class Consoles(interfaces.plugins.PluginInterface): else: raise NotImplementedError( - "This version of Windows is not supported: {}.{} {}.{}!".format( - nt_major_version, - nt_minor_version, - vers.MajorVersion, - vers_minor_version, - ) + f"This version of Windows is not supported: {nt_major_version}.{nt_minor_version} {vers.MajorVersion}.{vers_minor_version}!" ) vollog.debug(f"Determined symbol filename: {filename}") diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 5a1b37fcf..57f19f620 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -5,9 +5,9 @@ import contextlib import datetime import logging import re -from typing import List, Optional, Type +from typing import List -from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed @@ -199,16 +199,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _depth, row_data = row if not isinstance(row_data[6], datetime.datetime): continue - description = ( - "DLL Load: Process {} {} Loaded {} ({}) Size {} Offset {}".format( - row_data[0], - row_data[1], - row_data[4], - row_data[5], - row_data[3], - row_data[2], - ) - ) + description = f"DLL Load: Process {row_data[0]} {row_data[1]} Loaded {row_data[4]} ({row_data[5]}) Size {row_data[3]} Offset {row_data[2]}" yield (description, timeliner.TimeLinerType.CREATED, row_data[6]) def run(self): diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index bc554c0bf..64d9be4db 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -192,13 +192,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] - desired_file_name = "file.{0:#x}.{1:#x}.{2}.{3}.{4}".format( - file_obj.vol.offset, - memory_object.vol.offset, - cache_name, - ntpath.basename(obj_name), - extension, - ) + desired_file_name = f"file.{file_obj.vol.offset:#x}.{memory_object.vol.offset:#x}.{cache_name}.{ntpath.basename(obj_name)}.{extension}" file_handle = cls.dump_file_producer( file_obj, memory_object, open_method, layer, desired_file_name diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index 66db03c9c..cac4ecf40 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -92,7 +92,7 @@ class Envars(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing global environment variables keys (some keys might be excluded)", @@ -113,7 +113,7 @@ class Envars(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing user environment variables keys (some keys might be excluded)", @@ -134,7 +134,7 @@ class Envars(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing volatile environment variables keys (some keys might be excluded)", diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index 9b20ed2d0..37df940a2 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -55,7 +55,7 @@ class GetServiceSIDs(interfaces.plugins.PluginInterface): ) # Get service sids dictionary (we need only the service sids). - with open(sids_json_file_name, "r") as file_handle: + with open(sids_json_file_name) as file_handle: self.servicesids = json.load(file_handle)["service sids"] @classmethod diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index 3e332f85d..df0c7a835 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -58,7 +58,7 @@ class GetSIDs(interfaces.plugins.PluginInterface): ) # Get all the sids from the json file. - with open(sids_json_file_name, "r") as file_handle: + with open(sids_json_file_name) as file_handle: sids_json_data = json.load(file_handle) self.servicesids = sids_json_data["service sids"] self.well_known_sids = sids_json_data["well known"] @@ -122,7 +122,7 @@ class GetSIDs(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, layers.registry.RegistryFormatException, - ) as excp: + ): continue try: value_data = node.decode_data() @@ -156,7 +156,7 @@ class GetSIDs(interfaces.plugins.PluginInterface): ValueError, exceptions.InvalidAddressException, layers.registry.RegistryFormatException, - ) as excp: + ): continue except (KeyError, exceptions.InvalidAddressException): continue diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index 69fa94f06..30d4b602c 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -12,20 +12,15 @@ from volatility3.plugins.windows import pslist, vadinfo vollog = logging.getLogger(__name__) -VadData = NamedTuple( - "VadData", - [ - ("protection", str), - ("path", str), - ], -) -DLLData = NamedTuple( - "DLLData", - [ - ("path", str), - ], -) +class VadData(NamedTuple): + protection: str + path: str + + +class DLLData(NamedTuple): + path: str + ### Useful references on process hollowing # https://cysinfo.com/detecting-deceptive-hollowing-techniques/ @@ -146,9 +141,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): """ image_base = self._get_image_base(proc) if image_base is not None and image_base != proc.SectionBaseAddress: - yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format( - image_base, proc.SectionBaseAddress - ) + yield f"The ImageBaseAddress reported from the PEB ({image_base:#x}) does not match the process SectionBaseAddress ({proc.SectionBaseAddress:#x})" def _check_exe_protection( self, proc, vads: Dict[int, VadData], __ @@ -166,13 +159,9 @@ class HollowProcesses(interfaces.plugins.PluginInterface): base = proc.SectionBaseAddress if base not in vads: - yield "There is no VAD starting at the base address of the process executable ({:#x})".format( - base - ) + yield f"There is no VAD starting at the base address of the process executable ({base:#x})" elif vads[base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for VAD hosting the process executable ({:#x}) with path {}".format( - vads[base].protection, base, vads[base].path - ) + yield f"Unexpected protection ({vads[base].protection}) for VAD hosting the process executable ({base:#x}) with path {vads[base].path}" def _check_dlls_protection( self, _, vads: Dict[int, VadData], dlls: Dict[int, DLLData] @@ -184,9 +173,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): # PAGE_EXECUTE_WRITECOPY is the only valid permission for mapped DLLs and .exe files if vads[dll_base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format( - vads[dll_base].protection, dll_base, dlls[dll_base].path - ) + yield f"Unexpected protection ({vads[dll_base].protection}) for DLL in the PEB's load order list ({dll_base:#x}) with path {dlls[dll_base].path}" def _generator(self, procs): checks = [ diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index d2fdc0ad8..3bf7f57ed 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,7 +1,9 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -import logging, io, pefile +import logging +import io +import pefile from volatility3.framework.symbols import intermed from volatility3.framework import renderers, interfaces, exceptions, constants from volatility3.framework.configuration import requirements @@ -119,9 +121,7 @@ class IAT(interfaces.plugins.PluginInterface): ) except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 1df090b5b..510719352 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -106,9 +106,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None @@ -211,9 +209,7 @@ class Malfind(interfaces.plugins.PluginInterface): file_output = file_handle.preferred_filename except (exceptions.InvalidAddressException, OverflowError) as excp: vollog.debug( - "Unable to dump PE with pid {0}.{1:#x}: {2}".format( - proc.UniqueProcessId, vad.get_start(), excp - ) + f"Unable to dump PE with pid {proc.UniqueProcessId}.{vad.get_start():#x}: {excp}" ) yield ( diff --git a/volatility3/framework/plugins/windows/memmap.py b/volatility3/framework/plugins/windows/memmap.py index b5c9a211e..62ab3c510 100644 --- a/volatility3/framework/plugins/windows/memmap.py +++ b/volatility3/framework/plugins/windows/memmap.py @@ -53,9 +53,7 @@ class Memmap(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue @@ -80,11 +78,7 @@ class Memmap(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: file_output = "Error outputting to file" vollog.debug( - "Unable to write {}'s address {} to {}".format( - proc_layer_name, - offset, - file_handle.preferred_filename, - ) + f"Unable to write {proc_layer_name}'s address {offset} to {file_handle.preferred_filename}" ) yield ( diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index ba45834d5..2e8dc1b0e 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -177,9 +177,7 @@ class Modules(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, - "Process {} does not have a valid Session or a layer could not be constructed for it".format( - proc_id - ), + f"Process {proc_id} does not have a valid Session or a layer could not be constructed for it", ) continue diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 66a24da5a..77bd22ab9 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -169,12 +169,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) vollog.debug( - "Determined OS Version: {}.{} {}.{}".format( - kuser.NtMajorVersion, - kuser.NtMinorVersion, - vers.MajorVersion, - vers.MinorVersion, - ) + f"Determined OS Version: {kuser.NtMajorVersion}.{kuser.NtMinorVersion} {vers.MajorVersion}.{vers.MinorVersion}" ) if nt_major_version == 10 and arch == "x64": @@ -272,9 +267,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if ver: tcpip_mod_version = ver[3] vollog.debug( - "Determined tcpip.sys's FileVersion: {}".format( - tcpip_mod_version - ) + f"Determined tcpip.sys's FileVersion: {tcpip_mod_version}" ) else: vollog.debug("Could not determine tcpip.sys's FileVersion.") @@ -316,12 +309,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): else: raise NotImplementedError( - "This version of Windows is not supported: {}.{} {}.{}!".format( - nt_major_version, - nt_minor_version, - vers.MajorVersion, - vers_minor_version, - ) + f"This version of Windows is not supported: {nt_major_version}.{nt_minor_version} {vers.MajorVersion}.{vers_minor_version}!" ) vollog.debug(f"Determined symbol filename: {filename}") @@ -510,17 +498,8 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for i in row_data ] description = ( - "Network connection: Process {} {} Local Address {}:{} " - "Remote Address {}:{} State {} Protocol {} ".format( - row_data[7], - row_data[8], - row_data[2], - row_data[3], - row_data[4], - row_data[5], - row_data[6], - row_data[1], - ) + f"Network connection: Process {row_data[7]} {row_data[8]} Local Address {row_data[2]}:{row_data[3]} " + f"Remote Address {row_data[4]}:{row_data[5]} State {row_data[6]} Protocol {row_data[1]} " ) yield (description, timeliner.TimeLinerType.CREATED, row_data[9]) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 0908767fc..c774e23a3 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -311,9 +311,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): part_table.Partitions.count = part_count vollog.debug( - "Found TCP connection PartitionTable @ 0x{:x} (partition count: {})".format( - part_table_addr, part_count - ) + f"Found TCP connection PartitionTable @ 0x{part_table_addr:x} (partition count: {part_count})" ) entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset( "ListEntry" @@ -624,9 +622,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): proto = "TCPv6" else: vollog.debug( - "TCP Endpoint @ 0x{:2x} has unknown address family 0x{:x}".format( - netw_obj.vol.offset, netw_obj.get_address_family() - ) + f"TCP Endpoint @ 0x{netw_obj.vol.offset:2x} has unknown address family 0x{netw_obj.get_address_family():x}" ) proto = "TCPv?" diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 955098d6b..002577241 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -645,7 +645,7 @@ class PESymbols(interfaces.plugins.PluginInterface): and wanted_addresses_identifier not in wanted_symbols ): vollog.warning( - f"Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." + "Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." ) return diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 5b4bb07d7..d2a8a7370 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -64,27 +64,30 @@ class PEDump(interfaces.plugins.PluginInterface): """ Returns the filename of the dump file or None """ - with open_method(file_name) as file_handle: - try: - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=base, - layer_name=layer_name, - ) + try: + file_handle = open_method(file_name) - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - IOError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") - return None + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) - return file_handle.preferred_filename + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + OSError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None + finally: + file_handle.close() + + return file_handle.preferred_filename @classmethod def dump_ldr_entry( @@ -116,12 +119,7 @@ class PEDump(interfaces.plugins.PluginInterface): if layer_name is None: layer_name = ldr_entry.vol.layer_name - file_name = "{}{}.{:#x}.{:#x}.dmp".format( - prefix, - ntpath.basename(name), - ldr_entry.vol.offset, - ldr_entry.DllBase, - ) + file_name = f"{prefix}{ntpath.basename(name)}.{ldr_entry.vol.offset:#x}.{ldr_entry.DllBase:#x}.dmp" return cls.dump_pe( context, @@ -143,11 +141,7 @@ class PEDump(interfaces.plugins.PluginInterface): pid: int, base: int, ) -> Optional[str]: - file_name = "PE.{:#x}.{:d}.{:#x}.dmp".format( - proc_offset, - pid, - base, - ) + file_name = f"PE.{proc_offset:#x}.{pid:d}.{base:#x}.dmp" return PEDump.dump_pe( context, pe_table_name, layer_name, open_method, file_name, base diff --git a/volatility3/framework/plugins/windows/privileges.py b/volatility3/framework/plugins/windows/privileges.py index 0370dfc92..7b4d00205 100644 --- a/volatility3/framework/plugins/windows/privileges.py +++ b/volatility3/framework/plugins/windows/privileges.py @@ -39,7 +39,7 @@ class Privs(interfaces.plugins.PluginInterface): ) # Get service sids dictionary (we need only the service sids). - with open(sids_json_file_name, "r") as file_handle: + with open(sids_json_file_name) as file_handle: temp_json = json.load(file_handle)["privileges"] self.privilege_info = { int(priv_num): temp_json[priv_num] for priv_num in temp_json diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 6c845bf81..053ec20d5 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -14,7 +14,6 @@ from volatility3.plugins.windows import ( info, pslist, psscan, - sessions, thrdscan, ) diff --git a/volatility3/framework/plugins/windows/registry/hivelist.py b/volatility3/framework/plugins/windows/registry/hivelist.py index ddc9c1855..91a99a9fb 100644 --- a/volatility3/framework/plugins/windows/registry/hivelist.py +++ b/volatility3/framework/plugins/windows/registry/hivelist.py @@ -232,10 +232,8 @@ class HiveList(interfaces.plugins.PluginInterface): for hive in hg: if hive.vol.offset in seen: vollog.debug( - "Hivelist found an already seen offset {} while " - "traversing forwards, this should not occur".format( - hex(hive.vol.offset) - ) + f"Hivelist found an already seen offset {hex(hive.vol.offset)} while " + "traversing forwards, this should not occur" ) break seen.add(hive.vol.offset) @@ -249,18 +247,14 @@ class HiveList(interfaces.plugins.PluginInterface): forward_invalid = hg.invalid if forward_invalid: vollog.debug( - "Hivelist failed traversing the list forwards at {}, traversing backwards".format( - hex(forward_invalid) - ) + f"Hivelist failed traversing the list forwards at {hex(forward_invalid)}, traversing backwards" ) hg = HiveGenerator(cmhive, forward=False) for hive in hg: if hive.vol.offset in seen: vollog.debug( - "Hivelist found an already seen offset {} while " - "traversing backwards, list walking met in the middle".format( - hex(hive.vol.offset) - ) + f"Hivelist found an already seen offset {hex(hive.vol.offset)} while " + "traversing backwards, list walking met in the middle" ) break seen.add(hive.vol.offset) @@ -281,10 +275,8 @@ class HiveList(interfaces.plugins.PluginInterface): # by walking the list, so revert to scanning, and walk the list forwards and backwards from each # found hive vollog.debug( - "Hivelist failed traversing backwards at {}, a different " - "location from forwards, revert to scanning".format( - hex(backward_invalid) - ) + f"Hivelist failed traversing backwards at {hex(backward_invalid)}, a different " + "location from forwards, revert to scanning" ) for hive in hivescan.HiveScan.scan_hives( context, layer_name, symbol_table @@ -320,9 +312,7 @@ class HiveList(interfaces.plugins.PluginInterface): yield linked_hive except exceptions.InvalidAddressException: vollog.debug( - "InvalidAddressException when traversing hive {} found from scan, skipping".format( - hex(hive.vol.offset) - ) + f"InvalidAddressException when traversing hive {hex(hive.vol.offset)} found from scan, skipping" ) def run(self) -> renderers.TreeGrid: diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index bd832b20c..932ee9d6f 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -39,7 +39,7 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac os.path.join(os.path.dirname(__file__), "userassist.json"), "rb" ) as fp: self._folder_guids = json.load(fp) - except IOError: + except OSError: vollog.error("Usersassist data file not found") @classmethod @@ -308,9 +308,7 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac ) except exceptions.InvalidAddressException as excp: vollog.debug( - "Invalid address identified in lower layer {}: {}".format( - excp.layer_name, excp.invalid_address - ) + f"Invalid address identified in lower layer {excp.layer_name}: {excp.invalid_address}" ) except KeyError: vollog.debug( diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index d321c2cc0..f5d7e1b3a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -172,9 +172,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.debug( - "Unable to construct cSystems array at given offset: {:x}".format( - array_start - ) + f"Unable to construct cSystems array at given offset: {array_start:x}" ) array = None @@ -291,9 +289,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None, None diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 0eaa65884..b8dea0cdd 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -170,9 +170,7 @@ class Strings(interfaces.plugins.PluginInterface): proc_layer_name = process.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index a59581063..7c26a09bd 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -85,9 +85,7 @@ class SvcList(svcscan.SvcScan): layer_name = proc.add_process_layer() except exceptions.InvalidAddressException: vollog.warning( - "Unable to access memory of services.exe running with PID: {}".format( - proc.UniqueProcessId - ) + f"Unable to access memory of services.exe running with PID: {proc.UniqueProcessId}" ) continue diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index ca390561f..bd477ba27 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -26,13 +26,9 @@ from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) -ServiceBinaryInfo = NamedTuple( - "ServiceBinaryInfo", - [ - ("dll", Union[str, interfaces.renderers.BaseAbsentValue]), - ("binary", Union[str, interfaces.renderers.BaseAbsentValue]), - ], -) +class ServiceBinaryInfo(NamedTuple): + dll: Union[str, interfaces.renderers.BaseAbsentValue] + binary: Union[str, interfaces.renderers.BaseAbsentValue] class SvcScan(interfaces.plugins.PluginInterface): @@ -306,9 +302,7 @@ class SvcScan(interfaces.plugins.PluginInterface): proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index b812a15ff..c0963e754 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -82,7 +82,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ethread.get_exit_time() ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object except exceptions.InvalidAddressException: - vollog.debug("Thread invalid address {:#x}".format(ethread.vol.offset)) + vollog.debug(f"Thread invalid address {ethread.vol.offset:#x}") return None return ( diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index 98a3169a5..a34818fc1 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -3,7 +3,7 @@ # import logging -from typing import Callable, Iterable, List, Generator +from typing import Iterable, List, Generator from volatility3.framework import interfaces, constants from volatility3.framework.configuration import requirements diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index d49c28784..54bca1a1e 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -141,7 +141,7 @@ class Timers(interfaces.plugins.PluginInterface): if dpc.DeferredRoutine == 0: continue deferred_routine = dpc.DeferredRoutine - except Exception as e: + except Exception: continue module_symbols = list( diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index 1a1e59940..5b21225c8 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -191,7 +191,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): # gather processes on small_idx since these are the malware infected ones for pid, pname in cb[small_idx]: - ps.append("{:d}:{}".format(pid, pname)) + ps.append(f"{pid:d}:{pname}") proc_names = ", ".join(ps) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 2c6ed4daf..0c4a8aaca 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -169,9 +169,7 @@ class VadInfo(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 4a06ed0c9..4930789d2 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -212,9 +212,7 @@ class VerInfo(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 43bb59a21..02805acc2 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -88,9 +88,7 @@ class TreeNode(interfaces.renderers.TreeNode): val = values[index] if not isinstance(val, (column.type, interfaces.renderers.BaseAbsentValue)): raise TypeError( - "Values item with index {} is the wrong type for column {} (got {} but expected {})".format( - index, column.name, type(val), column.type - ) + f"Values item with index {index} is the wrong type for column {column.name} (got {type(val)} but expected {column.type})" ) # TODO: Consider how to deal with timezone naive/aware datetimes (and alert plugin uses to be precise) # if isinstance(val, datetime.datetime): @@ -189,9 +187,7 @@ class TreeGrid(interfaces.renderers.TreeGrid): is_simple_type = issubclass(column_type, self.base_types) if not is_simple_type: raise TypeError( - "Column {}'s type is not a simple type: {}".format( - name, column_type.__class__.__name__ - ) + f"Column {name}'s type is not a simple type: {column_type.__class__.__name__}" ) converted_columns.append(interfaces.renderers.Column(name, column_type)) self.RowStructure = RowStructureConstructor( diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 5f558bf12..8a28d732f 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -171,7 +171,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): (indicating that only additive changes have been made) than the consumer (in this case, the file reader). """ - major, minor, patch = [int(x) for x in version.split(".")] + major, minor, patch = (int(x) for x in version.split(".")) supported_versions = [x for x in versions if x[0] == major and x[1] >= minor] if not supported_versions: raise ValueError( diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..537b729ad 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -798,7 +798,7 @@ class RadixTree(IDStorage): return True -class PageCache(object): +class PageCache: """Linux Page Cache abstraction""" def __init__( diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 829622154..e9fd09eaa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1469,7 +1469,7 @@ class mount(objects.StructType): def next_peer(self): table_name = self.vol.type_name.split(constants.BANG)[0] - mount_struct = "{0}{1}mount".format(table_name, constants.BANG) + mount_struct = f"{table_name}{constants.BANG}mount" offset = self._context.symbol_space.get_type( mount_struct ).relative_child_offset("mnt_share") @@ -2487,7 +2487,7 @@ class address_space(objects.StructType): class page(objects.StructType): @property - @functools.lru_cache() + @functools.lru_cache def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 793e506c3..07ab5f5a8 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1081,7 +1081,7 @@ class KTIMER(objects.StructType): return self.Header.Type in self.VALID_TYPES def get_due_time(self): - return "{0:#010x}:{1:#010x}".format(self.DueTime.HighPart, self.DueTime.LowPart) + return f"{self.DueTime.HighPart:#010x}:{self.DueTime.LowPart:#010x}" def get_dpc(self): """Return Dpc, and if Windows 7 or later, decode it""" @@ -1388,7 +1388,7 @@ class SHARED_CACHE_MAP(objects.StructType): ) # Iterate through the entries - for counter in range(0, self.VACB_ARRAY): + for counter in range(self.VACB_ARRAY): # Check if the VACB entry is in use if not vacb_array[counter]: continue @@ -1472,7 +1472,7 @@ class SHARED_CACHE_MAP(objects.StructType): if not section_size > self.VACB_SIZE_OF_FIRST_LEVEL: array_head = vacb_obj - for counter in range(0, full_blocks): + for counter in range(full_blocks): vacb_entry = self._context.object( symbol_table_name + constants.BANG + "pointer", layer_name=self.vol.layer_name, @@ -1531,7 +1531,7 @@ class SHARED_CACHE_MAP(objects.StructType): # Walk the array and if any entry points to the shared cache map object then we extract it. # Otherwise, if it is non-zero, then traverse to the next level. - for counter in range(0, self.VACB_ARRAY): + for counter in range(self.VACB_ARRAY): if not vacb_array[counter]: continue diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 2312149c7..cf6f43a9b 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -73,7 +73,7 @@ class ROW(objects.StructType): ) for i in range(0, len(char_row), 3) ) - except Exception as e: + except Exception: line = "" if truncate: diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index afdc73a17..078c4beb0 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -8,12 +8,7 @@ from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: """Get Disk Signature (GUID).""" - return "{0:02x}-{1:02x}-{2:02x}-{3:02x}".format( - self.DiskSignature[0], - self.DiskSignature[1], - self.DiskSignature[2], - self.DiskSignature[3], - ) + return f"{self.DiskSignature[0]:02x}-{self.DiskSignature[1]:02x}-{self.DiskSignature[2]:02x}-{self.DiskSignature[3]:02x}" class PARTITION_ENTRY(objects.StructType): diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 9b7573c2e..00c24f176 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -22,7 +22,7 @@ def inet_ntop(address_family: int, packed_ip: Union[List[int], Array]) -> str: raise RuntimeError( "This version of python does not have socket.inet_ntop, please upgrade" ) - raise socket.error("[Errno 97] Address family not supported by protocol") + raise OSError("[Errno 97] Address family not supported by protocol") # Python's socket.AF_INET6 is 0x1e but Microsoft defines it @@ -167,11 +167,9 @@ class _TCP_LISTENER(objects.StructType): def is_valid(self): try: - if not self.get_address_family() in (AF_INET, AF_INET6): + if self.get_address_family() not in (AF_INET, AF_INET6): vollog.debug( - "netw obj 0x{:x} invalid due to invalid address_family {}".format( - self.vol.offset, self.get_address_family() - ) + f"netw obj 0x{self.vol.offset:x} invalid due to invalid address_family {self.get_address_family()}" ) return False diff --git a/volatility3/framework/symbols/windows/extensions/pe.py b/volatility3/framework/symbols/windows/extensions/pe.py index 3f34fc3dd..2c7400f25 100644 --- a/volatility3/framework/symbols/windows/extensions/pe.py +++ b/volatility3/framework/symbols/windows/extensions/pe.py @@ -101,9 +101,9 @@ class IMAGE_DOS_HEADER(objects.StructType): ) except OverflowError: vollog.warning( - "Volatility was unable to fix the image base for the PE file at base address {:#x}. " + f"Volatility was unable to fix the image base for the PE file at base address {self.vol.offset:#x}. " "This will cause issues with many static analysis tools if you do not inform the " - "tool of the in-memory load address.".format(self.vol.offset) + "tool of the in-memory load address." ) new_pe = raw_data diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index b761ddad8..5a7847986 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -217,7 +217,7 @@ class POOL_HEADER(objects.StructType): yield mem_object @classmethod - @functools.lru_cache() + @functools.lru_cache def _calculate_optional_header_lengths( cls, context: interfaces.context.ContextInterface, symbol_table_name: str ) -> Tuple[List[str], List[int]]: @@ -430,9 +430,7 @@ class OBJECT_HEADER(objects.StructType): if header_offset == 0: raise ValueError( - "Could not find _OBJECT_HEADER_NAME_INFO for object at {} of layer {}".format( - self.vol.offset, self.vol.layer_name - ) + f"Could not find _OBJECT_HEADER_NAME_INFO for object at {self.vol.offset} of layer {self.vol.layer_name}" ) header = ntkrnlmp.object( diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index bebfaea89..9e2f8df3b 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -196,9 +196,7 @@ class CM_KEY_NODE(objects.StructType): yield cast("CM_KEY_NODE", node) else: vollog.debug( - "Unexpected node type encountered when traversing subkeys: {}, signature: {}".format( - node.vol.type_name, signature - ) + f"Unexpected node type encountered when traversing subkeys: {node.vol.type_name}, signature: {signature}" ) if listjump: diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index 82ec31ccb..4feb396e7 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -263,9 +263,7 @@ class PdbReader: ) if header.index_max < header.index_min: raise ValueError( - "Maximum {} index is smaller than minimum TPI index, found: {} < {} ".format( - stream_name, header.index_max, header.index_min - ) + f"Maximum {stream_name} index is smaller than minimum TPI index, found: {header.index_max} < {header.index_min} " ) # Reset the state info_references: Dict[str, int] = {} @@ -976,7 +974,7 @@ class PdbRetreiver: if __name__ == "__main__": import argparse - class PrintedProgress(object): + class PrintedProgress: """A progress handler that prints the progress value and the description onto the command line.""" diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 3816312cd..1a8644fa8 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -94,7 +94,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): if not requirements.VersionRequirement.matches_required( (1, 0, 0), symbol_cache.SqliteCache.version ): - vollog.debug(f"Required version of SQLiteCache not found") + vollog.debug("Required version of SQLiteCache not found") return None identifiers_path = os.path.join( @@ -291,9 +291,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): break except PermissionError: vollog.warning( - "Cannot write necessary symbol file, please check permissions on {}".format( - potential_output_filename - ) + f"Cannot write necessary symbol file, please check permissions on {potential_output_filename}" ) continue finally: diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 5ef840f32..8587b3719 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -60,7 +60,7 @@ class Certificates(interfaces.plugins.PluginInterface): open_method: Type[interfaces.plugins.FileHandlerInterface], ) -> Optional[interfaces.plugins.FileHandlerInterface]: try: - dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) + dump_name = f"{hive_offset}-{reg_section}-{key_hash}.crt" file_handle = open_method(dump_name) file_handle.write(certificate_data) return file_handle diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 7f56b75f8..e7557dc0c 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -64,9 +64,7 @@ class Statistics(plugins.PluginInterface): other_invalid += 1 page_size = expected_page_size vollog.debug( - "A non-page lookup invalid address exception occurred at: {} in layer {}".format( - hex(excp.invalid_address), excp.layer_name - ) + f"A non-page lookup invalid address exception occurred at: {hex(excp.invalid_address)} in layer {excp.layer_name}" ) page_addr += page_size diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 3ca00e5dc..90cfaba48 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -20,7 +20,7 @@ def load_cached_validations() -> Set[str]: to revalidate them.""" validhashes: Set = set() if os.path.exists(cached_validation_filepath): - with open(cached_validation_filepath, "r") as f: + with open(cached_validation_filepath) as f: validhashes.update(json.load(f)) return validhashes @@ -46,7 +46,7 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: if not os.path.exists(schema_path): vollog.debug(f"Schema for format not found: {schema_path}") return False - with open(schema_path, "r") as s: + with open(schema_path) as s: schema = json.load(s) return valid(input, schema, use_cache) @@ -66,7 +66,7 @@ def create_json_hash( if not os.path.exists(schema_path): vollog.debug(f"Schema for format not found: {schema_path}") return None - with open(schema_path, "r") as s: + with open(schema_path) as s: schema = json.load(s) return hashlib.sha1( bytes(json.dumps((input, schema), sort_keys=True), "utf-8") From a63ea662f46bcbcf156df1d4bd69257c4ac0a0b3 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 17:05:34 +0100 Subject: [PATCH 172/348] apply unsafe fixes (`ruff check --unsafe-fixes --fix`) --- test/plugins/windows/test_scheduled_tasks.py | 3 +- volatility3/framework/__init__.py | 9 +++-- volatility3/framework/plugins/linux/lsmod.py | 3 +- volatility3/framework/plugins/linux/proc.py | 4 ++- .../framework/plugins/linux/sockstat.py | 2 +- .../framework/plugins/mac/check_sysctl.py | 5 ++- volatility3/framework/plugins/mac/kevents.py | 3 +- volatility3/framework/plugins/mac/mount.py | 3 +- .../framework/plugins/mac/proc_maps.py | 4 ++- volatility3/framework/plugins/mac/pslist.py | 4 ++- volatility3/framework/plugins/timeliner.py | 3 +- .../framework/plugins/windows/handles.py | 6 ++-- .../framework/plugins/windows/modules.py | 3 +- .../framework/plugins/windows/netstat.py | 5 ++- .../framework/plugins/windows/pslist.py | 35 +++++++++++++------ .../framework/plugins/windows/psscan.py | 35 ++++++++++++------- .../framework/plugins/windows/shimcachemem.py | 7 ++-- .../framework/plugins/windows/svclist.py | 5 ++- .../framework/plugins/windows/threads.py | 3 +- .../plugins/windows/unloadedmodules.py | 3 +- .../framework/plugins/windows/virtmap.py | 3 +- .../framework/renderers/format_hints.py | 30 +++++++++------- .../framework/symbols/linux/__init__.py | 6 ++-- .../symbols/linux/extensions/__init__.py | 12 +++---- volatility3/framework/symbols/mac/__init__.py | 15 ++++---- .../symbols/windows/extensions/__init__.py | 15 ++++---- .../symbols/windows/extensions/consoles.py | 23 +++++------- .../framework/symbols/windows/pdbconv.py | 5 ++- 28 files changed, 129 insertions(+), 125 deletions(-) diff --git a/test/plugins/windows/test_scheduled_tasks.py b/test/plugins/windows/test_scheduled_tasks.py index 8f771b323..fdb19fbae 100644 --- a/test/plugins/windows/test_scheduled_tasks.py +++ b/test/plugins/windows/test_scheduled_tasks.py @@ -84,8 +84,7 @@ class TestActionsDecoding(unittest.TestCase): self.assertEqual(actions[0].action_type, scheduled_tasks.ActionType.Exe) except Exception: self.fail( - "ActionDecoder.decode should not raise exception:\n%s" - % traceback.format_exc() + f"ActionDecoder.decode should not raise exception:\n{traceback.format_exc()}" ) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index e0f7c778a..244b353a2 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -97,8 +97,7 @@ def class_subclasses(cls: Type[T]) -> Generator[Type[T], None, None]: # The typing system is not clever enough to realize that clazz has a hidden attr after the hasattr check if not hasattr(clazz, "hidden") or not clazz.hidden: # type: ignore yield clazz - for return_value in class_subclasses(clazz): - yield return_value + yield from class_subclasses(clazz) def import_files(base_module, ignore_errors: bool = False) -> List[str]: @@ -159,9 +158,9 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]: def _filter_files(filename: str): """Ensures that a filename traversed is an importable python file""" - return ( - filename.endswith(".py") or filename.endswith(".pyc") - ) and not filename.startswith("__") + return (filename.endswith((".py", ".pyc"))) and not filename.startswith( + "__" + ) def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str]: diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index a65b0d00b..49e990e93 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -54,8 +54,7 @@ class Lsmod(plugins.PluginInterface): table_name = modules.vol.type_name.split(constants.BANG)[0] - for module in modules.to_list(table_name + constants.BANG + "module", "list"): - yield module + yield from modules.to_list(table_name + constants.BANG + "module", "list") def _generator(self): try: diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 065f239a9..893eea71e 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -163,7 +163,9 @@ class Maps(plugins.PluginInterface): address_list = self.config.get("address", None) if not address_list: # do not filter as no address_list was supplied - vma_filter_func = lambda _: True + def vma_filter_func(_): + return True + else: # filter for any vm_start that matches the supplied address config def vma_filter_function(x: interfaces.objects.ObjectInterface) -> bool: diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index e5cf48d16..aee0b1e2e 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -372,7 +372,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): bt_sock = sock.cast("bt_sock") def bt_addr(addr): - return ":".join(reversed(["%02x" % x for x in addr.b])) + return ":".join(reversed([f"{x:02x}" for x in addr.b])) src_addr = src_port = dst_addr = dst_port = None bt_protocol = bt_sock.get_protocol() diff --git a/volatility3/framework/plugins/mac/check_sysctl.py b/volatility3/framework/plugins/mac/check_sysctl.py index 4f64eaed8..e8218962d 100644 --- a/volatility3/framework/plugins/mac/check_sysctl.py +++ b/volatility3/framework/plugins/mac/check_sysctl.py @@ -93,10 +93,9 @@ class Check_sysctl(plugins.PluginInterface): val = self._parse_global_variable_sysctls(kernel, name) elif ctltype == "CTLTYPE_NODE": if sysctl.oid_handler == 0: - for info in self._process_sysctl_list( + yield from self._process_sysctl_list( kernel, sysctl.oid_arg1, recursive=1 - ): - yield info + ) val = "Node" diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 2a8692b77..41fde31ca 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -119,8 +119,7 @@ class Kevents(interfaces.plugins.PluginInterface): return None for klist in klist_array: - for kn in mac.MacUtilities.walk_slist(klist, "kn_link"): - yield kn + yield from mac.MacUtilities.walk_slist(klist, "kn_link") @classmethod def _get_task_kevents(cls, kernel, task): diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index ff654e1a7..1a1e33571 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -49,8 +49,7 @@ class Mount(plugins.PluginInterface): list_head = kernel.object_from_symbol(symbol_name="mountlist") - for mount in mac.MacUtilities.walk_tailq(list_head, "mnt_list"): - yield mount + yield from mac.MacUtilities.walk_tailq(list_head, "mnt_list") def _generator(self): for mount in self.list_mounts(self.context, self.config["kernel"]): diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 5c002e472..bd905615d 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -152,7 +152,9 @@ class Maps(interfaces.plugins.PluginInterface): address_list = self.config.get("address", None) if not address_list: # do not filter as no address_list was supplied - vma_filter_func = lambda _: True + def vma_filter_func(_): + return True + else: # filter for any vm_start that matches the supplied address config def vma_filter_function(task: interfaces.objects.ObjectInterface) -> bool: diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 9b570f3f9..74d044ba9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -83,7 +83,9 @@ class PsList(interfaces.plugins.PluginInterface): @classmethod def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[int], bool]: - filter_func = lambda _: False + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 pid_list = pid_list or [] filter_list = [x for x in pid_list if x is not None] diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index ba729f898..4e483922b 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -204,8 +204,7 @@ class Timeliner(interfaces.plugins.PluginInterface): ) vollog.log(logging.DEBUG, traceback.format_exc()) - for data_item in sorted(data, key=self._sort_function): - yield data_item + yield from sorted(data, key=self._sort_function) # Write out a body file if necessary if self.config.get("create-bodyfile", True): diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index a3067b09f..62eceb973 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -227,8 +227,7 @@ class Handles(interfaces.plugins.PluginInterface): for entry in table: if level > 0: - for x in self._make_handle_array(entry, level - 1, depth): - yield x + yield from self._make_handle_array(entry, level - 1, depth) depth += 1 else: handle_multiplier = 4 @@ -264,8 +263,7 @@ class Handles(interfaces.plugins.PluginInterface): ) return None - for handle_table_entry in self._make_handle_array(TableCode, table_levels): - yield handle_table_entry + yield from self._make_handle_array(TableCode, table_levels) def _generator(self, procs): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 2e8dc1b0e..00424938f 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -248,8 +248,7 @@ class Modules(interfaces.plugins.PluginInterface): object_type=type_name, offset=list_entry.vol.offset - reloff, absolute=True ) - for mod in module.InLoadOrderLinks: - yield mod + yield from module.InLoadOrderLinks def run(self): return renderers.TreeGrid( diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index c774e23a3..a1521a8c6 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -488,14 +488,13 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ # first, TCP endpoints by parsing the partition table - for endpoint in cls.parse_partitions( + yield from cls.parse_partitions( context, layer_name, net_symbol_table, tcpip_symbol_table, tcpip_module_offset, - ): - yield endpoint + ) # then, towards the UDP and TCP port pools # first, find their addresses diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 478cc8b1b..f262aeae6 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -126,15 +126,24 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Returns: Filter function for passing to the `list_processes` method """ - filter_func = lambda _: False + + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 pid_list = pid_list or [] filter_list = [x for x in pid_list if x is not None] if filter_list: if exclude: - filter_func = lambda x: x.UniqueProcessId in filter_list + + def filter_func(x): + return x.UniqueProcessId in filter_list + else: - filter_func = lambda x: x.UniqueProcessId not in filter_list + + def filter_func(x): + return x.UniqueProcessId not in filter_list + return filter_func @classmethod @@ -173,20 +182,24 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Returns: Filter function for passing to the `list_processes` method """ - filter_func = lambda _: False + + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 name_list = name_list or [] filter_list = [x for x in name_list if x is not None] if filter_list: if exclude: - filter_func = ( - lambda x: utility.array_to_string(x.ImageFileName) in filter_list - ) + + def filter_func(x): + return utility.array_to_string(x.ImageFileName) in filter_list + else: - filter_func = ( - lambda x: utility.array_to_string(x.ImageFileName) - not in filter_list - ) + + def filter_func(x): + return utility.array_to_string(x.ImageFileName) not in filter_list + return filter_func @classmethod diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 5ce470cd8..86eb47300 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -102,29 +102,38 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Returns: Filter function to be passed to the list of processes. """ - filter_func = lambda _: False + + def filter_func(_): + return False if offset: if physical: if exclude: - filter_func = ( - lambda proc: cls.physical_offset_from_virtual( - context, layer_name, proc + + def filter_func(proc): + return ( + cls.physical_offset_from_virtual(context, layer_name, proc) + == offset ) - == offset - ) + else: - filter_func = ( - lambda proc: cls.physical_offset_from_virtual( - context, layer_name, proc + + def filter_func(proc): + return ( + cls.physical_offset_from_virtual(context, layer_name, proc) + != offset ) - != offset - ) + else: if exclude: - filter_func = lambda proc: proc.vol.offset == offset + + def filter_func(proc): + return proc.vol.offset == offset + else: - filter_func = lambda proc: proc.vol.offset != offset + + def filter_func(proc): + return proc.vol.offset != offset return filter_func diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 6afaf4356..3cf6d60d8 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -285,10 +285,9 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf if not shim_head: return - for shim_entry in shim_head.ListEntry.to_list( + yield from shim_head.ListEntry.to_list( shimcache_symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", "ListEntry" - ): - yield shim_entry + ) @classmethod def try_get_shim_head_at_offset( @@ -333,7 +332,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf eresource_rel_off = ersrc_size + ((offset - ersrc_size) % ersrc_alignment) eresource_offset = offset - eresource_rel_off - vollog.debug("Constructing ERESOURCE at %s" % hex(eresource_offset)) + vollog.debug(f"Constructing ERESOURCE at {hex(eresource_offset)}") eresource = context.object( kernel_symbol_table + constants.BANG + "_ERESOURCE", layer_name, diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index 7c26a09bd..ea73247ce 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -103,11 +103,10 @@ class SvcList(svcscan.SvcScan): scanner=scanners.BytesScanner(needle=b"Sc27"), sections=exe_range, ): - for record in cls.enumerate_vista_or_later_header( + yield from cls.enumerate_vista_or_later_header( context, service_table_name, service_binary_dll_map, layer_name, offset, - ): - yield record + ) diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index a34818fc1..84daa8595 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -82,5 +82,4 @@ class Threads(thrdscan.ThrdScan): symbol_table=symbol_table_name, filter_func=filter_func, ): - for thread in cls.list_threads(module, proc): - yield thread + yield from cls.list_threads(module, proc) diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py index 01e575818..077fe33cb 100644 --- a/volatility3/framework/plugins/windows/unloadedmodules.py +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -116,8 +116,7 @@ class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInt ) unloadedmodules_array.UnloadedDrivers.count = unloaded_count - for mod in unloadedmodules_array.UnloadedDrivers: - yield mod + yield from unloadedmodules_array.UnloadedDrivers def _generator(self): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/virtmap.py b/volatility3/framework/plugins/windows/virtmap.py index 3f3f270e2..e02cca89e 100644 --- a/volatility3/framework/plugins/windows/virtmap.py +++ b/volatility3/framework/plugins/windows/virtmap.py @@ -138,8 +138,7 @@ class VirtMap(interfaces.plugins.PluginInterface): mapping = cls.determine_map(module) for entry in mapping: if "Unused" not in entry: - for value in mapping[entry]: - yield value + yield from mapping[entry] def run(self): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index 194e38099..d57c7e9f1 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -70,15 +70,21 @@ class MultiTypeData(bytes): ) -BinOrAbsent = lambda x: ( - Bin(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -HexOrAbsent = lambda x: ( - Hex(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -HexBytesOrAbsent = lambda x: ( - HexBytes(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -MultiTypeDataOrAbsent = lambda x: ( - MultiTypeData(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) +def BinOrAbsent(x): + return Bin(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def HexOrAbsent(x): + return Hex(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def HexBytesOrAbsent(x): + return HexBytes(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def MultiTypeDataOrAbsent(x): + return ( + MultiTypeData(x) + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else x + ) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 537b729ad..0230a9c48 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -629,8 +629,7 @@ class IDStorage(ABC): if self.is_valid_node(nodep): yield nodep else: - for child_node in self._iter_node(nodep, height - 1): - yield child_node + yield from self._iter_node(nodep, height - 1) def get_entries(self, root: interfaces.objects.ObjectInterface) -> Iterator[int]: """Walks the tree data structure @@ -659,8 +658,7 @@ class IDStorage(ABC): if self.is_valid_node(nodep): yield nodep else: - for child_node in self._iter_node(nodep, height): - yield child_node + yield from self._iter_node(nodep, height) class XArray(IDStorage): diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e9fd09eaa..61b7b8f73 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -200,8 +200,7 @@ class module(generic.GenericIntelProcess): count=num_sects, ) - for attr in arr: - yield attr + yield from arr def get_elf_table_name(self): elf_table_name = intermed.IntermediateSymbolTable.create( @@ -237,8 +236,7 @@ class module(generic.GenericIntelProcess): count=self.num_symtab + 1, ) if self.section_strtab: - for sym in syms: - yield sym + yield from syms def get_symbols_names_and_addresses(self) -> Iterable[Tuple[str, int]]: """Get names and addresses for each symbol of the module @@ -2665,8 +2663,7 @@ class IDR(objects.StructType): id_storage = linux.IDStorage.choose_id_storage( self._context, kernel_module_name="kernel" ) - for page_addr in id_storage.get_entries(root=self.idr_rt): - yield page_addr + yield from id_storage.get_entries(root=self.idr_rt) def get_entries(self) -> Iterable[int]: """Walks the IDR and yield a pointer associated with each element. @@ -2684,8 +2681,7 @@ class IDR(objects.StructType): # Kernels < 4.11 get_entries_func = self._old_kernel_get_entries - for page_addr in get_entries_func(): - yield page_addr + yield from get_entries_func() class rb_root(objects.StructType): diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index c695ca77a..ee6dd10a3 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -232,10 +232,9 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "tqh_first", "tqe_next", next_member, max_elements - ): - yield element + ) @classmethod def walk_list_head( @@ -244,10 +243,9 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "lh_first", "le_next", next_member, max_elements - ): - yield element + ) @classmethod def walk_slist( @@ -256,7 +254,6 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "slh_first", "sle_next", next_member, max_elements - ): - yield element + ) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 07ab5f5a8..d40df6bd6 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -749,11 +749,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() - for entry in peb.Ldr.InLoadOrderModuleList.to_list( + yield from peb.Ldr.InLoadOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InLoadOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None @@ -762,11 +761,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() - for entry in peb.Ldr.InInitializationOrderModuleList.to_list( + yield from peb.Ldr.InInitializationOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InInitializationOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None @@ -775,11 +773,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() - for entry in peb.Ldr.InMemoryOrderModuleList.to_list( + yield from peb.Ldr.InMemoryOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InMemoryOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index cf6f43a9b..9666fd79c 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -107,11 +107,10 @@ class EXE_ALIAS_LIST(objects.StructType): def get_aliases(self) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Generator for the individual aliases for a particular executable.""" - for alias in self.AliasList.to_list( + yield from self.AliasList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_ALIAS", "ListEntry", - ): - yield alias + ) class SCREEN_INFORMATION(objects.StructType): @@ -245,11 +244,10 @@ class CONSOLE_INFORMATION(objects.StructType): def get_histories( self, ) -> Generator[interfaces.objects.ObjectInterface, None, None]: - for cmd_hist in self.HistoryList.to_list( + yield from self.HistoryList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_COMMAND_HISTORY", "ListEntry", - ): - yield cmd_hist + ) def get_exe_aliases( self, @@ -258,20 +256,18 @@ class CONSOLE_INFORMATION(objects.StructType): # Windows 10 22000 and Server 20348 made this a Pointer if isinstance(exe_alias_list, objects.Pointer): exe_alias_list = exe_alias_list.dereference() - for exe_alias_list_item in exe_alias_list.to_list( + yield from exe_alias_list.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_EXE_ALIAS_LIST", "ListEntry", - ): - yield exe_alias_list_item + ) def get_processes( self, ) -> Generator[interfaces.objects.ObjectInterface, None, None]: - for proc in self.ConsoleProcessList.to_list( + yield from self.ConsoleProcessList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_CONSOLE_PROCESS_LIST", "ListEntry", - ): - yield proc + ) def get_title(self) -> Union[str, None]: try: @@ -393,8 +389,7 @@ class COMMAND_HISTORY(objects.StructType): rest are coalesced. """ - for i, cmd in self.scan_command_bucket(self.CommandBucket.End): - yield i, cmd + yield from self.scan_command_bucket(self.CommandBucket.End) win10_x64_class_types = { diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index 4feb396e7..ea2884bb2 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -128,7 +128,10 @@ class PdbReader: self._layer_name, self._context = self.load_pdb_layer(context, location) self._dbiheader: Optional[interfaces.objects.ObjectInterface] = None if not progress_callback: - progress_callback = lambda x, y: None + + def progress_callback(x, y): + return None + self._progress_callback = progress_callback self.types: List[ Tuple[ From e708a62eefa6bc48823050850959e689f67abd9e Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 17:28:43 +0100 Subject: [PATCH 173/348] make ruff happy --- development/mac-kdk/parse_pbzx2.py | 7 ++-- development/pdbparse-to-json.py | 6 ++-- development/schema_validate.py | 4 +-- doc/source/conf.py | 12 +++---- volatility3/cli/__init__.py | 2 +- volatility3/framework/__init__.py | 16 ++++----- .../framework/configuration/__init__.py | 2 ++ volatility3/framework/constants/__init__.py | 35 ++++++++++++++++++- volatility3/framework/interfaces/__init__.py | 11 ++++++ volatility3/framework/layers/resources.py | 2 +- .../framework/layers/scanners/__init__.py | 3 ++ volatility3/framework/objects/__init__.py | 12 +++---- volatility3/framework/plugins/isfinfo.py | 9 ++--- volatility3/framework/plugins/linux/kmsg.py | 4 +-- .../framework/plugins/mac/check_sysctl.py | 2 +- .../framework/plugins/mac/kauth_scopes.py | 2 +- .../framework/plugins/windows/netscan.py | 2 +- .../framework/plugins/windows/psxview.py | 2 +- .../framework/plugins/windows/shimcachemem.py | 2 +- volatility3/framework/renderers/__init__.py | 4 +-- .../symbols/mac/extensions/__init__.py | 2 +- 21 files changed, 93 insertions(+), 48 deletions(-) diff --git a/development/mac-kdk/parse_pbzx2.py b/development/mac-kdk/parse_pbzx2.py index 173a4d648..1ca212211 100644 --- a/development/mac-kdk/parse_pbzx2.py +++ b/development/mac-kdk/parse_pbzx2.py @@ -7,6 +7,7 @@ # Cleaned up C version (as the basis for my code) here, thanks to Pepijn Bruienne / @bruienne # https://gist.github.com/bruienne/029494bbcfb358098b41 +import os import struct import sys @@ -22,7 +23,7 @@ def seekread(f, offset = None, length = 0, relative = True): def parse_pbzx(pbzx_path): section = 0 - xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) + xar_out_path = f'{pbzx_path}.part{section:02d}.cpio.xz' with open(pbzx_path, 'rb') as f: # pbzx = f.read() # f.close() @@ -50,12 +51,12 @@ def parse_pbzx(pbzx_path): # ... and split it out ... f_content = seekread(f, length = f_length) section += 1 - decomp_out = '%s.part%02d.cpio' % (pbzx_path, section) + decomp_out = f'{pbzx_path}.part{section:02d}.cpio' with open(decomp_out, 'wb') as g: g.write(f_content) # Now to start the next section, which should hopefully be .xz (we'll just assume it is ...) section += 1 - xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) + xar_out_path = f'{pbzx_path}.part{section:02d}.cpio.xz' else: f_length -= 6 # This part needs buffering diff --git a/development/pdbparse-to-json.py b/development/pdbparse-to-json.py index 6eb265227..6ffa4ea49 100644 --- a/development/pdbparse-to-json.py +++ b/development/pdbparse-to-json.py @@ -32,12 +32,12 @@ class PDBRetreiver: result = None for suffix in [file_name[:-1] + '_', file_name]: try: - logger.debug(f"Attempting to retrieve {url + suffix}") + logger.debug("Attempting to retrieve %s", url + suffix) result, _ = request.urlretrieve(url + suffix) except request.HTTPError as excp: - logger.debug(f"Failed with {excp}") + logger.debug("Failed with %s", excp) if result: - logger.debug(f"Successfully written to {result}") + logger.debug("Successfully written to %s", result) break return result diff --git a/development/schema_validate.py b/development/schema_validate.py index 031039e38..f44b3267e 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -6,7 +6,7 @@ import sys # TODO: Rather nasty hack, when volatility's actually installed this would be unnecessary sys.path += ".." -import logging +import logging # noqa: E402 console = logging.StreamHandler() console.setLevel(logging.DEBUG) @@ -17,7 +17,7 @@ logger = logging.getLogger("") logger.addHandler(console) logger.setLevel(logging.DEBUG) -from volatility3 import schemas +from volatility3 import schemas # noqa: E402 if __name__ == '__main__': parser = argparse.ArgumentParser("Validates ") diff --git a/doc/source/conf.py b/doc/source/conf.py index cabfdc327..7a9a72891 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -19,6 +19,8 @@ import sys import sphinx.ext.apidoc +from importlib.util import find_spec + def setup(app): volatility_directory = os.path.abspath( @@ -124,7 +126,7 @@ def setup(app): # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath("../..")) -from volatility3.framework import constants +from volatility3.framework import constants # noqa: E402 # -- General configuration ------------------------------------------------ @@ -147,13 +149,9 @@ extensions = [ autosectionlabel_prefix_document = True -try: - import sphinx_autodoc_typehints - +if find_spec("sphinx_autodoc_typehints") is not None: extensions.append("sphinx_autodoc_typehints") -except ImportError: - # If the autodoc typehints extension isn't available, carry on regardless - pass +# If the autodoc typehints extension isn't available, carry on regardless # Add any paths that contain templates here, relative to this directory. # templates_path = ['tools/templates'] diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index cf1335443..da046de57 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -879,7 +879,7 @@ class CommandLine: volatility3.framework.configuration.requirements.ListRequirement, ): # Allow a list of integers, specified with the convenient 0x hexadecimal format - if requirement.element_type == int: + if requirement.element_type is int: additional["type"] = lambda x: int(x, 0) else: additional["type"] = requirement.element_type diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 244b353a2..bf7d3ab74 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -22,14 +22,14 @@ if ( ) ) -import importlib -import inspect -import logging -import os -import traceback -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar +import importlib # noqa: E402 +import inspect # noqa: E402 +import logging # noqa: E402 +import os # noqa: E402 +import traceback # noqa: E402 +from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar # noqa: E402 -from volatility3.framework import constants, interfaces +from volatility3.framework import constants, interfaces # noqa: E402 # ## @@ -74,7 +74,7 @@ class NonInheritable: self.cls = cls def __get__(self, obj: Any, get_type: Type = None) -> Any: - if type == self.cls: + if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) return self.default_value diff --git a/volatility3/framework/configuration/__init__.py b/volatility3/framework/configuration/__init__.py index 7a84ee455..6ca8b4ee3 100644 --- a/volatility3/framework/configuration/__init__.py +++ b/volatility3/framework/configuration/__init__.py @@ -3,3 +3,5 @@ # from volatility3.framework.configuration import requirements + +__all__ = ["requirements"] diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 8bdf84730..427d666e0 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -14,7 +14,7 @@ import warnings from typing import Callable, Optional import volatility3.framework.constants.linux -import volatility3.framework.constants.windows +import volatility3.framework.constants.windows # noqa: F401 from volatility3.framework.constants._version import ( PACKAGE_VERSION, VERSION_MAJOR, @@ -141,3 +141,36 @@ def __getattr__(name): return globals()[f"{deprecated_tag}{name}"] return getattr(__import__(__name__), name) + + +__all__ = [ + "PACKAGE_VERSION", + "VERSION_MAJOR", + "VERSION_MINOR", + "VERSION_PATCH", + "VERSION_SUFFIX", + "PLUGINS_PATH", + "SYMBOL_BASEPATHS", + "ISF_EXTENSIONS", + "BANG", + "AUTOMAGIC_CONFIG_PATH", + "LOGLEVEL_INFO", + "LOGLEVEL_DEBUG", + "LOGLEVEL_V", + "LOGLEVEL_VV", + "LOGLEVEL_VVV", + "LOGLEVEL_VVVV", + "CACHE_PATH", + "SQLITE_CACHE_PERIOD", + "IDENTIFIERS_FILENAME", + "CACHE_SQLITE_SCHEMA_VERSION", + "BUG_URL", + "ProgressCallback", + "OS_CATEGORIES", + "Parallelism", + "PARALLELISM", + "ISF_MINIMUM_SUPPORTED", + "ISF_MINIMUM_DEPRECATED", + "OFFLINE", + "REMOTE_ISF_URL", +] diff --git a/volatility3/framework/interfaces/__init__.py b/volatility3/framework/interfaces/__init__.py index 51d81d63a..19d11e2f4 100644 --- a/volatility3/framework/interfaces/__init__.py +++ b/volatility3/framework/interfaces/__init__.py @@ -22,3 +22,14 @@ from volatility3.framework.interfaces import ( symbols, automagic, ) + +__all__ = [ + "renderers", + "configuration", + "context", + "layers", + "objects", + "plugins", + "symbols", + "automagic", +] diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index c7a7fee67..6b17db5ff 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -29,7 +29,7 @@ except ImportError: try: # Import so that the handler is found by the framework.class_subclasses callc - import smb.SMBHandler # lgtm [py/unused-import] + import smb.SMBHandler # lgtm [py/unused-import] # noqa: F401 except ImportError: # If we fail to import this, it means that SMB handling won't be available pass diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index dd8dc46be..a36236a11 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -136,3 +136,6 @@ class MultiStringScanner(layers.ScannerInterface): ) for match in re.finditer(self._regex, haystack): yield match.start(0), match.group() + + +__all__ = ["multiregexp", "BytesScanner", "RegExScanner", "MultiStringScanner"] diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index b65277067..5846da070 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -35,13 +35,13 @@ def convert_data_to_value( data_format: DataFormatInfo, ) -> TUnion[int, float, bytes, str, bool]: """Converts a series of bytes to a particular type of value.""" - if struct_type == int: + if struct_type is int: return int.from_bytes( data, byteorder=data_format.byteorder, signed=data_format.signed ) - if struct_type == bool: + if struct_type is bool: struct_format = "?" - elif struct_type == float: + elif struct_type is float: float_vals = "zzezfzzzd" if ( data_format.length > len(float_vals) @@ -70,7 +70,7 @@ def convert_value_to_data( f"Written value is not of the correct type for {struct_type.__name__}" ) - if struct_type == int and isinstance(value, int): + if struct_type is int and isinstance(value, int): # Doubling up on the isinstance is for mypy return int.to_bytes( value, @@ -78,9 +78,9 @@ def convert_value_to_data( byteorder=data_format.byteorder, signed=data_format.signed, ) - if struct_type == bool: + if struct_type is bool: struct_format = "?" - elif struct_type == float: + elif struct_type is float: float_vals = "zzezfzzzd" if ( data_format.length > len(float_vals) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 4f07bd5a8..78e78fb9e 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -7,6 +7,7 @@ import os import pathlib import zipfile from typing import Generator, List +from importlib.util import find_spec from volatility3 import schemas, symbols from volatility3.framework import constants, interfaces, renderers @@ -96,16 +97,12 @@ class IsfInfo(plugins.PluginInterface): if filter_item in isf_file: filtered_list.append(isf_file) - try: - import jsonschema - - if not self.config["validate"]: - raise ImportError # Act as if we couldn't import if validation is turned off + if find_spec("jsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" - except ImportError: + else: def check_valid(data): return "Unknown" diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index e26d69543..d66e3b9ca 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return "%lu.%06lu" % (nsec / 1000000000, (nsec % 1000000000) / 1000) + return f"{nsec / 1000000000:lu}.{(nsec % 1000000000) / 1000:06lu}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = "%s(%u)" % (caller_name, caller_id & ~0x80000000) + caller = f"{caller_name}({caller_id & ~0x80000000:u})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: diff --git a/volatility3/framework/plugins/mac/check_sysctl.py b/volatility3/framework/plugins/mac/check_sysctl.py index e8218962d..ed3e34aea 100644 --- a/volatility3/framework/plugins/mac/check_sysctl.py +++ b/volatility3/framework/plugins/mac/check_sysctl.py @@ -60,7 +60,7 @@ class Check_sysctl(plugins.PluginInterface): return var_str def _process_sysctl_list(self, kernel, sysctl_list, recursive=0): - if type(sysctl_list) == volatility3.framework.objects.Pointer: + if type(sysctl_list) is volatility3.framework.objects.Pointer: sysctl_list = sysctl_list.dereference().cast("sysctl_oid_list") sysctl = sysctl_list.slh_first diff --git a/volatility3/framework/plugins/mac/kauth_scopes.py b/volatility3/framework/plugins/mac/kauth_scopes.py index afb320a07..c2c473eac 100644 --- a/volatility3/framework/plugins/mac/kauth_scopes.py +++ b/volatility3/framework/plugins/mac/kauth_scopes.py @@ -80,7 +80,7 @@ class Kauth_scopes(interfaces.plugins.PluginInterface): ( identifier, format_hints.Hex(scope.ks_idata), - len([l for l in scope.get_listeners()]), + len([listener for listener in scope.get_listeners()]), format_hints.Hex(callback), module_name, symbol_name, diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 77bd22ab9..dd8e4b133 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -161,7 +161,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): raise NotImplementedError( "Kernel Debug Structure version format not supported!" ) - except: + except: # noqa: E722 # unsure what to raise here. Also, it might be useful to add some kind of fallback, # either to a user-provided version or to another method to determine tcpip.sys's version raise exceptions.VolatilityException( diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 053ec20d5..e3ec216dd 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -218,7 +218,7 @@ class PsXView(plugins.PluginInterface): name = self._proc_name_to_string(proc) exit_time = proc.get_exit_time() - if type(exit_time) != datetime.datetime: + if type(exit_time) is not datetime.datetime: exit_time = "" else: exit_time = str(exit_time) diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 3cf6d60d8..59f33510d 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -146,7 +146,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf context, layer_name, kernel_symbol_table ): pid = process.UniqueProcessId - vollog.debug("checking process %d" % pid) + vollog.debug("checking process %d", pid) for vad in vadinfo.VadInfo.list_vads( process, lambda x: x.get_tag() == b"Vad " and x.Protection == 4 ): diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 02805acc2..39ce1135d 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -430,10 +430,10 @@ class ColumnSortKey(interfaces.renderers.ColumnSortKey): value = datetime.datetime.min elif self._type in [int, float]: value = -1 - elif self._type == bool: + elif self._type is bool: value = False elif self._type in [str, renderers.Disassembly]: value = "-" - elif self._type == bytes: + elif self._type is bytes: value = b"" return value diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 15fe7aeda..d2573fb95 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -237,7 +237,7 @@ class vm_map_entry(objects.StructType): def get_path(self, context, config_prefix): node = self.get_vnode(context, config_prefix) - if type(node) == str and node == "sub_map": + if type(node) is str and node == "sub_map": ret = node elif node: path = [] From 75a324ddba73462d13362ab3f0f5bbdfeb0a3d33 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 18:04:22 +0100 Subject: [PATCH 174/348] chore(developement/schema_validate): move logging import up --- development/schema_validate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/development/schema_validate.py b/development/schema_validate.py index f44b3267e..0ea82d537 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -1,13 +1,12 @@ import argparse import json +import logging import os import sys # TODO: Rather nasty hack, when volatility's actually installed this would be unnecessary sys.path += ".." -import logging # noqa: E402 - console = logging.StreamHandler() console.setLevel(logging.DEBUG) formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') From 2e093c1a53520c12c209d49cb108b654a2cc2290 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 18:08:22 +0100 Subject: [PATCH 175/348] chore(framework/automagic/linux): convert lambda to function --- volatility3/framework/automagic/linux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 6b58577a3..93131a120 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -196,5 +196,8 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder): banner_config_key = "kernel_banner" operating_system = "linux" symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" - find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] exclusion_list = ["mac", "windows"] + + @classmethod + def find_aslr(cls, *args): + return LinuxIntelStacker.find_aslr(*args)[1] From c33e378c847b3cddcf3730f8b6ea5fc5efa8384e Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 18:18:54 +0100 Subject: [PATCH 176/348] chore(framework/__init__): move up imports --- volatility3/framework/__init__.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index bf7d3ab74..364e6f5ec 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -6,6 +6,14 @@ import glob import sys import zipfile +import importlib +import inspect +import logging +import os +import traceback +from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar + +from volatility3.framework import constants, interfaces required_python_version = (3, 8, 0) if ( @@ -22,15 +30,6 @@ if ( ) ) -import importlib # noqa: E402 -import inspect # noqa: E402 -import logging # noqa: E402 -import os # noqa: E402 -import traceback # noqa: E402 -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar # noqa: E402 - -from volatility3.framework import constants, interfaces # noqa: E402 - # ## # From 2119b8c7b820e3fde75f9f4871cb7504ef0b5473 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 18:37:46 +0100 Subject: [PATCH 177/348] fix framework/__init__ --- volatility3/framework/__init__.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 364e6f5ec..df8d2d2b8 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,15 +5,6 @@ # Check the python version to ensure it's suitable import glob import sys -import zipfile -import importlib -import inspect -import logging -import os -import traceback -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar - -from volatility3.framework import constants, interfaces required_python_version = (3, 8, 0) if ( @@ -25,10 +16,18 @@ if ( ) ): raise RuntimeError( - "Volatility framework requires python version {}.{}.{} or greater".format( - *required_python_version - ) + f"Volatility framework requires python version {'.'.join(map(str, required_python_version))} or greater" ) +else: + import zipfile + import importlib + import inspect + import logging + import os + import traceback + from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar + + from volatility3.framework import constants, interfaces # ## From ccd3e8f367df22d60eb5be300a0c8a830b8db59a Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 18:43:27 +0100 Subject: [PATCH 178/348] move python version check into its own module --- volatility3/framework/__init__.py | 31 ++++++------------- volatility3/framework/check_python_version.py | 14 +++++++++ 2 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 volatility3/framework/check_python_version.py diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index df8d2d2b8..bf71c3c99 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,29 +5,16 @@ # Check the python version to ensure it's suitable import glob import sys +import volatility3.framework.check_python_version # noqa: F401 +import zipfile +import importlib +import inspect +import logging +import os +import traceback +from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar -required_python_version = (3, 8, 0) -if ( - sys.version_info.major != required_python_version[0] - or sys.version_info.minor < required_python_version[1] - or ( - sys.version_info.minor == required_python_version[1] - and sys.version_info.micro < required_python_version[2] - ) -): - raise RuntimeError( - f"Volatility framework requires python version {'.'.join(map(str, required_python_version))} or greater" - ) -else: - import zipfile - import importlib - import inspect - import logging - import os - import traceback - from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar - - from volatility3.framework import constants, interfaces +from volatility3.framework import constants, interfaces # ## diff --git a/volatility3/framework/check_python_version.py b/volatility3/framework/check_python_version.py new file mode 100644 index 000000000..f2d284f2a --- /dev/null +++ b/volatility3/framework/check_python_version.py @@ -0,0 +1,14 @@ +import sys + +required_python_version = (3, 8, 0) +if ( + sys.version_info.major != required_python_version[0] + or sys.version_info.minor < required_python_version[1] + or ( + sys.version_info.minor == required_python_version[1] + and sys.version_info.micro < required_python_version[2] + ) +): + raise RuntimeError( + f"Volatility framework requires python version {required_python_version[0]}.{required_python_version[1]}.{required_python_version[2]} or greater" + ) From 53133f4478bab042d96c468d5fd14c95f96b9913 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 21:27:38 +0100 Subject: [PATCH 179/348] fix codeQL error --- volatility3/framework/plugins/windows/modules.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 00424938f..3c0f5ab7c 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -104,12 +104,11 @@ class Modules(interfaces.plugins.PluginInterface): try: BaseDllName = mod.BaseDllName.get_string() + if self.config["name"] and self.config["name"] not in BaseDllName: + continue except exceptions.InvalidAddressException: BaseDllName = interfaces.renderers.BaseAbsentValue() - if self.config["name"] and self.config["name"] not in BaseDllName: - continue - try: FullDllName = mod.FullDllName.get_string() except exceptions.InvalidAddressException: From 1c0d0b086adba32763668f2b06dd1b659163d350 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 23:07:42 +0100 Subject: [PATCH 180/348] use redundant import aliases instead of __all__ and noqa's --- volatility3/framework/__init__.py | 2 +- .../framework/configuration/__init__.py | 4 +- volatility3/framework/constants/__init__.py | 47 +++---------------- volatility3/framework/interfaces/__init__.py | 27 ++++------- .../framework/layers/scanners/__init__.py | 5 +- 5 files changed, 18 insertions(+), 67 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index bf71c3c99..b60ae5576 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,7 +5,7 @@ # Check the python version to ensure it's suitable import glob import sys -import volatility3.framework.check_python_version # noqa: F401 +from volatility3.framework import check_python_version as check_python_version import zipfile import importlib import inspect diff --git a/volatility3/framework/configuration/__init__.py b/volatility3/framework/configuration/__init__.py index 6ca8b4ee3..7b914cf16 100644 --- a/volatility3/framework/configuration/__init__.py +++ b/volatility3/framework/configuration/__init__.py @@ -2,6 +2,4 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework.configuration import requirements - -__all__ = ["requirements"] +from volatility3.framework.configuration import requirements as requirements diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 427d666e0..23cc2dde5 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -13,14 +13,14 @@ import sys import warnings from typing import Callable, Optional -import volatility3.framework.constants.linux -import volatility3.framework.constants.windows # noqa: F401 +from volatility3.framework.constants import linux as linux +from volatility3.framework.constants import windows as windows from volatility3.framework.constants._version import ( - PACKAGE_VERSION, - VERSION_MAJOR, - VERSION_MINOR, - VERSION_PATCH, - VERSION_SUFFIX, + PACKAGE_VERSION as PACKAGE_VERSION, + VERSION_MAJOR as VERSION_MAJOR, + VERSION_MINOR as VERSION_MINOR, + VERSION_PATCH as VERSION_PATCH, + VERSION_SUFFIX as VERSION_SUFFIX, ) PLUGINS_PATH = [ @@ -141,36 +141,3 @@ def __getattr__(name): return globals()[f"{deprecated_tag}{name}"] return getattr(__import__(__name__), name) - - -__all__ = [ - "PACKAGE_VERSION", - "VERSION_MAJOR", - "VERSION_MINOR", - "VERSION_PATCH", - "VERSION_SUFFIX", - "PLUGINS_PATH", - "SYMBOL_BASEPATHS", - "ISF_EXTENSIONS", - "BANG", - "AUTOMAGIC_CONFIG_PATH", - "LOGLEVEL_INFO", - "LOGLEVEL_DEBUG", - "LOGLEVEL_V", - "LOGLEVEL_VV", - "LOGLEVEL_VVV", - "LOGLEVEL_VVVV", - "CACHE_PATH", - "SQLITE_CACHE_PERIOD", - "IDENTIFIERS_FILENAME", - "CACHE_SQLITE_SCHEMA_VERSION", - "BUG_URL", - "ProgressCallback", - "OS_CATEGORIES", - "Parallelism", - "PARALLELISM", - "ISF_MINIMUM_SUPPORTED", - "ISF_MINIMUM_DEPRECATED", - "OFFLINE", - "REMOTE_ISF_URL", -] diff --git a/volatility3/framework/interfaces/__init__.py b/volatility3/framework/interfaces/__init__.py index 19d11e2f4..fd6b1e062 100644 --- a/volatility3/framework/interfaces/__init__.py +++ b/volatility3/framework/interfaces/__init__.py @@ -13,23 +13,12 @@ components of volatility to write plugins. # This will also avoid namespace issues, because people can use interfaces.layers to # avoid clashing with the layers package from volatility3.framework.interfaces import ( - renderers, - configuration, - context, - layers, - objects, - plugins, - symbols, - automagic, + renderers as renderers, + configuration as configuration, + context as context, + layers as layers, + objects as objects, + plugins as plugins, + symbols as symbols, + automagic as automagic, ) - -__all__ = [ - "renderers", - "configuration", - "context", - "layers", - "objects", - "plugins", - "symbols", - "automagic", -] diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index a36236a11..f54b44ff4 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -5,7 +5,7 @@ import re from typing import Generator, List, Tuple, Dict, Optional from volatility3.framework.interfaces import layers -from volatility3.framework.layers.scanners import multiregexp +from volatility3.framework.layers.scanners import multiregexp as multiregexp class BytesScanner(layers.ScannerInterface): @@ -136,6 +136,3 @@ class MultiStringScanner(layers.ScannerInterface): ) for match in re.finditer(self._regex, haystack): yield match.start(0), match.group() - - -__all__ = ["multiregexp", "BytesScanner", "RegExScanner", "MultiStringScanner"] From 6283ab6a1a7d871eacb1c7fc9e3d71f9333f4bab Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 11:26:25 +0100 Subject: [PATCH 181/348] adjust SMBHandler import to use redundat alias --- volatility3/framework/layers/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 6b17db5ff..dc510452a 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -29,7 +29,7 @@ except ImportError: try: # Import so that the handler is found by the framework.class_subclasses callc - import smb.SMBHandler # lgtm [py/unused-import] # noqa: F401 + from smb import SMBHandler as SMBHandler except ImportError: # If we fail to import this, it means that SMB handling won't be available pass From d95c32b25a65e87fc1a3a57f46dafa628d0b5568 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 11:27:41 +0100 Subject: [PATCH 182/348] run `ruff check --fix` --- volatility3/framework/automagic/linux.py | 2 -- volatility3/framework/automagic/mac.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 93131a120..c044fdd93 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,12 +3,10 @@ # import logging -import os from typing import Optional, Tuple, Type from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder -from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import linux diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 89dd5a187..f3679d160 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -3,13 +3,11 @@ # import logging -import os import struct from typing import Optional from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.automagic import symbol_cache, symbol_finder -from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import mac From 0b4d595bac567b478c343a9a9f18957e0270c410 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 11:35:33 +0100 Subject: [PATCH 183/348] don't use bare except in windows netscan plugin --- volatility3/framework/plugins/windows/netscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index dd8e4b133..9462df43d 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -161,7 +161,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): raise NotImplementedError( "Kernel Debug Structure version format not supported!" ) - except: # noqa: E722 + except Exception: # unsure what to raise here. Also, it might be useful to add some kind of fallback, # either to a user-provided version or to another method to determine tcpip.sys's version raise exceptions.VolatilityException( From 8dc9bd037d67d0dd3805ba3421a19d8b8a76304f Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 11:39:08 +0100 Subject: [PATCH 184/348] add fixme to windows/netscan plugin --- volatility3/framework/plugins/windows/netscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 9462df43d..162031104 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -162,7 +162,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): "Kernel Debug Structure version format not supported!" ) except Exception: - # unsure what to raise here. Also, it might be useful to add some kind of fallback, + # FIXME: unsure what to raise here. Also, it might be useful to add some kind of fallback, # either to a user-provided version or to another method to determine tcpip.sys's version raise exceptions.VolatilityException( "Kernel Debug Structure missing VERSION/KUSER structure, unable to determine Windows version!" From eee015033ac3a7b17742ea149d3f3e01b9a7698e Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 12:30:44 +0100 Subject: [PATCH 185/348] add back lgtm imperative --- volatility3/framework/layers/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index dc510452a..00215f624 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -29,7 +29,7 @@ except ImportError: try: # Import so that the handler is found by the framework.class_subclasses callc - from smb import SMBHandler as SMBHandler + from smb import SMBHandler as SMBHandler # lgtm [py/unused-import] except ImportError: # If we fail to import this, it means that SMB handling won't be available pass From f6c852c478e5bf676cbc3def50900f8af66679b5 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 13:04:13 +0100 Subject: [PATCH 186/348] format with black --- development/banner_server.py | 45 ++- development/compare-vol.py | 340 ++++++++++++------- development/mac-kdk/parse_pbzx2.py | 50 +-- development/pdbparse-to-json.py | 188 ++++++---- development/schema_validate.py | 8 +- development/stock-linux-json.py | 98 +++--- test/plugins/windows/test_scheduled_tasks.py | 2 + volatility3/framework/layers/resources.py | 2 +- 8 files changed, 462 insertions(+), 271 deletions(-) diff --git a/development/banner_server.py b/development/banner_server.py index 3aea41c82..b62477a26 100644 --- a/development/banner_server.py +++ b/development/banner_server.py @@ -28,10 +28,10 @@ class BannerCacheGenerator: def run(self): context = contexts.Context() - json_output = {'version': 1} + json_output = {"version": 1} path = self._path - filename = '*' + filename = "*" for banner_cache in [linux.LinuxBannerCache, mac.MacBannerCache]: sub_path = banner_cache.os @@ -39,37 +39,54 @@ class BannerCacheGenerator: for extension in constants.ISF_EXTENSIONS: # Hopefully these will not be large lists, otherwise this might be slow try: - for found in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + extension): + for found in ( + pathlib.Path(path) + .joinpath(sub_path) + .resolve() + .rglob(filename + extension) + ): potentials.append(found.as_uri()) except FileNotFoundError: # If there's no linux symbols, don't cry about it pass - new_banners = banner_cache.read_new_banners(context, 'BannerServer', potentials, banner_cache.symbol_name, - banner_cache.os, progress_callback = PrintedProgress()) + new_banners = banner_cache.read_new_banners( + context, + "BannerServer", + potentials, + banner_cache.symbol_name, + banner_cache.os, + progress_callback=PrintedProgress(), + ) result_banners = {} for new_banner in new_banners: # Only accept file schemes - value = [self.convert_url(url) for url in new_banners[new_banner] if - urllib.parse.urlparse(url).scheme == 'file'] + value = [ + self.convert_url(url) + for url in new_banners[new_banner] + if urllib.parse.urlparse(url).scheme == "file" + ] if value and new_banner: # Convert files into URLs - result_banners[str(base64.b64encode(new_banner), 'latin-1')] = value + result_banners[str(base64.b64encode(new_banner), "latin-1")] = value json_output[banner_cache.os] = result_banners - output_path = os.path.join(self._path, 'banners.json') - with open(output_path, 'w') as fp: + output_path = os.path.join(self._path, "banners.json") + with open(output_path, "w") as fp: vollog.warning(f"Banners file written to {output_path}") json.dump(json_output, fp) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--path', default = os.path.dirname(__file__)) - parser.add_argument('--urlprefix', help = 'Web prefix that will eventually serve the ISF files', - default = 'http://localhost/symbols') + parser.add_argument("--path", default=os.path.dirname(__file__)) + parser.add_argument( + "--urlprefix", + help="Web prefix that will eventually serve the ISF files", + default="http://localhost/symbols", + ) args = parser.parse_args() diff --git a/development/compare-vol.py b/development/compare-vol.py index 1074c5d9c..a01d8e93c 100644 --- a/development/compare-vol.py +++ b/development/compare-vol.py @@ -15,17 +15,17 @@ class VolatilityImage: filepath: str = "" vol2_profile: str = "" vol2_imageinfo_time: float = None - vol2_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) - vol3_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) - rekall_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) + vol2_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) + vol3_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) + rekall_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) @dataclass class VolatilityPlugin: name: str = "" - vol2_plugin_parameters: List[str] = field(default_factory = list) - vol3_plugin_parameters: List[str] = field(default_factory = list) - rekall_plugin_parameters: List[str] = field(default_factory = list) + vol2_plugin_parameters: List[str] = field(default_factory=list) + vol3_plugin_parameters: List[str] = field(default_factory=list) + rekall_plugin_parameters: List[str] = field(default_factory=list) class VolatilityTest: @@ -39,32 +39,50 @@ class VolatilityTest: def result_titles(self) -> List[str]: return [self.long_name] - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> None: + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> None: pass - def create_results(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> List[float]: + def create_results( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> List[float]: self.create_prerequisites(plugin, image, image_hash) # Volatility 2 Test - print(f"[*] Testing {self.short_name} {plugin.name} with image {image.filepath}") + print( + f"[*] Testing {self.short_name} {plugin.name} with image {image.filepath}" + ) os.chdir(self.path) cmd = self.plugin_cmd(plugin, image) start_time = time.perf_counter() try: - completed = subprocess.run(cmd, cwd = self.path, capture_output = True, timeout = 420) + completed = subprocess.run( + cmd, cwd=self.path, capture_output=True, timeout=420 + ) except subprocess.TimeoutExpired as excp: completed = excp end_time = time.perf_counter() total_time = end_time - start_time - print(f" Tested {self.short_name} {plugin.name} with image {image.filepath}: {total_time}") + print( + f" Tested {self.short_name} {plugin.name} with image {image.filepath}: {total_time}" + ) with open( - os.path.join(self.output_directory, f'{self.short_name}_{plugin.name}_{image_hash}_stdout'), - "wb") as f: + os.path.join( + self.output_directory, + f"{self.short_name}_{plugin.name}_{image_hash}_stdout", + ), + "wb", + ) as f: f.write(completed.stdout) if completed.stderr: with open( - os.path.join(self.output_directory, f'{self.short_name}_{plugin.name}_{image_hash}_stderr'), - "wb") as f: + os.path.join( + self.output_directory, + f"{self.short_name}_{plugin.name}_{image_hash}_stderr", + ), + "wb", + ) as f: f.write(completed.stderr) return [total_time] @@ -77,31 +95,57 @@ class Volatility2Test(VolatilityTest): long_name = "Volatility 2" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage): - return ["python2", "-u", "vol.py", "-f", image.filepath, "--profile", image.vol2_profile - ] + plugin.vol2_plugin_parameters + image.vol2_plugin_parameters.get(plugin.name, []) + return ( + [ + "python2", + "-u", + "vol.py", + "-f", + image.filepath, + "--profile", + image.vol2_profile, + ] + + plugin.vol2_plugin_parameters + + image.vol2_plugin_parameters.get(plugin.name, []) + ) def result_titles(self): return [self.long_name, "Imageinfo", f"{self.long_name} + Imageinfo"] - def create_results(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash) -> List[float]: + def create_results( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash + ) -> List[float]: result = super().create_results(plugin, image, image_hash) result += [image.vol2_imageinfo_time, result[0] + image.vol2_imageinfo_time] return result - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash): + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash + ): # Volatility 2 image info if not image.vol2_profile: - print(f"[*] Testing {self.short_name} imageinfo with image {image.filepath}") + print( + f"[*] Testing {self.short_name} imageinfo with image {image.filepath}" + ) os.chdir(self.path) cmd = ["python2", "-u", "vol.py", "-f", image.filepath, "imageinfo"] start_time = time.perf_counter() - vol2_completed = subprocess.run(cmd, cwd = self.path, capture_output = True) + vol2_completed = subprocess.run(cmd, cwd=self.path, capture_output=True) end_time = time.perf_counter() image.vol2_imageinfo_time = end_time - start_time - print(f" Tested volatility2 imageinfo with image {image.filepath}: {end_time - start_time}") - with open(os.path.join(self.output_directory, f'vol2_imageinfo_{image_hash}_stdout'), "wb") as f: + print( + f" Tested volatility2 imageinfo with image {image.filepath}: {end_time - start_time}" + ) + with open( + os.path.join( + self.output_directory, f"vol2_imageinfo_{image_hash}_stdout" + ), + "wb", + ) as f: f.write(vol2_completed.stdout) - image.vol2_profile = re.search(rb"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout)[1] + image.vol2_profile = re.search( + rb"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout + )[1] class RekallTest(VolatilityTest): @@ -113,11 +157,16 @@ class RekallTest(VolatilityTest): plugin.rekall_plugin_parameters = plugin.vol2_plugin_parameters if not image.rekall_plugin_parameters: image.rekall_plugin_parameters = image.vol2_plugin_parameters - return ["rekall", "-f", image.filepath] + plugin.rekall_plugin_parameters + image.rekall_plugin_parameters.get( - plugin.name, []) + return ( + ["rekall", "-f", image.filepath] + + plugin.rekall_plugin_parameters + + image.rekall_plugin_parameters.get(plugin.name, []) + ) - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> None: - shutil.rmtree('/home/mike/.rekall_cache/sessions') + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> None: + shutil.rmtree("/home/mike/.rekall_cache/sessions") class Volatility3Test(VolatilityTest): @@ -125,14 +174,18 @@ class Volatility3Test(VolatilityTest): long_name = "Volatility 3" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage) -> List[str]: - return [ - "python", - "-u", - "vol.py", - "-q", - "-f", - image.filepath, - ] + plugin.vol3_plugin_parameters + image.vol3_plugin_parameters.get(plugin.name, []) + return ( + [ + "python", + "-u", + "vol.py", + "-q", + "-f", + image.filepath, + ] + + plugin.vol3_plugin_parameters + + image.vol3_plugin_parameters.get(plugin.name, []) + ) class Volatility3PyPyTest(VolatilityTest): @@ -140,26 +193,32 @@ class Volatility3PyPyTest(VolatilityTest): long_name = "Volatility 3 (PyPy)" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage) -> List[str]: - return [ - "pypy3", - "-u", - "vol.py", - "-q", - "-f", - image.filepath, - ] + plugin.vol3_plugin_parameters + image.vol3_plugin_parameters.get(plugin.name, []) + return ( + [ + "pypy3", + "-u", + "vol.py", + "-q", + "-f", + image.filepath, + ] + + plugin.vol3_plugin_parameters + + image.vol3_plugin_parameters.get(plugin.name, []) + ) class VolatilityTester: - def __init__(self, - images: List[VolatilityImage], - plugins: List[VolatilityPlugin], - frameworks: List[str], - output_dir: str, - vol2_path: str = None, - vol3_path: str = None, - rekall_path = None): + def __init__( + self, + images: List[VolatilityImage], + plugins: List[VolatilityPlugin], + frameworks: List[str], + output_dir: str, + vol2_path: str = None, + vol3_path: str = None, + rekall_path=None, + ): self.images = images self.plugins = plugins if not vol2_path: @@ -172,7 +231,7 @@ class VolatilityTester: Volatility3Test(vol3_path, output_dir), Volatility3PyPyTest(vol3_path, output_dir), Volatility2Test(vol2_path, output_dir), - RekallTest(rekall_path, output_dir) + RekallTest(rekall_path, output_dir), ] self.tests = [x for x in available_tests if x.short_name.lower() in frameworks] self.csv_writer = None @@ -183,7 +242,7 @@ class VolatilityTester: print(f"[?] Frameworks: {[x.long_name for x in self.tests]}") def run_tests(self): - with open("volatility-timings.csv", 'w') as csvfile: + with open("volatility-timings.csv", "w") as csvfile: self.csv_writer = csv.writer(csvfile) titles = ["Image Hash", "Image Path", "Plugin Name"] for test in self.tests: @@ -203,72 +262,121 @@ class VolatilityTester: self.csv_writer.writerow([image_hash, image.filepath, plugin.name] + results) -if __name__ == '__main__': +if __name__ == "__main__": plugins = [ - VolatilityPlugin(name = "pslist", - vol2_plugin_parameters = ["pslist"], - vol3_plugin_parameters = ["windows.pslist"]), - VolatilityPlugin(name = "psscan", - vol2_plugin_parameters = ["psscan"], - vol3_plugin_parameters = ["windows.psscan"], - rekall_plugin_parameters = ["psscan", "--scan_kernel"]), - VolatilityPlugin(name = "driverscan", - vol2_plugin_parameters = ["driverscan"], - vol3_plugin_parameters = ["windows.driverscan"], - rekall_plugin_parameters = ["driverscan", "--scan_kernel"]), - VolatilityPlugin(name = "handles", - vol2_plugin_parameters = ["handles"], - vol3_plugin_parameters = ["windows.handles"]), - VolatilityPlugin(name = "modules", - vol2_plugin_parameters = ["modules"], - vol3_plugin_parameters = ["windows.modules"]), - VolatilityPlugin(name = "hivelist", - vol2_plugin_parameters = ["hivelist"], - vol3_plugin_parameters = ["registry.hivelist"], - rekall_plugin_parameters = ["hives"]), - VolatilityPlugin(name = "vadinfo", - vol2_plugin_parameters = ["vadinfo"], - vol3_plugin_parameters = ["windows.vadinfo"], - rekall_plugin_parameters = ["vad"]), - VolatilityPlugin(name = "modscan", - vol2_plugin_parameters = ["modscan"], - vol3_plugin_parameters = ["windows.modscan"], - rekall_plugin_parameters = ["modscan", "--scan_kernel"]), - VolatilityPlugin(name = "svcscan", - vol2_plugin_parameters = ["svcscan"], - vol3_plugin_parameters = ["windows.svcscan"], - rekall_plugin_parameters = ["svcscan"]), - VolatilityPlugin(name = "ssdt", vol2_plugin_parameters = ["ssdt"], vol3_plugin_parameters = ["windows.ssdt"]), - VolatilityPlugin(name = "printkey", - vol2_plugin_parameters = ["printkey", "-K", "Classes"], - vol3_plugin_parameters = ["registry.printkey", "--key", "Classes"], - rekall_plugin_parameters = ["printkey", "--key", "Classes"]) + VolatilityPlugin( + name="pslist", + vol2_plugin_parameters=["pslist"], + vol3_plugin_parameters=["windows.pslist"], + ), + VolatilityPlugin( + name="psscan", + vol2_plugin_parameters=["psscan"], + vol3_plugin_parameters=["windows.psscan"], + rekall_plugin_parameters=["psscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="driverscan", + vol2_plugin_parameters=["driverscan"], + vol3_plugin_parameters=["windows.driverscan"], + rekall_plugin_parameters=["driverscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="handles", + vol2_plugin_parameters=["handles"], + vol3_plugin_parameters=["windows.handles"], + ), + VolatilityPlugin( + name="modules", + vol2_plugin_parameters=["modules"], + vol3_plugin_parameters=["windows.modules"], + ), + VolatilityPlugin( + name="hivelist", + vol2_plugin_parameters=["hivelist"], + vol3_plugin_parameters=["registry.hivelist"], + rekall_plugin_parameters=["hives"], + ), + VolatilityPlugin( + name="vadinfo", + vol2_plugin_parameters=["vadinfo"], + vol3_plugin_parameters=["windows.vadinfo"], + rekall_plugin_parameters=["vad"], + ), + VolatilityPlugin( + name="modscan", + vol2_plugin_parameters=["modscan"], + vol3_plugin_parameters=["windows.modscan"], + rekall_plugin_parameters=["modscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="svcscan", + vol2_plugin_parameters=["svcscan"], + vol3_plugin_parameters=["windows.svcscan"], + rekall_plugin_parameters=["svcscan"], + ), + VolatilityPlugin( + name="ssdt", + vol2_plugin_parameters=["ssdt"], + vol3_plugin_parameters=["windows.ssdt"], + ), + VolatilityPlugin( + name="printkey", + vol2_plugin_parameters=["printkey", "-K", "Classes"], + vol3_plugin_parameters=["registry.printkey", "--key", "Classes"], + rekall_plugin_parameters=["printkey", "--key", "Classes"], + ), ] parser = argparse.ArgumentParser() - parser.add_argument("--output-dir", type = str, default = os.getcwd(), help = "Directory to store all results") - parser.add_argument("--vol3path", - type = str, - default = os.path.join(os.getcwd(), 'volatility3'), - help = "Path ot the volatility 3 directory") - parser.add_argument("--vol2path", - type = str, - default = os.path.join(os.getcwd(), 'volatility'), - help = "Path to the volatility 2 directory") - parser.add_argument("--rekallpath", - type = str, - default = os.path.join(os.getcwd(), 'rekall'), - help = "Path to the rekall directory") - parser.add_argument("--frameworks", - nargs = "+", - type = str, - choices = [x.short_name.lower() for x in VolatilityTest.__subclasses__()], - default = [x.short_name.lower() for x in VolatilityTest.__subclasses__()], - help = "A comma separated list of frameworks to test") - parser.add_argument('images', metavar = 'IMAGE', type = str, nargs = '+', help = 'The list of images to compare') + parser.add_argument( + "--output-dir", + type=str, + default=os.getcwd(), + help="Directory to store all results", + ) + parser.add_argument( + "--vol3path", + type=str, + default=os.path.join(os.getcwd(), "volatility3"), + help="Path ot the volatility 3 directory", + ) + parser.add_argument( + "--vol2path", + type=str, + default=os.path.join(os.getcwd(), "volatility"), + help="Path to the volatility 2 directory", + ) + parser.add_argument( + "--rekallpath", + type=str, + default=os.path.join(os.getcwd(), "rekall"), + help="Path to the rekall directory", + ) + parser.add_argument( + "--frameworks", + nargs="+", + type=str, + choices=[x.short_name.lower() for x in VolatilityTest.__subclasses__()], + default=[x.short_name.lower() for x in VolatilityTest.__subclasses__()], + help="A comma separated list of frameworks to test", + ) + parser.add_argument( + "images", + metavar="IMAGE", + type=str, + nargs="+", + help="The list of images to compare", + ) args = parser.parse_args() - vt = VolatilityTester([VolatilityImage(filepath = x) for x in args.images], plugins, - [x.lower() for x in args.frameworks], args.output_dir, args.vol2path, args.vol3path, - args.rekallpath) + vt = VolatilityTester( + [VolatilityImage(filepath=x) for x in args.images], + plugins, + [x.lower() for x in args.frameworks], + args.output_dir, + args.vol2path, + args.vol3path, + args.rekallpath, + ) vt.run_tests() diff --git a/development/mac-kdk/parse_pbzx2.py b/development/mac-kdk/parse_pbzx2.py index 1ca212211..b175539b3 100644 --- a/development/mac-kdk/parse_pbzx2.py +++ b/development/mac-kdk/parse_pbzx2.py @@ -12,7 +12,7 @@ import struct import sys -def seekread(f, offset = None, length = 0, relative = True): +def seekread(f, offset=None, length=0, relative=True): if offset is not None: # offset provided, let's seek f.seek(offset, [0, 1, 2][relative]) @@ -23,55 +23,57 @@ def seekread(f, offset = None, length = 0, relative = True): def parse_pbzx(pbzx_path): section = 0 - xar_out_path = f'{pbzx_path}.part{section:02d}.cpio.xz' - with open(pbzx_path, 'rb') as f: + xar_out_path = f"{pbzx_path}.part{section:02d}.cpio.xz" + with open(pbzx_path, "rb") as f: # pbzx = f.read() # f.close() - magic = seekread(f, length = 4) - if magic != 'pbzx': + magic = seekread(f, length=4) + if magic != "pbzx": raise RuntimeError("Error: Not a pbzx file") # Read 8 bytes for initial flags - flags = seekread(f, length = 8) + flags = seekread(f, length=8) # Interpret the flags as a 64-bit big-endian unsigned int - flags = struct.unpack('>Q', flags)[0] + flags = struct.unpack(">Q", flags)[0] while flags & (1 << 24): - with open(xar_out_path, 'wb') as xar_f: + with open(xar_out_path, "wb") as xar_f: xar_f.seek(0, os.SEEK_END) # Read in more flags - flags = seekread(f, length = 8) - flags = struct.unpack('>Q', flags)[0] + flags = seekread(f, length=8) + flags = struct.unpack(">Q", flags)[0] # Read in length - f_length = seekread(f, length = 8) - f_length = struct.unpack('>Q', f_length)[0] - xzmagic = seekread(f, length = 6) - if xzmagic != '\xfd7zXZ\x00': + f_length = seekread(f, length=8) + f_length = struct.unpack(">Q", f_length)[0] + xzmagic = seekread(f, length=6) + if xzmagic != "\xfd7zXZ\x00": # This isn't xz content, this is actually _raw decompressed cpio_ chunk of 16MB in size... # Let's back up ... - seekread(f, offset = -6, length = 0) + seekread(f, offset=-6, length=0) # ... and split it out ... - f_content = seekread(f, length = f_length) + f_content = seekread(f, length=f_length) section += 1 - decomp_out = f'{pbzx_path}.part{section:02d}.cpio' - with open(decomp_out, 'wb') as g: + decomp_out = f"{pbzx_path}.part{section:02d}.cpio" + with open(decomp_out, "wb") as g: g.write(f_content) # Now to start the next section, which should hopefully be .xz (we'll just assume it is ...) section += 1 - xar_out_path = f'{pbzx_path}.part{section:02d}.cpio.xz' + xar_out_path = f"{pbzx_path}.part{section:02d}.cpio.xz" else: f_length -= 6 # This part needs buffering - f_content = seekread(f, length = f_length) - tail = seekread(f, offset = -2, length = 2) + f_content = seekread(f, length=f_length) + tail = seekread(f, offset=-2, length=2) xar_f.write(xzmagic) xar_f.write(f_content) - if tail != 'YZ': + if tail != "YZ": raise RuntimeError("Error: Footer is not xar file footer") def main(): parse_pbzx(sys.argv[1]) - print("Now xz decompress the .xz chunks, then 'cat' them all together in order into a single new.cpio file") + print( + "Now xz decompress the .xz chunks, then 'cat' them all together in order into a single new.cpio file" + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/development/pdbparse-to-json.py b/development/pdbparse-to-json.py index 6ffa4ea49..49b4da009 100644 --- a/development/pdbparse-to-json.py +++ b/development/pdbparse-to-json.py @@ -13,10 +13,10 @@ import pdbparse.undecorate logger = logging.getLogger(__name__) logger.setLevel(1) -if __name__ == '__main__': +if __name__ == "__main__": console = logging.StreamHandler() console.setLevel(1) - formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') + formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) logger.addHandler(console) @@ -25,12 +25,12 @@ class PDBRetreiver: def retreive_pdb(self, guid: str, file_name: str) -> Optional[str]: logger.info("Download PDB file...") - file_name = ".".join(file_name.split(".")[:-1] + ['pdb']) - for sym_url in ['http://msdl.microsoft.com/download/symbols']: + file_name = ".".join(file_name.split(".")[:-1] + ["pdb"]) + for sym_url in ["http://msdl.microsoft.com/download/symbols"]: url = sym_url + f"/{file_name}/{guid}/" result = None - for suffix in [file_name[:-1] + '_', file_name]: + for suffix in [file_name[:-1] + "_", file_name]: try: logger.debug("Attempting to retrieve %s", url + suffix) result, _ = request.urlretrieve(url + suffix) @@ -69,7 +69,7 @@ class PDBConvertor: "float": "float", "double": "float", "long double": "float", - "void": "void" + "void": "void", } base_type_size = { @@ -122,13 +122,18 @@ class PDBConvertor: self._seen_ctypes.add(ctype) return self.ctype[ctype] - def lookup_ctype_pointers(self, ctype_pointer: str) -> Dict[str, Union[str, Dict[str, str]]]: - base_type = ctype_pointer.replace('32P', '').replace('64P', '') + def lookup_ctype_pointers( + self, ctype_pointer: str + ) -> Dict[str, Union[str, Dict[str, str]]]: + base_type = ctype_pointer.replace("32P", "").replace("64P", "") if base_type == ctype_pointer: # We raise a KeyError, because we've been asked about a type that isn't a pointer raise KeyError self._seen_ctypes.add(base_type) - return {"kind": "pointer", "subtype": {"kind": "base", "name": self.ctype[base_type]}} + return { + "kind": "pointer", + "subtype": {"kind": "base", "name": self.ctype[base_type]}, + } def read_pdb(self) -> Dict: """Reads in the PDB file and forms essentially a python dictionary of necessary data""" @@ -137,31 +142,31 @@ class PDBConvertor: "enums": self.read_enums(), "metadata": self.generate_metadata(), "symbols": self.read_symbols(), - "base_types": self.read_basetypes() + "base_types": self.read_basetypes(), } return output def generate_metadata(self) -> Dict[str, Any]: """Generates the metadata necessary for this object""" dbg = self._pdb.STREAM_DBI - last_bytes = str(binascii.hexlify(self._pdb.STREAM_PDB.GUID.Data4), 'ascii')[-16:] - guidstr = f'{self._pdb.STREAM_PDB.GUID.Data1:08x}{self._pdb.STREAM_PDB.GUID.Data2:04x}{self._pdb.STREAM_PDB.GUID.Data3:04x}{last_bytes}' + last_bytes = str(binascii.hexlify(self._pdb.STREAM_PDB.GUID.Data4), "ascii")[ + -16: + ] + guidstr = f"{self._pdb.STREAM_PDB.GUID.Data1:08x}{self._pdb.STREAM_PDB.GUID.Data2:04x}{self._pdb.STREAM_PDB.GUID.Data3:04x}{last_bytes}" pdb_data = { "GUID": guidstr.upper(), "age": self._pdb.STREAM_PDB.Age, "database": "ntkrnlmp.pdb", - "machine_type": int(dbg.machine) + "machine_type": int(dbg.machine), } result = { "format": "6.0.0", "producer": { "datetime": datetime.datetime.now().isoformat(), "name": "pdbconv", - "version": "0.1.0" + "version": "0.1.0", }, - "windows": { - "pdb": pdb_data - } + "windows": {"pdb": pdb_data}, } return result @@ -172,16 +177,21 @@ class PDBConvertor: stream = self._pdb.STREAM_TPI for type_index in stream.types: user_type = stream.types[type_index] - if (user_type.leaf_type == "LF_ENUM" and not user_type.prop.fwdref): + if user_type.leaf_type == "LF_ENUM" and not user_type.prop.fwdref: output.update(self._format_enum(user_type)) return output def _format_enum(self, user_enum): output = { user_enum.name: { - 'base': self.lookup_ctype(user_enum.utype), - 'size': self._determine_size(user_enum.utype), - 'constants': dict([(enum.name, enum.enum_value) for enum in user_enum.fieldlist.substructs]) + "base": self.lookup_ctype(user_enum.utype), + "size": self._determine_size(user_enum.utype), + "constants": dict( + [ + (enum.name, enum.enum_value) + for enum in user_enum.fieldlist.substructs + ] + ), } } return output @@ -201,7 +211,7 @@ class PDBConvertor: omap = None for sym in self._pdb.STREAM_GSYM.globals: - if not hasattr(sym, 'offset'): + if not hasattr(sym, "offset"): continue try: virt_base = sects[sym.segment - 1].VirtualAddress @@ -222,9 +232,9 @@ class PDBConvertor: stream = self._pdb.STREAM_TPI for type_index in stream.types: user_type = stream.types[type_index] - if (user_type.leaf_type == "LF_STRUCTURE" and not user_type.prop.fwdref): + if user_type.leaf_type == "LF_STRUCTURE" and not user_type.prop.fwdref: output.update(self._format_usertype(user_type, "struct")) - elif (user_type.leaf_type == "LF_UNION" and not user_type.prop.fwdref): + elif user_type.leaf_type == "LF_UNION" and not user_type.prop.fwdref: output.update(self._format_usertype(user_type, "union")) return output @@ -232,16 +242,22 @@ class PDBConvertor: """Produces a single usertype""" fields: Dict[str, Dict[str, Any]] = {} [fields.update(self._format_field(s)) for s in usertype.fieldlist.substructs] - return {usertype.name: {'fields': fields, 'kind': kind, 'size': usertype.size}} + return {usertype.name: {"fields": fields, "kind": kind, "size": usertype.size}} def _format_field(self, field) -> Dict[str, Dict[str, Any]]: - return {field.name: {"offset": field.offset, "type": self._format_kind(field.index)}} + return { + field.name: {"offset": field.offset, "type": self._format_kind(field.index)} + } def _determine_size(self, field): output = None if isinstance(field, str): output = self.base_type_size[field] - elif (field.leaf_type == "LF_STRUCTURE" or field.leaf_type == "LF_ARRAY" or field.leaf_type == "LF_UNION"): + elif ( + field.leaf_type == "LF_STRUCTURE" + or field.leaf_type == "LF_ARRAY" + or field.leaf_type == "LF_UNION" + ): output = field.size elif field.leaf_type == "LF_POINTER": output = self.base_type_size[field.ptr_attr.type] @@ -255,6 +271,7 @@ class PDBConvertor: output = self._determine_size(field.index) if output is None: import pdb + pdb.set_trace() raise ValueError(f"Unknown size for field: {field.name}") return output @@ -266,36 +283,37 @@ class PDBConvertor: output = self.lookup_ctype_pointers(kind) except KeyError: try: - output = {'kind': 'base', 'name': self.lookup_ctype(kind)} + output = {"kind": "base", "name": self.lookup_ctype(kind)} except KeyError: - output = {'kind': 'base', 'name': kind} - elif kind.leaf_type == 'LF_MODIFIER': + output = {"kind": "base", "name": kind} + elif kind.leaf_type == "LF_MODIFIER": output = self._format_kind(kind.modified_type) - elif kind.leaf_type == 'LF_STRUCTURE': - output = {'kind': 'struct', 'name': kind.name} - elif kind.leaf_type == 'LF_UNION': - output = {'kind': 'union', 'name': kind.name} - elif kind.leaf_type == 'LF_BITFIELD': + elif kind.leaf_type == "LF_STRUCTURE": + output = {"kind": "struct", "name": kind.name} + elif kind.leaf_type == "LF_UNION": + output = {"kind": "union", "name": kind.name} + elif kind.leaf_type == "LF_BITFIELD": output = { - 'kind': 'bitfield', - 'type': self._format_kind(kind.base_type), - 'bit_length': kind.length, - 'bit_position': kind.position + "kind": "bitfield", + "type": self._format_kind(kind.base_type), + "bit_length": kind.length, + "bit_position": kind.position, } - elif kind.leaf_type == 'LF_POINTER': - output = {'kind': 'pointer', 'subtype': self._format_kind(kind.utype)} - elif kind.leaf_type == 'LF_ARRAY': + elif kind.leaf_type == "LF_POINTER": + output = {"kind": "pointer", "subtype": self._format_kind(kind.utype)} + elif kind.leaf_type == "LF_ARRAY": output = { - 'kind': 'array', - 'count': kind.size // self._determine_size(kind.element_type), - 'subtype': self._format_kind(kind.element_type) + "kind": "array", + "count": kind.size // self._determine_size(kind.element_type), + "subtype": self._format_kind(kind.element_type), } - elif kind.leaf_type == 'LF_ENUM': - output = {'kind': 'enum', 'name': kind.name} - elif kind.leaf_type == 'LF_PROCEDURE': - output = {'kind': "function"} + elif kind.leaf_type == "LF_ENUM": + output = {"kind": "enum", "name": kind.name} + elif kind.leaf_type == "LF_PROCEDURE": + output = {"kind": "function"} else: import pdb + pdb.set_trace() return output @@ -305,40 +323,70 @@ class PDBConvertor: if "64" in self._pdb.STREAM_DBI.machine: ptr_size = 8 - output = {"pointer": {"endian": "little", "kind": "int", "signed": False, "size": ptr_size}} + output = { + "pointer": { + "endian": "little", + "kind": "int", + "signed": False, + "size": ptr_size, + } + } for index in self._seen_ctypes: output[self.ctype[index]] = { "endian": "little", "kind": self.ctype_python_types.get(self.ctype[index], "int"), "signed": False if "_U" in index else True, - "size": self.base_type_size[index] + "size": self.base_type_size[index], } return output -if __name__ == '__main__': - parser = argparse.ArgumentParser(description = "Convertor for PDB files to Volatility 3 Intermediate Symbol Format") - parser.add_argument("-o", "--output", metavar = "OUTPUT", help = "Filename for data output", required = True) - file_group = parser.add_argument_group("file", description = "File-based conversion of PDB to ISF") - file_group.add_argument("-f", "--file", metavar = "FILE", help = "PDB file to translate to ISF") - data_group = parser.add_argument_group("data", description = "Convert based on a GUID and filename pattern") - data_group.add_argument("-p", "--pattern", metavar = "PATTERN", help = "Filename pattern to recover PDB file") - data_group.add_argument("-g", - "--guid", - metavar = "GUID", - help = "GUID + Age string for the required PDB file", - default = None) - data_group.add_argument("-k", - "--keep", - action = "store_true", - default = False, - help = "Keep the downloaded PDB file") +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Convertor for PDB files to Volatility 3 Intermediate Symbol Format" + ) + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT", + help="Filename for data output", + required=True, + ) + file_group = parser.add_argument_group( + "file", description="File-based conversion of PDB to ISF" + ) + file_group.add_argument( + "-f", "--file", metavar="FILE", help="PDB file to translate to ISF" + ) + data_group = parser.add_argument_group( + "data", description="Convert based on a GUID and filename pattern" + ) + data_group.add_argument( + "-p", + "--pattern", + metavar="PATTERN", + help="Filename pattern to recover PDB file", + ) + data_group.add_argument( + "-g", + "--guid", + metavar="GUID", + help="GUID + Age string for the required PDB file", + default=None, + ) + data_group.add_argument( + "-k", + "--keep", + action="store_true", + default=False, + help="Keep the downloaded PDB file", + ) args = parser.parse_args() delfile = False filename = None if args.guid is not None and args.pattern is not None: - filename = PDBRetreiver().retreive_pdb(guid = args.guid, file_name = args.pattern) + filename = PDBRetreiver().retreive_pdb(guid=args.guid, file_name=args.pattern) delfile = True elif args.file: filename = args.file @@ -351,7 +399,7 @@ if __name__ == '__main__': convertor = PDBConvertor(filename) with open(args.output, "w") as f: - json.dump(convertor.read_pdb(), f, indent = 2, sort_keys = True) + json.dump(convertor.read_pdb(), f, indent=2, sort_keys=True) if args.keep: print(f"Temporary PDB file: {filename}") diff --git a/development/schema_validate.py b/development/schema_validate.py index 0ea82d537..cf9565d68 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -9,7 +9,7 @@ sys.path += ".." console = logging.StreamHandler() console.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') +formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) logger = logging.getLogger("") @@ -18,10 +18,10 @@ logger.setLevel(logging.DEBUG) from volatility3 import schemas # noqa: E402 -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser("Validates ") - parser.add_argument("-s", "--schema", dest = "schema", default = None) - parser.add_argument("filenames", metavar = "FILE", nargs = '+') + parser.add_argument("-s", "--schema", dest="schema", default=None) + parser.add_argument("filenames", metavar="FILE", nargs="+") args = parser.parse_args() diff --git a/development/stock-linux-json.py b/development/stock-linux-json.py index 877f78e1c..967cc7e18 100644 --- a/development/stock-linux-json.py +++ b/development/stock-linux-json.py @@ -9,7 +9,7 @@ import requests import rpmfile from debian import debfile -DWARF2JSON = './dwarf2json' +DWARF2JSON = "./dwarf2json" class Downloader: @@ -17,7 +17,7 @@ class Downloader: def __init__(self, url_lists: List[List[str]]) -> None: self.url_lists = url_lists - def download_lists(self, keep = False): + def download_lists(self, keep=False): for url_list in self.url_lists: print("Downloading files...") files_for_processing = self.download_list(url_list) @@ -35,43 +35,45 @@ class Downloader: with tempfile.NamedTemporaryFile() as archivedata: archivedata.write(data.content) archivedata.seek(0) - if url.endswith('.rpm'): + if url.endswith(".rpm"): processed_files[url] = self.process_rpm(archivedata) - elif url.endswith('.deb'): + elif url.endswith(".deb"): processed_files[url] = self.process_deb(archivedata) return processed_files def process_rpm(self, archivedata) -> Optional[str]: - rpm = rpmfile.RPMFile(fileobj = archivedata) + rpm = rpmfile.RPMFile(fileobj=archivedata) member = None extracted = None for member in rpm.getmembers(): - if 'vmlinux' in member.name or 'System.map' in member.name: + if "vmlinux" in member.name or "System.map" in member.name: print(f" - Extracting {member.name}") extracted = rpm.extractfile(member) break if not member or not extracted: return None - with tempfile.NamedTemporaryFile(delete = False, - prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: + with tempfile.NamedTemporaryFile( + delete=False, prefix="vmlinux" if "vmlinux" in member.name else "System.map" + ) as output: print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name def process_deb(self, archivedata) -> Optional[str]: - deb = debfile.DebFile(fileobj = archivedata) + deb = debfile.DebFile(fileobj=archivedata) member = None extracted = None for member in deb.data.tgz().getmembers(): - if member.name.endswith('vmlinux') or 'System.map' in member.name: + if member.name.endswith("vmlinux") or "System.map" in member.name: print(f" - Extracting {member.name}") extracted = deb.data.get_file(member.name) break if not member or not extracted: return None - with tempfile.NamedTemporaryFile(delete = False, - prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: + with tempfile.NamedTemporaryFile( + delete=False, prefix="vmlinux" if "vmlinux" in member.name else "System.map" + ) as output: print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name @@ -83,43 +85,55 @@ class Downloader: if named_files[i] is None: print(f"FAILURE: None encountered for {i}") return - args = [DWARF2JSON, 'linux'] - output_filename = 'unknown-kernel.json' + args = [DWARF2JSON, "linux"] + output_filename = "unknown-kernel.json" for named_file in named_files: - prefix = '--system-map' - if 'System' not in named_files[named_file]: - prefix = '--elf' - output_filename = './' + '-'.join((named_file.split('/')[-1]).split('-')[2:])[:-4] + '.json.xz' + prefix = "--system-map" + if "System" not in named_files[named_file]: + prefix = "--elf" + output_filename = ( + "./" + + "-".join((named_file.split("/")[-1]).split("-")[2:])[:-4] + + ".json.xz" + ) args += [prefix, named_files[named_file]] print(f" - Running {args}") - proc = subprocess.run(args, capture_output = True) + proc = subprocess.run(args, capture_output=True) print(f" - Writing to {output_filename}") - with lzma.open(output_filename, 'w') as f: + with lzma.open(output_filename, "w") as f: f.write(proc.stdout) -if __name__ == '__main__': - parser = argparse.ArgumentParser(description = "Takes a list of URLs for Centos and downloads them") - parser.add_argument("-f", - "--file", - dest = 'filename', - metavar = "FILENAME", - help = "Filename to be read", - required = True) - parser.add_argument("-d", - "--dwarf2json", - dest = 'dwarfpath', - metavar = "PATH", - default = DWARF2JSON, - help = "Path to the dwarf2json binary", - required = True) - parser.add_argument("-k", - "--keep", - dest = 'keep', - action = 'store_true', - help = 'Keep extracted temporary files after completion', - default = False) +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Takes a list of URLs for Centos and downloads them" + ) + parser.add_argument( + "-f", + "--file", + dest="filename", + metavar="FILENAME", + help="Filename to be read", + required=True, + ) + parser.add_argument( + "-d", + "--dwarf2json", + dest="dwarfpath", + metavar="PATH", + default=DWARF2JSON, + help="Path to the dwarf2json binary", + required=True, + ) + parser.add_argument( + "-k", + "--keep", + dest="keep", + action="store_true", + help="Keep extracted temporary files after completion", + default=False, + ) args = parser.parse_args() DWARF2JSON = args.dwarfpath @@ -132,4 +146,4 @@ if __name__ == '__main__': urls += [[lines[2 * i].strip(), lines[(2 * i) + 1].strip()]] d = Downloader(urls) - d.download_lists(keep = args.keep) + d.download_lists(keep=args.keep) diff --git a/test/plugins/windows/test_scheduled_tasks.py b/test/plugins/windows/test_scheduled_tasks.py index fdb19fbae..15d7f79a6 100644 --- a/test/plugins/windows/test_scheduled_tasks.py +++ b/test/plugins/windows/test_scheduled_tasks.py @@ -2,9 +2,11 @@ import sys import struct import traceback import unittest + sys.path.insert(0, "../../volatility3") from volatility3.plugins.windows import scheduled_tasks + class TestActionsDecoding(unittest.TestCase): def test_decode_exe_action(self): # fmt: off diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 00215f624..236d256f1 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -29,7 +29,7 @@ except ImportError: try: # Import so that the handler is found by the framework.class_subclasses callc - from smb import SMBHandler as SMBHandler # lgtm [py/unused-import] + from smb import SMBHandler as SMBHandler # lgtm [py/unused-import] except ImportError: # If we fail to import this, it means that SMB handling won't be available pass From 503999ddcafc6eee23d53a0e08c3ebad3139892e Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 13:04:58 +0100 Subject: [PATCH 187/348] update black workflow (formatter not a linter) --- .github/workflows/black.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index e29ab6f29..d755d9402 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,4 +1,4 @@ -name: Black python linter +name: Black python formatter on: [push, pull_request] From ab19b52d5b85a6fc7275bf8108fa74e966c72694 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Wed, 27 Nov 2024 11:58:14 +0100 Subject: [PATCH 188/348] run `ruff check --fix` --- volatility3/framework/automagic/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index c044fdd93..f22cae012 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,7 +3,7 @@ # import logging -from typing import Optional, Tuple, Type +from typing import Optional, Tuple from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder From d9c370c6ae94ba7ff94075279f65608034b91830 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Mon, 9 Dec 2024 15:01:32 +0100 Subject: [PATCH 189/348] format with black --- volatility3/framework/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index b60ae5576..d5e2c50b4 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -143,9 +143,7 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]: def _filter_files(filename: str): """Ensures that a filename traversed is an importable python file""" - return (filename.endswith((".py", ".pyc"))) and not filename.startswith( - "__" - ) + return (filename.endswith((".py", ".pyc"))) and not filename.startswith("__") def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str]: From c6c3b35f52a33d8a4c7ddb57245d73c6270e08f3 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Thu, 12 Dec 2024 09:21:36 +0100 Subject: [PATCH 190/348] run `ruff check . --unsafe-fixes --fix` --- volatility3/framework/plugins/windows/mftscan.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index feea78ece..c4d05e634 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -275,19 +275,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): display_data = True if display_data: - for record in cls.parse_data_record( + yield from cls.parse_data_record( mft_record, attr, record_map, return_first_record - ): - yield record + ) def _generator(self): - for record in self.enumerate_mft_records( + yield from self.enumerate_mft_records( self.context, self.config_path, self.config["primary"], self.parse_mft_records, - ): - yield record + ) def generate_timeline(self): for row in self._generator(): From 44e8ac645fe08de903df830b15eca32d006d8f50 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Tue, 17 Dec 2024 22:59:04 +0100 Subject: [PATCH 191/348] chore: make ruff happy the import is not needed, was previously used for type annotations --- .../framework/plugins/windows/indirect_system_calls.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index f09851b30..dac0f9c4a 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -13,12 +13,6 @@ from volatility3.plugins.windows import pslist, direct_system_calls vollog = logging.getLogger(__name__) -try: - import capstone -except ImportError: - # The generator of DirectSystemCalls will bail with a warning if capstone is not installed - pass - class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): _required_framework_version = (2, 4, 0) From 0d7c2906eecadcc083e4919176350f113e6d8931 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Tue, 17 Dec 2024 16:01:51 -0600 Subject: [PATCH 192/348] #1324 - update pedump error messages --- volatility3/framework/plugins/windows/pedump.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 5b4bb07d7..d9ab39b4a 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -224,11 +224,11 @@ class PEDump(interfaces.plugins.PluginInterface): ) if self.config["kernel_module"] and self.config["pid"]: - vollog.error("Only --kernel_module or --pid should be set. Not both") + vollog.error("Only 'kernel-module' or 'pid' should be set, not both") return if not self.config["kernel_module"] and not self.config["pid"]: - vollog.error("--kernel_module or --pid must be set") + vollog.error("Either 'kernel-module' or 'pid' argument must be set") return if self.config["kernel_module"]: From e035faa32392bc27d20f4b7b0c01feb0e9da2210 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 09:55:13 +1100 Subject: [PATCH 193/348] bump framework minor version --- volatility3/framework/constants/_version.py | 2 +- volatility3/framework/plugins/linux/capabilities.py | 2 +- volatility3/framework/plugins/linux/envars.py | 2 +- volatility3/framework/plugins/linux/psaux.py | 2 +- volatility3/framework/plugins/linux/pslist.py | 2 +- volatility3/framework/plugins/linux/psscan.py | 2 +- volatility3/framework/plugins/linux/pstree.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 55ef19e4b..93e9c8432 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 12 # Number of changes that only add to the interface +VERSION_MINOR = 13 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index a06ee4c1b..afd91c48e 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -49,7 +49,7 @@ class CapabilitiesData: class Capabilities(plugins.PluginInterface): """Lists process capabilities""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (1, 1, 0) @classmethod diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index a3eb21cf5..aec3eeb14 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -16,7 +16,7 @@ vollog = logging.getLogger(__name__) class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (1, 1, 0) @classmethod diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 5a4d75c70..60424a990 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -14,7 +14,7 @@ from volatility3.plugins.linux import pslist class PsAux(plugins.PluginInterface): """Lists processes with their command line arguments""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (1, 1, 0) @classmethod diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index edfc0688c..a6d2e6538 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -17,7 +17,7 @@ from volatility3.plugins.linux import elfs class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (3, 1, 0) @classmethod diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 55e3778ab..2b20ce583 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -27,7 +27,7 @@ class DescExitStateEnum(Enum): class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (1, 1, 0) @classmethod diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 7ea9df3d6..c80cfbec7 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -12,7 +12,7 @@ class PsTree(interfaces.plugins.PluginInterface): """Plugin for listing processes in a tree based on their parent process ID.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (1, 1, 0) @classmethod From 9c587b037885489442c55525add1ab6bfb69b852 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 10:12:32 +1100 Subject: [PATCH 194/348] linux: testcases: improve existent testcases code and checks --- test/test_volatility.py | 89 ++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 49 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index b5910e1c8..300ab572c 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -334,84 +334,84 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): def test_linux_pslist(image, volatility, python): rc, out, _err = runvol_plugin("linux.pslist.PsList", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.find(b"watchdog") != -1 assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_check_idt(image, volatility, python): rc, out, _err = runvol_plugin( "linux.check_idt.Check_idt", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.count(b"__kernel__") >= 10 assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_check_syscall(image, volatility, python): rc, out, _err = runvol_plugin( "linux.check_syscall.Check_syscall", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.find(b"sys_close") != -1 assert out.find(b"sys_open") != -1 assert out.count(b"\n") > 100 - assert rc == 0 def test_linux_lsmod(image, volatility, python): rc, out, _err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + out = out.lower() + assert out.count(b"\n") > 10 def test_linux_lsof(image, volatility, python): rc, out, _err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.count(b"socket:") >= 10 assert out.count(b"\n") > 35 - assert rc == 0 def test_linux_proc_maps(image, volatility, python): rc, out, _err = runvol_plugin("linux.proc.Maps", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.count(b"anonymous mapping") >= 10 assert out.count(b"\n") > 100 - assert rc == 0 def test_linux_tty_check(image, volatility, python): rc, out, _err = runvol_plugin( "linux.tty_check.tty_check", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.find(b"__kernel__") != -1 assert out.count(b"\n") >= 5 - assert rc == 0 def test_linux_sockstat(image, volatility, python): rc, out, _err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) + assert rc == 0 assert out.count(b"AF_UNIX") >= 354 assert out.count(b"AF_BLUETOOTH") >= 5 assert out.count(b"AF_INET") >= 32 assert out.count(b"AF_INET6") >= 20 assert out.count(b"AF_PACKET") >= 1 assert out.count(b"AF_NETLINK") >= 43 - assert rc == 0 def test_linux_library_list(image, volatility, python): @@ -423,49 +423,48 @@ def test_linux_library_list(image, volatility, python): pluginargs=["--pids", "2363"], ) + assert rc == 0 assert re.search( rb"NetworkManager\s2363\s0x7f52cdda0000\s/lib/x86_64-linux-gnu/libnss_files.so.2", out, ) assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_pstree(image, volatility, python): rc, out, _err = runvol_plugin("linux.pstree.PsTree", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_pidhashtable(image, volatility, python): rc, out, _err = runvol_plugin( "linux.pidhashtable.PIDHashTable", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_bash(image, volatility, python): rc, out, _err = runvol_plugin("linux.bash.Bash", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_boottime(image, volatility, python): rc, out, _err = runvol_plugin("linux.boottime.Boottime", image, volatility, python) - out = out.lower() - assert out.count(b"utc") >= 1 assert rc == 0 + out = out.lower() + assert out.count(b"utc") >= 1 def test_linux_capabilities(image, volatility, python): @@ -482,36 +481,33 @@ def test_linux_capabilities(image, volatility, python): # However, we can still check that the plugin requirements are met. return None - out = out.lower() - - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_check_creds(image, volatility, python): - rc, _out, _err = runvol_plugin( + rc, out, _err = runvol_plugin( "linux.check_creds.Check_creds", image, volatility, python ) # linux-sample-1.bin has no processes sharing credentials. # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_elfs(image, volatility, python): rc, out, _err = runvol_plugin("linux.elfs.Elfs", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_envars(image, volatility, python): rc, out, _err = runvol_plugin("linux.envars.Envars", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_kthreads(image, volatility, python): @@ -528,44 +524,42 @@ def test_linux_kthreads(image, volatility, python): # However, we can still check that the plugin requirements are met. return None - out = out.lower() - - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_malfind(image, volatility, python): - rc, _out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) + rc, out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) # linux-sample-1.bin has no process memory ranges with potential injected code. # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_mountinfo(image, volatility, python): rc, out, _err = runvol_plugin( "linux.mountinfo.MountInfo", image, volatility, python ) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_psaux(image, volatility, python): rc, out, _err = runvol_plugin("linux.psaux.PsAux", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 50 assert rc == 0 + assert out.count(b"\n") > 50 def test_linux_ptrace(image, volatility, python): - rc, _out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) + rc, out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) - # linux-sample-1.bin has no processes being ptreaced. + # linux-sample-1.bin has no processes being ptraced. # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_vmaregexscan(image, volatility, python): @@ -576,10 +570,9 @@ def test_linux_vmaregexscan(image, volatility, python): python, pluginargs=["--pid", "1", "--pattern", "\\x7fELF"], ) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_vmayarascan_yara_rule(image, volatility, python): @@ -613,9 +606,8 @@ def test_linux_vmayarascan_yara_rule(image, volatility, python): with contextlib.suppress(FileNotFoundError): os.remove(filename) - out = out.lower() - assert out.count(b"\n") > 4 assert rc == 0 + assert out.count(b"\n") > 4 def test_linux_vmayarascan_yara_string(image, volatility, python): @@ -626,10 +618,9 @@ def test_linux_vmayarascan_yara_string(image, volatility, python): python, pluginargs=["--pid", "1", "--yara-string", "ELF"], ) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_page_cache_files(image, volatility, python): @@ -640,8 +631,8 @@ def test_linux_page_cache_files(image, volatility, python): python, pluginargs=["--find", "/etc/passwd"], ) - out = out.lower() + assert rc == 0 assert out.count(b"\n") > 4 # inode_num inode_addr ... file_path From 69d291c644910b14c9ff07df83b24f23601f945d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 10:16:59 +1100 Subject: [PATCH 195/348] linux: testcases: add 10 final test cases to achieve full plugin coverage --- test/test_volatility.py | 123 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index 300ab572c..4b0455596 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -640,7 +640,130 @@ def test_linux_page_cache_files(image, volatility, python): rb"146829\s0x88001ab5c270.*?/etc/passwd", out, ) + + +def test_linux_page_cache_inodepages(image, volatility, python): + + inode_address = hex(0x88001AB5C270) + inode_dump_filename = f"inode_{inode_address}.dmp" + try: + rc, out, _err = runvol_plugin( + "linux.pagecache.InodePages", + image, + volatility, + python, + pluginargs=["--inode", inode_address, "--dump"], + ) + + assert rc == 0 + assert out.count(b"\n") > 4 + + # PageVAddr PagePAddr MappingAddr .. DumpSafe + assert re.search( + rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", + out, + ) + assert os.path.exists(inode_dump_filename) + inode_contents = open(inode_dump_filename, "rb").read() + assert inode_contents.count(b"\n") > 30 + assert inode_contents.count(b"root:x:0:0:root:/root:/bin/bash") > 0 + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(inode_dump_filename) + + +def test_linux_check_afinfo(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_afinfo.Check_afinfo", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results. + # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_check_modules(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_modules.Check_modules", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_ebpf_progs(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.ebpf.EBPF", + image, + volatility, + python, + globalargs=["-vvv"], + ) + + if rc != 0 and err.count(b"Unsupported kernel") > 0: + # The linux-sample-1.bin kernel implementation isn't supported. + # However, we can still check that the plugin requirements are met. + return None + + assert rc == 0 + assert out.count(b"\n") > 4 + + +def test_linux_iomem(image, volatility, python): + rc, out, _err = runvol_plugin("linux.iomem.IOMem", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_keyboard_notifiers(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.keyboard_notifiers.Keyboard_notifiers", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results for this plugin. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_kmesg(image, volatility, python): + rc, out, _err = runvol_plugin("linux.kmsg.Kmsg", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_netfilter(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.netfilter.Netfilter", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results for this plugin. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_psscan(image, volatility, python): + rc, out, _err = runvol_plugin("linux.psscan.PsScan", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_hidden_modules(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.hidden_modules.Hidden_modules", image, volatility, python + ) + + # linux-sample-1.bin has no hidden modules. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 # MAC From 9c0dcad7b13cca5b3d75b1977ad4ad4a963bb0f9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 10:42:54 +1100 Subject: [PATCH 196/348] linux: page cache inodepages testcase: explicitly close the file to improve clarity for AI processing --- test/test_volatility.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 4b0455596..5bce07481 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -664,7 +664,8 @@ def test_linux_page_cache_inodepages(image, volatility, python): out, ) assert os.path.exists(inode_dump_filename) - inode_contents = open(inode_dump_filename, "rb").read() + with open(inode_dump_filename, "rb") as fp: + inode_contents = fp.read() assert inode_contents.count(b"\n") > 30 assert inode_contents.count(b"root:x:0:0:root:/root:/bin/bash") > 0 finally: From e8a73dfb2d5ae368c4af758739b5593643b6bd41 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 11:29:28 +1100 Subject: [PATCH 197/348] linux: envars plugin: Add function to retrieve environment variables for a specific task. Code improvements. --- volatility3/framework/plugins/linux/envars.py | 147 ++++++++++-------- 1 file changed, 84 insertions(+), 63 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index aec3eeb14..bb6f52a7a 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -3,8 +3,9 @@ # import logging +from typing import Iterable, Tuple -from volatility3.framework import exceptions, renderers +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -17,7 +18,7 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -39,76 +40,96 @@ class Envars(plugins.PluginInterface): ), ] + @staticmethod + def get_task_env_variables( + context: interfaces.context.ContextInterface, + task: interfaces.objects.ObjectInterface, + env_area_max_size: int = 8192, + ) -> Iterable[Tuple[str, str]]: + """Yields environment variables for a given task. + + Args: + context: The plugin's operational context. + task: The task object from which to extract environment variables. + + Yields: + Tuples of (key, value) representing each environment variable. + """ + + task_name = utility.array_to_string(task.comm) + task_pid = task.pid + env_start = task.mm.env_start + env_end = task.mm.env_end + env_area_size = env_end - env_start + if not (0 < env_area_size <= env_area_max_size): + vollog.debug( + f"Task {task_pid} {task_name} appears to have environment variables of size " + f"{env_area_size} bytes which fails the sanity checking, will not extract " + "any envars." + ) + return None + + # Get process layer to read envars from + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + return None + proc_layer = context.layers[proc_layer_name] + + # Ensure the entire buffer is readable to prevent relying on exception handling + if not proc_layer.is_valid(env_start, env_area_size): + # Not mapped / swapped out + vollog.debug( + f"Unable to read environment variables for {task_pid} {task_name} starting at " + f" virtual address 0x{env_start:x} for {env_area_size} bytes, will not " + "extract any envars." + ) + return None + + # Read the full task environment variable buffer. + envar_data = proc_layer.read(env_start, env_area_size) + + # Parse envar data, envars are null terminated, keys and values are separated by '=' + envar_data = envar_data.rstrip(b"\x00") + for envar_pair in envar_data.split(b"\x00"): + try: + env_key, env_value = envar_pair.decode().split("=", 1) + except ValueError: + # Some legitimate programs, like 'avahi-daemon', avoid reallocating the args + # and instead exploit the fact that the environment variables area is contiguous + # to the args. This allows them to include a longer process name in the listing, + # causing overwrites and incorrect results. In such cases, it's better to abort + # the current task rather than displaying misleading or incorrect output. + break + + yield env_key, env_value + def _generator(self, tasks): """Generates a listing of processes along with environment variables""" # walk the process list and return the envars for task in tasks: - pid = task.pid - - # get process name as string - name = utility.array_to_string(task.comm) - ppid = task.get_parent_pid() - - # kernel threads never have an mm as they do not have userland mappings - try: - mm = task.mm - except exceptions.InvalidAddressException: - # no mm so cannot get envars - vollog.debug( - f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." - ) - mm = None + if task.is_kernel_thread: continue - # if mm exists attempt to get envars - if mm: - # get process layer to read envars from - proc_layer_name = task.add_process_layer() - if proc_layer_name is None: - vollog.debug( - f"Unable to construct process layer for task {pid} {name}, will not extract any envars." - ) - continue - proc_layer = self.context.layers[proc_layer_name] + task_pid = task.pid + task_name = utility.array_to_string(task.comm) + task_ppid = task.get_parent_pid() - # get the size of the envars with sanity checking - envars_size = task.mm.env_end - task.mm.env_start - if not (0 < envars_size <= 8192): - vollog.debug( - f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." - ) - continue - - # attempt to read all envars data - try: - envar_data = proc_layer.read(task.mm.env_start, envars_size) - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." - ) - continue - - # parse envar data, envars are null terminated, keys and values are separated by '=' - envar_data = envar_data.rstrip(b"\x00") - for envar_pair in envar_data.split(b"\x00"): - try: - key, value = envar_pair.decode().split("=", 1) - except ValueError: - vollog.debug( - f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" - ) - continue - yield (0, (pid, ppid, name, key, value)) + for env_key, env_value in self.get_task_env_variables(self.context, task): + yield (0, (task_pid, task_ppid, task_name, env_key, env_value)) def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - - return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], - self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ), + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func ) + + headers = [ + ("PID", int), + ("PPID", int), + ("COMM", str), + ("KEY", str), + ("VALUE", str), + ] + + return renderers.TreeGrid(headers, self._generator(tasks)) From e231d826a96f416e988b6f35d5a180e2d9342579 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 11:38:51 +1100 Subject: [PATCH 198/348] linux: envars plugin: Complete get_task_env_variables() docstring --- volatility3/framework/plugins/linux/envars.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index bb6f52a7a..05ce17f8a 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -51,6 +51,8 @@ class Envars(plugins.PluginInterface): Args: context: The plugin's operational context. task: The task object from which to extract environment variables. + env_area_max_size: Maximum allowable size for the environment variables area. + Tasks exceeding this size will be skipped. Default is 8192. Yields: Tuples of (key, value) representing each environment variable. From d934d4421b94c3d7285cc1594830bf84c4846373 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 12:02:14 +1100 Subject: [PATCH 199/348] linux: get_parent_pid: Fix parent ID to correctly mimic getppid() syscall behavior by using TGID instead of PID --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7b025450c..3c26805b7 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -641,7 +641,7 @@ class task_struct(generic.GenericIntelProcess): """ if self.real_parent and self.real_parent.is_readable(): - ppid = self.real_parent.pid + ppid = self.real_parent.tgid else: ppid = 0 From 2de553e1c17eab61cc566308cdabe9f9ba60b4ed Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 15:25:07 +1100 Subject: [PATCH 200/348] linux: cred: add user identifiers to the cred object extension --- .../symbols/linux/extensions/__init__.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7b025450c..2c6d9147d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2079,13 +2079,40 @@ class cred(objects.StructType): return int(value) @property - def euid(self): + def uid(self) -> int: + """Returns the real user ID + + Returns: + The real user ID value + """ + return self._get_cred_int_value("uid") + + @property + def gid(self) -> int: + """Returns the real user ID + + Returns: + The real user ID value + """ + return self._get_cred_int_value("gid") + + @property + def euid(self) -> int: """Returns the effective user ID + Returns: + The effective user ID value + """ + return self._get_cred_int_value("euid") + + @property + def egid(self) -> int: + """Returns the effective group ID + Returns: int: the effective user ID value """ - return self._get_cred_int_value("euid") + return self._get_cred_int_value("egid") class kernel_cap_struct(objects.StructType): From 448ba24eac7bb11aba30d9f3e4d0dedcd9246d43 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 15:53:07 +1100 Subject: [PATCH 201/348] linux: pslist: add user/group , real/effective identifiers to the output: uid,gid, euid and egid. We reimplemented get_task_fields() using a dataclass, reducing the size of the function's interface and preventing unbounded growth. This change simplifies future modifications and enhances maintainability. --- volatility3/framework/plugins/linux/pslist.py | 86 +++++++++++++------ 1 file changed, 61 insertions(+), 25 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index a6d2e6538..2244b91ce 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -2,7 +2,9 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import datetime -from typing import Any, Callable, Iterable, List, Tuple +import dataclasses +import contextlib +from typing import Any, Callable, Iterable, List, Optional from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -14,11 +16,25 @@ from volatility3.plugins import timeliner from volatility3.plugins.linux import elfs +@dataclasses.dataclass +class TaskFields: + offset: int + user_pid: int + user_tid: int + user_ppid: int + name: str + uid: Optional[int] + gid: Optional[int] + euid: Optional[int] + egid: Optional[int] + creation_time: Optional[datetime.datetime] + + class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 13, 0) - _version = (3, 1, 0) + _version = (4, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -82,7 +98,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def get_task_fields( cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False - ) -> Tuple[int, int, int, int, str, datetime.datetime]: + ) -> TaskFields: """Extract the fields needed for the final output Args: @@ -91,21 +107,34 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): and of Kernel threads in square brackets. Defaults to False. Returns: - A tuple with the fields to show in the plugin output. + A TaskFields object with the fields to show in the plugin output. """ - pid = task.tgid - tid = task.pid - ppid = task.get_parent_pid() name = utility.array_to_string(task.comm) - start_time = task.get_create_time() if decorate_comm: if task.is_kernel_thread: name = f"[{name}]" elif task.is_user_thread: name = f"{{{name}}}" - task_fields = (task.vol.offset, pid, tid, ppid, name, start_time) - return task_fields + # This function may be called with a partially initialized/uninitialized task. + # Ensure it always returns a valid TaskFields object, ready for use in a plugin. + valid_cred = task.cred and task.cred.is_readable() + creation_time = None + with contextlib.suppress(Exception): + creation_time = task.get_create_time() + + return TaskFields( + offset=task.vol.offset, + user_pid=task.tgid, + user_tid=task.pid, + user_ppid=task.get_parent_pid(), + name=name, + uid=task.cred.uid if valid_cred else None, + gid=task.cred.gid if valid_cred else None, + euid=task.cred.euid if valid_cred else None, + egid=task.cred.egid if valid_cred else None, + creation_time=creation_time, + ) def _get_file_output(self, task: interfaces.objects.ObjectInterface) -> str: """Extract the elf for the process if requested @@ -179,17 +208,19 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): else: file_output = "Disabled" - offset, pid, tid, ppid, name, creation_time = self.get_task_fields( - task, decorate_comm - ) + task_fields = self.get_task_fields(task, decorate_comm) yield 0, ( - format_hints.Hex(offset), - pid, - tid, - ppid, - name, - creation_time or renderers.NotAvailableValue(), + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, + task_fields.uid, + task_fields.gid, + task_fields.euid, + task_fields.egid, + task_fields.creation_time or renderers.NotAvailableValue(), file_output, ) @@ -238,6 +269,10 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ("TID", int), ("PPID", int), ("COMM", str), + ("UID", int), + ("GID", int), + ("EUID", int), + ("EGID", int), ("CREATION TIME", datetime.datetime), ("File output", str), ] @@ -251,10 +286,11 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for task in self.list_tasks( self.context, self.config["kernel"], filter_func, include_threads=True ): - offset, user_pid, user_tid, _user_ppid, name, creation_time = ( - self.get_task_fields(task) + task_fields = self.get_task_fields(task) + description = f"Process {task_fields.user_pid}/{task_fields.user_tid} {task_fields.name} ({task_fields.offset})" + + yield ( + description, + timeliner.TimeLinerType.CREATED, + task_fields.creation_time, ) - - description = f"Process {user_pid}/{user_tid} {name} ({offset})" - - yield (description, timeliner.TimeLinerType.CREATED, creation_time) From 8e12cf02e3720c3c58df6a3fb66930278d797b38 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 15:56:55 +1100 Subject: [PATCH 202/348] linux: psscan: reimplemented to make use of pslist.get_task_fields() --- volatility3/framework/plugins/linux/psscan.py | 49 +++++++------------ 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index c93f06088..ba68c4856 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -2,15 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import Iterable, List, Tuple +from typing import Iterable, List import struct from enum import Enum from volatility3.framework import renderers, interfaces, symbols, constants, exceptions from volatility3.framework.configuration import requirements -from volatility3.framework.objects import utility from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints +from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) @@ -28,7 +28,7 @@ class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -38,34 +38,11 @@ class PsScan(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) + ), ] - def _get_task_fields( - self, task: interfaces.objects.ObjectInterface - ) -> Tuple[int, int, int, str, str]: - """Extract the fields needed for the final output - - Args: - task: A task object from where to get the fields. - Returns: - A tuple with the fields to show in the plugin output. - """ - pid = task.tgid - tid = task.pid - ppid = task.get_parent_pid() - name = utility.array_to_string(task.comm) - exit_state = DescExitStateEnum(task.exit_state).name - - task_fields = ( - format_hints.Hex(task.vol.offset), - pid, - tid, - ppid, - name, - exit_state, - ) - return task_fields - def _generator(self): """Generates the tasks found from scanning.""" @@ -75,8 +52,18 @@ class PsScan(interfaces.plugins.PluginInterface): for task in self.scan_tasks( self.context, vmlinux_module_name, vmlinux.layer_name ): - row = self._get_task_fields(task) - yield (0, row) + task_fields = pslist.PsList.get_task_fields(task) + exit_state = DescExitStateEnum(task.exit_state).name + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, + exit_state, + ) + + yield (0, fields) @classmethod def scan_tasks( From 40bdbf62daec2047f7c0e1bc06a5feb85b10ca77 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 16:00:14 +1100 Subject: [PATCH 203/348] linux: pidhashtable: Update to use TaskFields from pslist.get_task_fields(). Fix some type annotations --- .../framework/plugins/linux/pidhashtable.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 2d210c233..060b3928e 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -3,7 +3,7 @@ # import logging -from typing import List +from typing import List, Iterable from volatility3.framework import renderers, interfaces, constants from volatility3.framework.symbols import linux @@ -19,7 +19,7 @@ class PIDHashTable(plugins.PluginInterface): """Enumerates processes through the PID hash table""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -30,7 +30,7 @@ class PIDHashTable(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) @@ -218,7 +218,7 @@ class PIDHashTable(plugins.PluginInterface): return None - def get_tasks(self) -> interfaces.objects.ObjectInterface: + def get_tasks(self) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerates processes through the PID hash table Yields: @@ -231,14 +231,16 @@ class PIDHashTable(plugins.PluginInterface): yield from sorted(pid_func(), key=lambda t: (t.tgid, t.pid)) - def _generator( - self, decorate_comm: bool = False - ) -> interfaces.objects.ObjectInterface: + def _generator(self, decorate_comm: bool = False): for task in self.get_tasks(): - offset, pid, tid, ppid, name, _creation_time = ( - pslist.PsList.get_task_fields(task, decorate_comm) + task_fields = pslist.PsList.get_task_fields(task, decorate_comm) + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, ) - fields = format_hints.Hex(offset), pid, tid, ppid, name yield 0, fields def run(self): From f3cf182206cd00fd70742c304ce48dbfe80ba246 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 16:00:50 +1100 Subject: [PATCH 204/348] linux: pstree: Update to use TaskFields from pslist.get_task_fields() --- volatility3/framework/plugins/linux/pstree.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index c80cfbec7..74e172139 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -13,7 +13,7 @@ class PsTree(interfaces.plugins.PluginInterface): ID.""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls): @@ -25,7 +25,7 @@ class PsTree(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", @@ -101,13 +101,17 @@ class PsTree(interfaces.plugins.PluginInterface): def yield_processes(pid): task = self._tasks[pid] - offset, pid, tid, ppid, name, _creation_time = ( - pslist.PsList.get_task_fields(task, decorate_comm) + task_fields = pslist.PsList.get_task_fields(task, decorate_comm) + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, ) - fields = format_hints.Hex(offset), pid, tid, ppid, name - yield (self._levels[tid] - 1, fields) + yield (self._levels[task_fields.user_tid] - 1, fields) - for child_pid in sorted(self._children.get(tid, [])): + for child_pid in sorted(self._children.get(task_fields.user_tid, [])): yield from yield_processes(child_pid) for pid, level in self._levels.items(): From e92b55ac862ad1f8df2a7a10bbf2b542eb26b6cb Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 16:03:46 +1100 Subject: [PATCH 205/348] linux: Update version requirements in all plugins dependent on pslist --- volatility3/framework/plugins/linux/bash.py | 4 ++-- volatility3/framework/plugins/linux/boottime.py | 4 ++-- volatility3/framework/plugins/linux/capabilities.py | 4 ++-- volatility3/framework/plugins/linux/check_creds.py | 4 ++-- volatility3/framework/plugins/linux/elfs.py | 4 ++-- volatility3/framework/plugins/linux/envars.py | 4 ++-- volatility3/framework/plugins/linux/kthreads.py | 4 ++-- volatility3/framework/plugins/linux/library_list.py | 4 ++-- volatility3/framework/plugins/linux/lsof.py | 4 ++-- volatility3/framework/plugins/linux/malfind.py | 4 ++-- volatility3/framework/plugins/linux/mountinfo.py | 4 ++-- volatility3/framework/plugins/linux/proc.py | 4 ++-- volatility3/framework/plugins/linux/psaux.py | 4 ++-- volatility3/framework/plugins/linux/ptrace.py | 4 ++-- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- volatility3/framework/plugins/linux/vmaregexscan.py | 4 ++-- volatility3/framework/plugins/linux/vmayarascan.py | 4 ++-- 17 files changed, 34 insertions(+), 34 deletions(-) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index 77a433a3b..056e3cd51 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -22,7 +22,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): """Recovers bash command history from memory.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -33,7 +33,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py index 56de52883..c57bdd65a 100644 --- a/volatility3/framework/plugins/linux/boottime.py +++ b/volatility3/framework/plugins/linux/boottime.py @@ -15,7 +15,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """Shows the time the system was started""" _required_framework_version = (2, 11, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -26,7 +26,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index afd91c48e..1d0c60c11 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -50,7 +50,7 @@ class Capabilities(plugins.PluginInterface): """Lists process capabilities""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -61,7 +61,7 @@ class Capabilities(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pids", diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 4916b67d2..96f77ce4d 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -12,7 +12,7 @@ class Check_creds(interfaces.plugins.PluginInterface): """Checks if any processes are sharing credential structures""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 2) @classmethod def get_requirements(cls): @@ -23,7 +23,7 @@ class Check_creds(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 9f3bd274b..2fd740941 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -25,7 +25,7 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 2) + _version = (2, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -36,7 +36,7 @@ class Elfs(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index aec3eeb14..9f29ef74e 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -17,7 +17,7 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls): @@ -29,7 +29,7 @@ class Envars(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index b9ced73f3..40e992069 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -20,7 +20,7 @@ class Kthreads(plugins.PluginInterface): """Enumerates kthread functions""" _required_framework_version = (2, 11, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -34,7 +34,7 @@ class Kthreads(plugins.PluginInterface): name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py index 7ec1f7f7f..e251b5689 100644 --- a/volatility3/framework/plugins/linux/library_list.py +++ b/volatility3/framework/plugins/linux/library_list.py @@ -21,7 +21,7 @@ class LibraryList(interfaces.plugins.PluginInterface): """Enumerate libraries loaded into processes""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls): @@ -32,7 +32,7 @@ class LibraryList(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pids", diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 802954f43..daa8e5a3d 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -110,7 +110,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -121,7 +121,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 0b10e60c6..e45688e97 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -18,7 +18,7 @@ class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -29,7 +29,7 @@ class Malfind(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 65775c4aa..b4f80e4f5 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -36,7 +36,7 @@ class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 2, 0) - _version = (1, 2, 2) + _version = (1, 2, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -47,7 +47,7 @@ class MountInfo(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 893eea71e..441c6bc93 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,7 +21,7 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @@ -35,7 +35,7 @@ class Maps(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 60424a990..a544c9d67 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -15,7 +15,7 @@ class PsAux(plugins.PluginInterface): """Lists processes with their command line arguments""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls): @@ -27,7 +27,7 @@ class PsAux(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index 271c0e75e..6493f22b9 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -19,7 +19,7 @@ class Ptrace(plugins.PluginInterface): """Enumerates ptrace's tracer and tracee tasks""" _required_framework_version = (2, 10, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -30,7 +30,7 @@ class Ptrace(plugins.PluginInterface): architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index aee0b1e2e..7376bcbee 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -438,7 +438,7 @@ class Sockstat(plugins.PluginInterface): """Lists all network connections for all processes.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 1) + _version = (3, 0, 2) @classmethod def get_requirements(cls): @@ -455,7 +455,7 @@ class Sockstat(plugins.PluginInterface): name="lsof", plugin=lsof.Lsof, version=(2, 0, 0) ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/vmaregexscan.py b/volatility3/framework/plugins/linux/vmaregexscan.py index 4446fc550..8fb96da1e 100644 --- a/volatility3/framework/plugins/linux/vmaregexscan.py +++ b/volatility3/framework/plugins/linux/vmaregexscan.py @@ -21,7 +21,7 @@ class VmaRegExScan(plugins.PluginInterface): """Scans all virtual memory areas for tasks using RegEx.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) MAXSIZE_DEFAULT = 128 @@ -35,7 +35,7 @@ class VmaRegExScan(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 650fcf078..4db23e50b 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -18,7 +18,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -31,7 +31,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): optional=True, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) From ddb4db5eab4d5e6a9e9ea40b81339be89a8be4cc Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 16:22:27 +1100 Subject: [PATCH 206/348] linux: pslist: Handle cases where credential IDs are unavailable due to an invalid credential pointer --- volatility3/framework/plugins/linux/pslist.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 2244b91ce..641a27b92 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -216,10 +216,10 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): task_fields.user_tid, task_fields.user_ppid, task_fields.name, - task_fields.uid, - task_fields.gid, - task_fields.euid, - task_fields.egid, + task_fields.uid or renderers.NotAvailableValue(), + task_fields.gid or renderers.NotAvailableValue(), + task_fields.euid or renderers.NotAvailableValue(), + task_fields.egid or renderers.NotAvailableValue(), task_fields.creation_time or renderers.NotAvailableValue(), file_output, ) From 3da14b3873a3f19e291e21b79735ae4a8c5794a8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 06:40:19 +0000 Subject: [PATCH 207/348] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/cli/text_renderer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 937ba4ef4..b1944ae5a 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -176,7 +176,7 @@ class QuickTextRenderer(CLIRenderer): format_hints.HexBytes: optional(hex_bytes_as_text), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), interfaces.renderers.Disassembly: optional(display_disassembly), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), "default": optional(lambda x: f"{x}"), } @@ -256,7 +256,7 @@ class CSVRenderer(CLIRenderer): format_hints.HexBytes: optional(hex_bytes_as_text), format_hints.MultiTypeData: optional(multitypedata_as_text), interfaces.renderers.Disassembly: optional(display_disassembly), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), "default": optional(lambda x: f"{x}"), } @@ -450,7 +450,7 @@ class JsonRenderer(CLIRenderer): format_hints.HexBytes: quoted_optional(hex_bytes_as_text), interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: lambda x: ( x.isoformat() if not isinstance(x, interfaces.renderers.BaseAbsentValue) From 860a8146fbd529da524bac2282fb00a31b3568bc Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 06:44:23 +0000 Subject: [PATCH 208/348] Use floor division --- volatility3/framework/layers/msf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 8d84a774b..03e144e25 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -225,7 +225,7 @@ class PdbMSFStream(linear.LinearlyMappedLayer): returned = 0 page_size = self._pdb_layer.page_size while length > 0: - page = math.floor((offset + returned) / page_size) + page = (offset + returned) // page_size page_position = (offset + returned) % page_size chunk_size = min(page_size - page_position, length) if page >= self._pages_len: From d7f678879d8982b1222e6f5676caed4b0e5a9f70 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Wed, 18 Dec 2024 15:17:26 +0800 Subject: [PATCH 209/348] Minor improvements for `mypy` --- pyproject.toml | 3 ++- volatility3/cli/__init__.py | 6 ++--- volatility3/cli/text_filter.py | 2 +- volatility3/cli/volshell/generic.py | 10 ++++---- volatility3/cli/volshell/linux.py | 6 ++--- volatility3/cli/volshell/mac.py | 6 ++--- volatility3/cli/volshell/windows.py | 6 ++--- volatility3/framework/__init__.py | 7 +++--- volatility3/framework/automagic/stacker.py | 4 +++- .../framework/automagic/symbol_cache.py | 9 +++++++ .../framework/configuration/requirements.py | 24 +++++++++---------- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/automagic.py | 2 +- .../framework/interfaces/configuration.py | 7 +++--- volatility3/framework/interfaces/context.py | 14 +++++++++-- volatility3/framework/interfaces/layers.py | 2 +- volatility3/framework/interfaces/objects.py | 1 + volatility3/framework/interfaces/renderers.py | 4 ++-- volatility3/framework/interfaces/symbols.py | 1 + .../framework/layers/scanners/__init__.py | 2 +- volatility3/framework/objects/__init__.py | 6 ++--- volatility3/framework/plugins/linux/pslist.py | 6 +++-- volatility3/framework/plugins/mac/pslist.py | 6 +++-- volatility3/framework/plugins/timeliner.py | 4 +++- .../framework/plugins/windows/modules.py | 4 ++-- .../framework/plugins/windows/pedump.py | 2 +- .../framework/plugins/windows/poolscanner.py | 2 +- .../framework/plugins/windows/pslist.py | 6 ++--- .../framework/plugins/windows/psscan.py | 2 +- .../plugins/windows/registry/printkey.py | 10 ++++---- .../plugins/windows/scheduled_tasks.py | 1 - volatility3/framework/renderers/__init__.py | 2 +- volatility3/framework/symbols/__init__.py | 8 +++---- .../framework/symbols/generic/__init__.py | 6 ++--- volatility3/framework/symbols/intermed.py | 4 ++-- .../symbols/linux/extensions/__init__.py | 2 +- volatility3/framework/symbols/mac/__init__.py | 4 ++-- .../symbols/mac/extensions/__init__.py | 2 +- volatility3/framework/symbols/metadata.py | 2 +- .../symbols/windows/extensions/__init__.py | 4 +++- .../symbols/windows/extensions/pool.py | 2 +- .../framework/symbols/windows/pdbconv.py | 4 +++- .../framework/symbols/windows/pdbutil.py | 14 +++++------ 43 files changed, 127 insertions(+), 94 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7035f7a15..cc09922e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dev = [ "jsonschema>=4.23.0,<5", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", + "types-jsonschema>=4.23.0,<5", ] test = [ @@ -68,7 +69,7 @@ include = ["volatility3*"] mypy_path = "./stubs" show_traceback = true -[tool.mypy.overrides] +[[tool.mypy.overrides]] ignore_missing_imports = true [tool.ruff] diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index da046de57..6172a17f3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,7 @@ import os import sys import tempfile import traceback -from typing import Any, Dict, List, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Tuple, Type, Union from urllib import parse, request try: @@ -64,7 +64,7 @@ class PrintedProgress: def __init__(self): self._max_message_len = 0 - def __call__(self, progress: Union[int, float], description: str = None): + def __call__(self, progress: Union[int, float], description: Optional[str] = None): """A simple function for providing text-based feedback. .. warning:: Only for development use. @@ -81,7 +81,7 @@ class PrintedProgress: class MuteProgress(PrintedProgress): """A dummy progress handler that produces no output when called.""" - def __call__(self, progress: Union[int, float], description: str = None): + def __call__(self, progress: Union[int, float], description: Optional[str] = None): pass diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 955d647f5..6bd6878a5 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -74,7 +74,7 @@ class ColumnFilter: """Identifies whether an item is found in the appropriate column""" try: if self.regex: - return re.search(self.pattern, f"{item}") + return bool(re.search(self.pattern, f"{item}")) return self.pattern in f"{item}" except OSError: return False diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 93a75ca19..12f5499f6 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -240,7 +240,7 @@ class Volshell(interfaces.plugins.PluginInterface): return None return self.context.modules[self.current_kernel_name] - def change_layer(self, layer_name: str = None): + def change_layer(self, layer_name: Optional[str] = None): """Changes the current default layer""" if not layer_name: layer_name = self.current_layer @@ -250,7 +250,7 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_layer = layer_name sys.ps1 = f"({self.current_layer}) >>> " - def change_symbol_table(self, symbol_table_name: str = None): + def change_symbol_table(self, symbol_table_name: Optional[str] = None): """Changes the current_symbol_table""" if not symbol_table_name: print("No symbol table provided, not changing current symbol table") @@ -262,7 +262,7 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_symbol_table = symbol_table_name print(f"Current Symbol Table: {self.current_symbol_table}") - def change_kernel(self, kernel_name: str = None): + def change_kernel(self, kernel_name: Optional[str] = None): if not kernel_name: print("No kernel module name provided, not changing current kernel") if kernel_name not in self.context.modules: @@ -347,7 +347,7 @@ class Volshell(interfaces.plugins.PluginInterface): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if not isinstance( @@ -479,7 +479,7 @@ class Volshell(interfaces.plugins.PluginInterface): if treegrid is not None: self.render_treegrid(treegrid) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: print("No symbol table provided") diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index c5e555ec7..41b86f78b 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -61,7 +61,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -69,7 +69,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 2b32ad677..0ed35eb27 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -63,7 +63,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -71,7 +71,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 5c2190c02..303d4d5c3 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -60,7 +60,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -68,7 +68,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index c9a2c92ea..754939460 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -12,7 +12,7 @@ import inspect import logging import os import traceback -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar +from typing import Any, Dict, Generator, List, Optional, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces @@ -58,7 +58,7 @@ class NonInheritable: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Type = None) -> Any: + def __get__(self, obj: Any, get_type: Type = Optional[None]) -> Any: if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) @@ -185,8 +185,7 @@ def _zipwalk(path: str): zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( dirlist ) - for value in zip_results: - yield value, zip_results[value] + yield from zip_results.items() def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: diff --git a/volatility3/framework/automagic/stacker.py b/volatility3/framework/automagic/stacker.py index c251d3c46..596864264 100644 --- a/volatility3/framework/automagic/stacker.py +++ b/volatility3/framework/automagic/stacker.py @@ -166,7 +166,9 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): cls, context: interfaces.context.ContextInterface, initial_layer: str, - stack_set: List[Type[interfaces.automagic.StackerLayerInterface]] = None, + stack_set: Optional[ + List[Type[interfaces.automagic.StackerLayerInterface]] + ] = None, progress_callback: constants.ProgressCallback = None, ): """Stacks as many possible layers on top of the initial layer as can be done. diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 9fad506ae..065eb6d43 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -104,9 +104,11 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): for subclazz in framework.class_subclasses(IdentifierProcessor): self._classifiers[subclazz.operating_system] = subclazz + @abstractmethod def add_identifier(self, location: str, operating_system: str, identifier: str): """Adds an identifier to the store""" + @abstractmethod def find_location( self, identifier: bytes, operating_system: Optional[str] ) -> Optional[str]: @@ -120,15 +122,18 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): The location of the symbols file that matches the identifier """ + @abstractmethod def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" + @abstractmethod def update(self): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. """ + @abstractmethod def get_identifier_dictionary( self, operating_system: Optional[str] = None, local_only: bool = False ) -> Dict[bytes, str]: @@ -142,12 +147,15 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): A dictionary of identifiers mapped to a location """ + @abstractmethod def get_identifier(self, location: str) -> Optional[bytes]: """Returns an identifier based on a specific location or None""" + @abstractmethod def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" + @abstractmethod def get_location_statistics( self, location: str ) -> Optional[Tuple[int, int, int, int]]: @@ -157,6 +165,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): A tuple of base_types, types, enums, symbols, or None is location not found """ + @abstractmethod def get_hash(self, location: str) -> Optional[str]: """Returns the hash of the JSON from within a location ISF""" diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 0cfaf5693..812b8ec59 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -11,7 +11,7 @@ expect to be in the context (such as particular layers or symboltables). import abc import logging import os -from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type +from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Type from urllib import parse, request from volatility3.framework import constants, interfaces @@ -314,11 +314,11 @@ class TranslationLayerRequirement( def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: interfaces.configuration.ConfigSimpleType = None, optional: bool = False, - oses: List = None, - architectures: List = None, + oses: Optional[List] = None, + architectures: Optional[List[str]] = None, ) -> None: """Constructs a Translation Layer Requirement. @@ -526,18 +526,18 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): description: Optional[str] = None, default: bool = False, optional: bool = False, - component: Type[interfaces.configuration.VersionableInterface] = None, + component: Optional[Type[interfaces.configuration.VersionableInterface]] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: if version is None: raise TypeError("Version cannot be None") + if component is None: + raise TypeError("Component cannot be None") if description is None: description = f"Version {'.'.join(str(x) for x in version)} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) - if component is None: - raise TypeError("Component cannot be None") self._component: Type[interfaces.configuration.VersionableInterface] = component self._version = version @@ -546,7 +546,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): context: interfaces.context.ContextInterface, config_path: str, accumulator: Optional[ - List[interfaces.configuration.VersionableInterface] + Set[interfaces.configuration.VersionableInterface] ] = None, ) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type @@ -580,7 +580,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): ) if result: - result.update({config_path: self}) + result[config_path] = self return result context.config[interfaces.configuration.path_join(config_path, self.name)] = ( @@ -604,10 +604,10 @@ class PluginRequirement(VersionRequirement): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, optional: bool = False, - plugin: Type[interfaces.plugins.PluginInterface] = None, + plugin: Optional[Type[interfaces.plugins.PluginInterface]] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: super().__init__( @@ -627,7 +627,7 @@ class ModuleRequirement( def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, architectures: Optional[List[str]] = None, optional: bool = False, diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 5111b168a..f527544c0 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -229,7 +229,7 @@ class Module(interfaces.context.ModuleInterface): def object( self, object_type: str, - offset: int = None, + offset: Optional[int] = None, native_layer_name: Optional[str] = None, absolute: bool = False, **kwargs, diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index 0867b1608..4ac386fc0 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -42,7 +42,7 @@ class AutomagicInterface( priority = 10 """An ordering to indicate how soon this automagic should be run""" - exclusion_list = [] + exclusion_list: List[str] = [] """A list of plugin categories (typically operating systems) which the plugin will not operate on""" def __init__( diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index cbbf7e342..2e4f580a7 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -82,7 +82,7 @@ class HierarchicalDict(collections.abc.Mapping): def __init__( self, - initial_dict: Dict[str, "SimpleTypeRequirement"] = None, + initial_dict: Optional[Dict[str, "SimpleTypeRequirement"]] = None, separator: str = CONFIG_SEPARATOR, ) -> None: """ @@ -328,7 +328,7 @@ class RequirementInterface(metaclass=ABCMeta): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: ConfigSimpleType = None, optional: bool = False, ) -> None: @@ -618,7 +618,7 @@ class ConstructableRequirementInterface(RequirementInterface): self, context: "interfaces.context.ContextInterface", config_path: str, - requirement_dict: Dict[str, object] = None, + requirement_dict: Optional[Dict[str, object]] = None, ) -> Optional["interfaces.objects.ObjectInterface"]: """Constructs the class, handing args and the subrequirements as parameters to __init__""" @@ -652,6 +652,7 @@ class ConstructableRequirementInterface(RequirementInterface): class ConfigurableRequirementInterface(RequirementInterface): """Simple Abstract class to provide build_required_config.""" + @abstractmethod def build_configuration( self, context: "interfaces.context.ContextInterface", diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 8b5e816e8..a87e0f1e8 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -85,7 +85,7 @@ class ContextInterface(metaclass=ABCMeta): object_type: Union[str, "interfaces.objects.Template"], layer_name: str, offset: int, - native_layer_name: str = None, + native_layer_name: Optional[str] = None, **arguments, ) -> "interfaces.objects.ObjectInterface": """Object factory, takes a context, symbol, offset and optional @@ -114,6 +114,7 @@ class ContextInterface(metaclass=ABCMeta): """ return copy.deepcopy(self) + @abstractmethod def module( self, module_name: str, @@ -232,7 +233,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): def object( self, object_type: str, - offset: int = None, + offset: Optional[int] = None, native_layer_name: Optional[str] = None, absolute: bool = False, **kwargs, @@ -277,27 +278,35 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol = self.get_symbol(name) return self.offset + symbol.address + @abstractmethod def get_type(self, name: str) -> "interfaces.objects.Template": """Returns a type from the module's symbol table.""" + @abstractmethod def get_symbol(self, name: str) -> "interfaces.symbols.SymbolInterface": """Returns a symbol object from the module's symbol table.""" + @abstractmethod def get_enumeration(self, name: str) -> "interfaces.objects.Template": """Returns an enumeration from the module's symbol table.""" + @abstractmethod def has_type(self, name: str) -> bool: """Determines whether a type is present in the module's symbol table.""" + @abstractmethod def has_symbol(self, name: str) -> bool: """Determines whether a symbol is present in the module's symbol table.""" + @abstractmethod def has_enumeration(self, name: str) -> bool: """Determines whether an enumeration is present in the module's symbol table.""" + @abstractmethod def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" + @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: """Returns the symbols within table_name (or this module if not specified) that live at the specified absolute offset provided.""" @@ -343,6 +352,7 @@ class ModuleContainer(collections.abc.Mapping): def __iter__(self): return iter(self._modules) + @abstractmethod def free_module_name(self, prefix: str = "module") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table.""" diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 56798aca9..a90a78667 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -210,7 +210,7 @@ class DataLayerInterface( context: interfaces.context.ContextInterface, scanner: ScannerInterface, progress_callback: constants.ProgressCallback = None, - sections: Iterable[Tuple[int, int]] = None, + sections: Optional[Iterable[Tuple[int, int]]] = None, ) -> Iterable[Any]: """Scans a Translation layer by chunk. diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 51d25510d..23c90b13b 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -374,6 +374,7 @@ class Template: f"{self.__class__.__name__} object has no attribute {attr}" ) + @abc.abstractmethod def __call__( self, context: "interfaces.context.ContextInterface", diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index 7105274c0..e26164ee7 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -183,7 +183,7 @@ class TreeGrid(metaclass=ABCMeta): @abstractmethod def populate( self, - function: VisitorSignature = None, + function: Optional[VisitorSignature] = None, initial_accumulator: Any = None, fail_on_errors: bool = True, ) -> Optional[Exception]: @@ -235,7 +235,7 @@ class TreeGrid(metaclass=ABCMeta): node: Optional[TreeNode], function: VisitorSignature, initial_accumulator: _Type, - sort_key: ColumnSortKey = None, + sort_key: Optional[ColumnSortKey] = None, ) -> None: """Visits all the nodes in a tree, calling function on each one. diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index ead91fb4d..b8712e38d 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -256,6 +256,7 @@ class SymbolSpaceInterface(collections.abc.Mapping): """An interface for the container that holds all the symbol-containing tables for use within a context.""" + @abstractmethod def free_table_name(self, prefix: str = "layer") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table.""" diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index f54b44ff4..be9f1c39a 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -72,7 +72,7 @@ class MultiStringScanner(layers.ScannerInterface): return None for char in value: - trie[char] = trie.get(char, {}) + trie.setdefault(char, {}) trie = trie[char] # Mark the end of a string diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 5846da070..869d4dae6 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -152,7 +152,7 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): type_name: str, object_info: interfaces.objects.ObjectInformation, data_format: DataFormatInfo, - new_value: TUnion[int, float, bool, bytes, str] = None, + new_value: Optional[TUnion[int, float, bool, bytes, str]] = None, **kwargs, ) -> "PrimitiveObject": """Creates the appropriate class and returns it so that the native type @@ -601,7 +601,7 @@ class Enumeration(interfaces.objects.ObjectInterface, int): inverse_choices[v] = k return inverse_choices - def lookup(self, value: int = None) -> str: + def lookup(self, value: Optional[int] = None) -> str: """Looks up an individual value and returns the associated name. If multiple identifiers map to the same value, the first matching identifier will be returned @@ -690,7 +690,7 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): type_name: str, object_info: interfaces.objects.ObjectInformation, count: int = 0, - subtype: templates.ObjectTemplate = None, + subtype: Optional[templates.ObjectTemplate] = None, ) -> None: super().__init__(context=context, type_name=type_name, object_info=object_info) self._vol["count"] = count diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index a6d2e6538..82b8dcc67 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import datetime -from typing import Any, Callable, Iterable, List, Tuple +from typing import Any, Callable, Iterable, List, Optional, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -58,7 +58,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ] @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[Any], bool]: + def create_pid_filter( + cls, pid_list: Optional[List[int]] = None + ) -> Callable[[Any], bool]: """Constructs a filter function for process IDs. Args: diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 74d044ba9..8c5e5c1a5 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Dict, Iterable, List +from typing import Callable, Dict, Iterable, List, Optional from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -82,7 +82,9 @@ class PsList(interfaces.plugins.PluginInterface): return list_tasks @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[int], bool]: + def create_pid_filter( + cls, pid_list: Optional[List[int]] = None + ) -> Callable[[int], bool]: def filter_func(_): return False diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 4e483922b..0f4064d79 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -54,7 +54,9 @@ class Timeliner(interfaces.plugins.PluginInterface): self.automagics: Optional[List[interfaces.automagic.AutomagicInterface]] = None @classmethod - def get_usable_plugins(cls, selected_list: List[str] = None) -> List[Type]: + def get_usable_plugins( + cls, selected_list: Optional[List[str]] = None + ) -> List[Type]: # Initialize for the run plugin_list = list(framework.class_subclasses(TimeLinerInterface)) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index a3677ad34..85eb474a8 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import Generator, Iterable, List +from typing import Generator, Iterable, List, Optional from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -133,7 +133,7 @@ class Modules(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - pids: List[int] = None, + pids: Optional[List[int]] = None, ) -> Generator[str, None, None]: """Build a cache of possible virtual layers, in priority starting with the primary/kernel layer. Then keep one layer per session by cycling diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 85d5d14d1..678652624 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -96,7 +96,7 @@ class PEDump(interfaces.plugins.PluginInterface): pe_table_name: str, ldr_entry: interfaces.objects.ObjectInterface, open_method: Type[interfaces.plugins.FileHandlerInterface], - layer_name: str = None, + layer_name: Optional[str] = None, prefix: str = "", ) -> Optional[str]: """Extracts the PE file referenced an LDR_DATA_TABLE_ENTRY (DLL, kernel module) instance diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 8c56d202d..efde09638 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -183,7 +183,7 @@ class PoolScanner(plugins.PluginInterface): @staticmethod def builtin_constraints( - symbol_table: str, tags_filter: List[bytes] = None + symbol_table: str, tags_filter: Optional[List[bytes]] = None ) -> List[PoolConstraint]: """Get built-in PoolConstraints given a list of pool tags. diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index f262aeae6..579a235d8 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Iterator, List, Type +from typing import Callable, Iterator, List, Optional, Type from volatility3.framework import renderers, interfaces, layers, exceptions, constants from volatility3.framework.configuration import requirements @@ -114,7 +114,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def create_pid_filter( - cls, pid_list: List[int] = None, exclude: bool = False + cls, pid_list: Optional[List[int]] = None, exclude: bool = False ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process IDs. @@ -171,7 +171,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def create_name_filter( - cls, name_list: List[str] = None, exclude: bool = False + cls, name_list: Optional[List[str]] = None, exclude: bool = False ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process names. diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 86eb47300..cdf344ee6 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -89,7 +89,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, layer_name: str, - offset: int = None, + offset: Optional[int] = None, physical: bool = True, exclude: bool = False, ) -> Callable[[interfaces.objects.ObjectInterface], bool]: diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 4fe3f97fb..ed926805b 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import List, Sequence, Iterable, Tuple, Union +from typing import List, Optional, Sequence, Iterable, Tuple, Union from volatility3.framework import objects, renderers, exceptions, interfaces, constants from volatility3.framework.configuration import requirements @@ -51,7 +51,7 @@ class PrintKey(interfaces.plugins.PluginInterface): def key_iterator( cls, hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, + node_path: Optional[Sequence[objects.StructType]] = None, recurse: bool = False, ) -> Iterable[ Tuple[ @@ -121,7 +121,7 @@ class PrintKey(interfaces.plugins.PluginInterface): def _printkey_iterator( self, hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, + node_path: Optional[Sequence[objects.StructType]] = None, recurse: bool = False, ): """Method that wraps the more generic key_iterator, to provide output @@ -242,8 +242,8 @@ class PrintKey(interfaces.plugins.PluginInterface): self, layer_name: str, symbol_table: str, - hive_offsets: List[int] = None, - key: str = None, + hive_offsets: Optional[List[int]] = None, + key: Optional[str] = None, recurse: bool = False, ): for hive in hivelist.HiveList.list_hives( diff --git a/volatility3/framework/plugins/windows/scheduled_tasks.py b/volatility3/framework/plugins/windows/scheduled_tasks.py index 277a0d856..6dd5613c4 100644 --- a/volatility3/framework/plugins/windows/scheduled_tasks.py +++ b/volatility3/framework/plugins/windows/scheduled_tasks.py @@ -270,7 +270,6 @@ class _ScheduledTasksReader(io.BytesIO): return val def read_aligned_bstring_expand_sz(self) -> Optional[str]: - # type: () -> Optional[str] sz = self.read_aligned_u4() if sz is None: return None diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 39ce1135d..112e93751 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -214,7 +214,7 @@ class TreeGrid(interfaces.renderers.TreeGrid): def populate( self, - function: interfaces.renderers.VisitorSignature = None, + function: Optional[interfaces.renderers.VisitorSignature] = None, initial_accumulator: Any = None, fail_on_errors: bool = True, ) -> Optional[Exception]: diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index a8753bd4d..87f2288d7 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -53,10 +53,10 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} - def clear_symbol_cache(self, table_name: str = None) -> None: + def clear_symbol_cache(self, table_name: Optional[str] = None) -> None: """Clears the symbol cache for the specified table name. If no table name is specified, the caches of all symbol tables are cleared.""" - table_list: List[interfaces.symbols.BaseSymbolTableInterface] = list() + table_list: List[interfaces.symbols.BaseSymbolTableInterface] = [] if table_name is None: table_list = list(self._dict.values()) else: @@ -81,7 +81,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): yield table + constants.BANG + symbol_name def get_symbols_by_location( - self, offset: int, size: int = 0, table_name: str = None + self, offset: int, size: int = 0, table_name: Optional[str] = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( @@ -128,7 +128,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self, producer: str, validator: Callable[[Optional[Tuple], Optional[datetime.datetime]], bool], - tables: List[str] = None, + tables: Optional[List[str]] = None, ) -> bool: """Verifies the producer metadata and version of tables diff --git a/volatility3/framework/symbols/generic/__init__.py b/volatility3/framework/symbols/generic/__init__.py index 9d6da5aa4..7dd00fa75 100644 --- a/volatility3/framework/symbols/generic/__init__.py +++ b/volatility3/framework/symbols/generic/__init__.py @@ -4,7 +4,7 @@ import random import string -from typing import Union +from typing import Optional, Union from volatility3.framework import objects, interfaces @@ -14,8 +14,8 @@ class GenericIntelProcess(objects.StructType): self, context: interfaces.context.ContextInterface, dtb: Union[int, interfaces.objects.ObjectInterface], - config_prefix: str = None, - preferred_name: str = None, + config_prefix: Optional[str] = None, + preferred_name: Optional[str] = None, ) -> str: """Constructs a new layer based on the process's DirectoryTableBase.""" diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 8a28d732f..5b4aa22b8 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -86,7 +86,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): config_path: str, name: str, isf_url: str, - native_types: interfaces.symbols.NativeTableInterface = None, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, validate: bool = True, class_types: Optional[ @@ -319,7 +319,7 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass=ABCMeta): config_path: str, name: str, json_object: Any, - native_types: interfaces.symbols.NativeTableInterface = None, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, ) -> None: self._json_object = json_object diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7b025450c..4e2e80bc6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -308,7 +308,7 @@ class module(generic.GenericIntelProcess): class task_struct(generic.GenericIntelProcess): def add_process_layer( - self, config_prefix: str = None, preferred_name: str = None + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: """Constructs a new layer based on the process's DTB. diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index ee6dd10a3..dc54a8371 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Iterator, Any, Iterable, List, Tuple, Set +from typing import Iterator, Any, Iterable, List, Optional, Tuple, Set from volatility3.framework import interfaces, objects, exceptions, constants from volatility3.framework.symbols import intermed @@ -97,7 +97,7 @@ class MacUtilities(interfaces.configuration.VersionableInterface): context: interfaces.context.ContextInterface, handlers: Iterator[Any], target_address, - kernel_module_name: str = None, + kernel_module_name: Optional[str] = None, ): mod_name = "UNKNOWN" symbol_name = "N/A" diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index d2573fb95..cc700f209 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -18,7 +18,7 @@ class proc(generic.GenericIntelProcess): return self.task.dereference().cast("task") def add_process_layer( - self, config_prefix: str = None, preferred_name: str = None + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: """Constructs a new layer based on the process's DTB. diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 7e069e518..ea635f1f1 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -25,7 +25,7 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): return self._json_data.get("version", "") @property - def version(self) -> Optional[Tuple[int]]: + def version(self) -> Optional[Tuple[int, ...]]: """Returns the version of the ISF file producer""" version = self.version_string if not version: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 600f3e23f..d63f138b6 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -692,7 +692,9 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return True - def add_process_layer(self, config_prefix: str = None, preferred_name: str = None): + def add_process_layer( + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None + ): """Constructs a new layer based on the process's DirectoryTableBase.""" parent_layer = self._context.layers[self.vol.layer_name] diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index 5a7847986..de5c8271b 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -362,7 +362,7 @@ class OBJECT_HEADER(objects.StructType): return True def get_object_type( - self, type_map: Dict[int, str], cookie: int = None + self, type_map: Dict[int, str], cookie: Optional[int] = None ) -> Optional[str]: """Across all Windows versions, the _OBJECT_HEADER embeds details on the type of object (i.e. process, file) but the way its embedded diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index ea2884bb2..248ef7d0c 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -984,7 +984,9 @@ if __name__ == "__main__": def __init__(self): self._max_message_len = 0 - def __call__(self, progress: Union[int, float], description: str = None): + def __call__( + self, progress: Union[int, float], description: Optional[str] = None + ): """A simple function for providing text-based feedback. .. warning:: Only for development use. diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 1a8644fa8..b5e8ca70a 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -36,7 +36,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): layer_name: str, offset: int, symbol_table_class: str = "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path: str = None, + config_path: Optional[str] = None, progress_callback: constants.ProgressCallback = None, ) -> Optional[str]: """Produces the name of a symbol table loaded from the offset for an MZ header @@ -388,8 +388,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, ) -> str: """Creates symbol table for a module in the specified layer_name. @@ -418,8 +418,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, create_module: bool = False, ) -> Tuple[Optional[str], Optional[str]]: if module_offset is None: @@ -478,8 +478,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, ) -> str: """Creates a module in the specified layer_name based on a pdb name. From cf7aabead44aa1aee02384c57860f7e3ba1a5a84 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 08:12:49 +0000 Subject: [PATCH 210/348] Change semi-colon to colon --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 0cfaf5693..cc3ed847e 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -111,7 +111,7 @@ class ListRequirement(interfaces.configuration.RequirementInterface): Args: element_type: The (requirement) type of each element within the list - max_elements; The maximum number of acceptable elements this list can contain + max_elements: The maximum number of acceptable elements this list can contain min_elements: The minimum number of acceptable elements this list can contain """ super().__init__(*args, **kwargs) From 7464a3b883a50ee35a2e3c607d1906dd6d8c1807 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 18 Dec 2024 10:57:33 -0600 Subject: [PATCH 211/348] Windows Extensions: Fix potential AttributeErrors If `self.get_owner()` returns `None`, and the chained call to `is_valid()` is executed, an `AttributeError` will occur. This fixes two instances of this bug by intializing a local variable with the result of the `get_owner()` call, checking for `None`, and returning if that's the case. Also adds type-hints for these methods. --- .../symbols/windows/extensions/network.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 9b7573c2e..cfa2a7a30 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -4,7 +4,7 @@ import logging import socket -from typing import Dict, Tuple, List, Union +from typing import Dict, Tuple, List, Union, Optional from volatility3.framework import exceptions from volatility3.framework import objects, interfaces @@ -86,19 +86,29 @@ class _TCP_LISTENER(objects.StructType): except exceptions.InvalidAddressException: return None - def get_owner_pid(self): - if self.get_owner().is_valid(): - if self.get_owner().has_valid_member("UniqueProcessId"): - return self.get_owner().UniqueProcessId + def get_owner_pid(self) -> Optional[int]: + owner = self.get_owner() + + if owner is None: + return None + + if owner.is_valid(): + if owner.has_valid_member("UniqueProcessId"): + return owner.UniqueProcessId return None - def get_owner_procname(self): - if self.get_owner().is_valid(): - if self.get_owner().has_valid_member("ImageFileName"): - return self.get_owner().ImageFileName.cast( + def get_owner_procname(self) -> Optional[str]: + owner = self.get_owner() + + if owner is None: + return None + + if owner.is_valid(): + if owner.has_valid_member("ImageFileName"): + return owner.ImageFileName.cast( "string", - max_length=self.get_owner().ImageFileName.vol.count, + max_length=owner.ImageFileName.vol.count, errors="replace", ) From c865f4892c88d346d010a4d08ed75ecdb10a44c5 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:36:05 +0000 Subject: [PATCH 212/348] Slightly modify documentation --- doc/source/glossary.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index a9460b1a2..33e56883a 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -123,6 +123,11 @@ Page Table possible to use them as a way to map a particular address within a (potentially larger, but sparsely populated) virtual space to a concrete (and usually contiguous) physical space, through the process of :ref:`mapping`. +.. _Plugin: + +Plugin + Plugins are the "functions" of the volatility framework. They carry out algorithms on data stored in layers using objects constructed from symbols. Broadly, plugins take in a number of TranslationLayers (the data, which is a representation of part of an image, in a specified type described by templates) and outputs a TreeGrid. + .. _Pointer: Pointer From 1e2900d587acb1b752a1c8352bcf4fc112a9ec7a Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:39:03 +0000 Subject: [PATCH 213/348] Slightly modify documentation --- doc/source/glossary.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 33e56883a..04f1fb090 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -61,9 +61,7 @@ Map, mapping of the :ref:`Range`). Mappings can be seen as a mathematical function, and therefore volatility 3 attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). - For further information, please see - `https://en.wikipedia.org/wiki/Function_(mathematics)`. - + For further information, please see `[Function (mathematics) in Wikipedia](https://en.wikipedia.org/wiki/Function_(mathematics))`. .. _Member: From 7eca407b6d8fc6123486c79121f0d6db2bf7dc8b Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 18 Dec 2024 11:34:25 -0600 Subject: [PATCH 214/348] Windows PEDump: Revert overwritten changes When #1364 was merged, it may not have been rebased onto the changes introduced in #1422, and they ended up overwritten to the old version. This reverts those changes. --- .../framework/plugins/windows/pedump.py | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 85d5d14d1..275775ddb 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -64,30 +64,27 @@ class PEDump(interfaces.plugins.PluginInterface): """ Returns the filename of the dump file or None """ - try: - file_handle = open_method(file_name) + with open_method(file_name) as file_handle: + try: + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=base, - layer_name=layer_name, - ) + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + OSError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - OSError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") - return None - finally: - file_handle.close() - - return file_handle.preferred_filename + return file_handle.preferred_filename @classmethod def dump_ldr_entry( From a9417edd2810e682474966d501786bfb8e3206ec Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:49:56 +0000 Subject: [PATCH 215/348] Slightly modify documentation --- doc/source/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 04f1fb090..d3bc9613a 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -61,7 +61,7 @@ Map, mapping of the :ref:`Range`). Mappings can be seen as a mathematical function, and therefore volatility 3 attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). - For further information, please see `[Function (mathematics) in Wikipedia](https://en.wikipedia.org/wiki/Function_(mathematics))`. + For further information, please see `Function (mathematics) in Wikipedia_`. .. _Member: From 3c26955e34d68150e5d009557a66581fd7188bb4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 19 Dec 2024 12:02:16 +1100 Subject: [PATCH 216/348] xen_layer: fix potential uninitialized variable issue #1434 --- volatility3/framework/layers/xen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index e7aa0ccec..c0a5e1a7d 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -54,6 +54,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): segments = [] self._segment_headers = [] + segment_names = None for sindex in range(ehdr.e_shnum): shdr = self.context.object( From c82d432b10258136ff0777dfec1fbf5844316132 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Thu, 19 Dec 2024 12:16:55 +0800 Subject: [PATCH 217/348] Typing fix --- volatility3/framework/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 754939460..a1925faef 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,7 +5,6 @@ # Check the python version to ensure it's suitable import glob import sys -from volatility3.framework import check_python_version as check_python_version import zipfile import importlib import inspect @@ -58,7 +57,7 @@ class NonInheritable: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Type = Optional[None]) -> Any: + def __get__(self, obj: Any, get_type: Optional[Type] = None) -> Any: if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) From 95e103d9ea6c556e93872f35cc1a83d8453ab94f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:35:34 +0000 Subject: [PATCH 218/348] Remove commented out import --- volatility3/framework/plugins/windows/shimcachemem.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 59f33510d..b8e9b5bd7 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -17,8 +17,6 @@ from volatility3.framework.symbols.windows.extensions import pe, shimcache from volatility3.plugins import timeliner from volatility3.plugins.windows import modules, pslist, vadinfo -# from volatility3.plugins.windows import pslist, vadinfo, modules - vollog = logging.getLogger(__name__) From c56d9334f82f90e90d46e1026e43d5e209cf9466 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:00:27 +0000 Subject: [PATCH 219/348] Tweak comment --- volatility3/framework/plugins/windows/pe_symbols.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 002577241..21e657ab3 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -158,7 +158,7 @@ class PESymbolFinder: class PDBSymbolFinder(PESymbolFinder): """ - PESymbolFinder implementation for PDB modules + PESymbolFinder implementation for PDB modules """ def _do_get_address(self, name: str) -> Optional[int]: @@ -195,7 +195,7 @@ class PDBSymbolFinder(PESymbolFinder): class ExportSymbolFinder(PESymbolFinder): """ - PESymbolFinder implementation for PDB modules + PESymbolFinder implementation for PDB modules """ def _get_name(self, export: pefile.ExportData) -> Optional[str]: @@ -300,7 +300,7 @@ class PESymbols(interfaces.plugins.PluginInterface): base_address: int, ) -> Optional[pefile.PE]: """ - Attempts to pefile object from the bytes of the PE file + Attempts to create a pefile object from the bytes of the PE file Args: pe_table_name: name of the pe types table From 0e0c959cd0f3d4a7d14f5cd19a683609b64808f6 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:21:20 +0000 Subject: [PATCH 220/348] Swap two letters in a typo --- volatility3/framework/plugins/windows/psxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index e3ec216dd..b5ddd2ee5 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -25,7 +25,7 @@ class PsXView(plugins.PluginInterface): identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this plugin's output in a terminal.""" - # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality + # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the functionality # which the original plugin used to do it. # The sessions method is omitted because it begins with the list of processes found by Pslist anyway. From a11131b3d4767ec15619bab8084c4914ad528f75 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 14:59:58 +0000 Subject: [PATCH 221/348] Update how to write a simple plugin --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 39670a62d..84b921114 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -52,7 +52,7 @@ to be able to run properly. Any that are defined as optional need not necessari version = (2, 0, 0))] -This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how +This is a classmethod, so it can be called before the specific plugin object has been instantiated (in order to know how to instantiate the plugin). At the moment these requirements are fairly straightforward: :: From 4fe3db50df1a53d80d79ac100fdfaf577146fa86 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:32:30 +0000 Subject: [PATCH 222/348] Reorder requirements by type --- .../framework/plugins/windows/dlllist.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 57f19f620..35b9fb2dc 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -19,7 +19,7 @@ vollog = logging.getLogger(__name__) class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): - """Lists the loaded modules in a particular windows memory image.""" + """Lists the loaded DLLs in a particular windows memory image.""" _required_framework_version = (2, 0, 0) _version = (3, 0, 0) @@ -39,6 +39,9 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="psscan", component=psscan.PsScan, version=(1, 1, 0) ), + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), requirements.VersionRequirement( name="info", component=info.Info, version=(1, 0, 0) ), @@ -53,16 +56,16 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): description="Process offset in the physical address space", optional=True, ), - requirements.StringRequirement( - name="name", - description="Specify a regular expression to match dll name(s)", - optional=True, - ), requirements.IntRequirement( name="base", description="Specify a base virtual address in process memory", optional=True, ), + requirements.StringRequirement( + name="name", + description="Specify a regular expression to match dll name(s)", + optional=True, + ), requirements.BooleanRequirement( name="ignore-case", description="Specify case insensitivity for the regular expression name matching", @@ -75,9 +78,6 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): default=False, optional=True, ), - requirements.VersionRequirement( - name="pedump", component=pedump.PEDump, version=(1, 0, 0) - ), ] def _generator(self, procs): @@ -90,12 +90,15 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kuser = info.Info.get_kuser_structure( self.context, kernel.layer_name, kernel.symbol_table_name ) + nt_major_version = int(kuser.NtMajorVersion) nt_minor_version = int(kuser.NtMinorVersion) + # LoadTime only applies to versions higher or equal to Window 7 (6.1 and higher) dll_load_time_field = (nt_major_version > 6) or ( nt_major_version == 6 and nt_minor_version >= 1 ) + for proc in procs: proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() @@ -114,7 +117,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mod_re = re.compile(self.config["name"], flags) except re.error: vollog.debug( - "Error parsing regular expression: %s", self.config["name"] + f"Error parsing regular expression: {self.config["name"]}" ) return None From 59a5e85b504c48a381fb65f8fc2b42a84bf1bc9a Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:36:42 +0000 Subject: [PATCH 223/348] Reorder requirements by type --- volatility3/framework/plugins/windows/dlllist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 35b9fb2dc..65e337dfc 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -117,7 +117,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mod_re = re.compile(self.config["name"], flags) except re.error: vollog.debug( - f"Error parsing regular expression: {self.config["name"]}" + f'Error parsing regular expression: {self.config["name"]}' ) return None @@ -138,7 +138,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute - # and 32bit version shouldn't have the Quadpart according to MSDN + # and 32-bit version shouldn't have the Quadpart according to MSDN try: DllLoadTime = conversion.wintime_to_datetime( entry.LoadTime.QuadPart From 8b35031d0f443184953d80263f2689e4dd0a059f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 00:37:31 +0000 Subject: [PATCH 224/348] Volshell: Bump linux.pslist plugin requirement --- volatility3/cli/volshell/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index c5e555ec7..72193201c 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -20,7 +20,7 @@ class Volshell(generic.Volshell): name="kernel", description="Linux kernel module" ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.IntRequirement( name="pid", description="Process ID", optional=True From 658d40335a15bc40d2ecb5679198d759858492b5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 13:06:14 +1100 Subject: [PATCH 225/348] testcases: Add basic volshell testcases for each OS image --- .github/workflows/test.yaml | 5 ++++ test/test_volatility.py | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dfc42499d..e07673175 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -42,6 +42,11 @@ jobs: - name: Testing... run: | + # VolShell + pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_windows_volshell -v + pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_linux_volshell -v + + # Volatility pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_windows -v pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_linux -v diff --git a/test/test_volatility.py b/test/test_volatility.py index 5bce07481..8ef6d8b70 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -54,13 +54,56 @@ def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]) return runvol(args, volatility, python) +def runvolshell(img, volshell, python, volshellargs=[], globalargs=[]): + args = ( + globalargs + + [ + "--single-location", + img, + "-q", + ] + + volshellargs + ) + + return runvol(args, volshell, python) + + # # TESTS # + +def basic_volshell_test(image, volatility, python): + # Basic VolShell test to verify requirements and ensure VolShell runs without crashing + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".txt") + try: + with os.fdopen(fd, "w") as f: + f.write("exit()") + + rc, out, _err = runvolshell( + img=image, + volshell=volatility, + python=python, + volshellargs=["--script", filename], + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + assert rc == 0 + assert out.count(b"\n") >= 4 + + # WINDOWS +def test_windows_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python) + + def test_windows_pslist(image, volatility, python): rc, out, _err = runvol_plugin("windows.pslist.PsList", image, volatility, python) out = out.lower() @@ -332,6 +375,10 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): # LINUX +def test_linux_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python) + + def test_linux_pslist(image, volatility, python): rc, out, _err = runvol_plugin("linux.pslist.PsList", image, volatility, python) @@ -770,6 +817,10 @@ def test_linux_hidden_modules(image, volatility, python): # MAC +def test_mac_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python) + + def test_mac_pslist(image, volatility, python): rc, out, _err = runvol_plugin("mac.pslist.PsList", image, volatility, python) out = out.lower() From 90766f466f581561e17b2cc5e2c7d8eca56c56e6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 13:21:24 +1100 Subject: [PATCH 226/348] testcases: exclude volshell test from the volatility set --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e07673175..ce2722457 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,8 +47,8 @@ jobs: pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_linux_volshell -v # Volatility - pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_windows -v - pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_linux -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k "test_windows and not test_windows_volshell" -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k "test_linux and not test_linux_volshell" -v - name: Clean up post-test run: | From e6e0754fa523f9c0edb6a6e449bdbf08a16fa712 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Fri, 20 Dec 2024 10:46:54 +0800 Subject: [PATCH 227/348] Remove mypy overrides section --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cc09922e9..d695a4eac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,9 +69,6 @@ include = ["volatility3*"] mypy_path = "./stubs" show_traceback = true -[[tool.mypy.overrides]] -ignore_missing_imports = true - [tool.ruff] line-length = 88 target-version = "py38" From c317a45eb56a5da26ee50e2edc0d2e8c6821d262 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 14:31:40 +1100 Subject: [PATCH 228/348] testcases: Add missing operating system argument --- test/test_volatility.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 8ef6d8b70..f8a32fef3 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -73,7 +73,7 @@ def runvolshell(img, volshell, python, volshellargs=[], globalargs=[]): # -def basic_volshell_test(image, volatility, python): +def basic_volshell_test(image, volatility, python, globalargs): # Basic VolShell test to verify requirements and ensure VolShell runs without crashing # FIXME: When the minimum Python version includes 3.12, replace the following with: @@ -88,6 +88,7 @@ def basic_volshell_test(image, volatility, python): volshell=volatility, python=python, volshellargs=["--script", filename], + globalargs=globalargs, ) finally: with contextlib.suppress(FileNotFoundError): @@ -101,7 +102,7 @@ def basic_volshell_test(image, volatility, python): def test_windows_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python) + basic_volshell_test(image, volatility, python, globalargs=["-w"]) def test_windows_pslist(image, volatility, python): @@ -376,7 +377,7 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): def test_linux_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python) + basic_volshell_test(image, volatility, python, globalargs=["-l"]) def test_linux_pslist(image, volatility, python): @@ -818,7 +819,7 @@ def test_linux_hidden_modules(image, volatility, python): def test_mac_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python) + basic_volshell_test(image, volatility, python, globalargs=["-m"]) def test_mac_pslist(image, volatility, python): From d546c983f3012f3060aca2b2721fedcc48321370 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 14:33:36 +1100 Subject: [PATCH 229/348] testcases: Fix runvol* default list arguments --- test/test_volatility.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index f8a32fef3..47ef9769f 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -39,7 +39,9 @@ def runvol(args, volatility, python): return p.returncode, stdout, stderr -def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): +def runvol_plugin(plugin, img, volatility, python, pluginargs=None, globalargs=None): + pluginargs = pluginargs or [] + globalargs = globalargs or [] args = ( globalargs + [ @@ -54,7 +56,9 @@ def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]) return runvol(args, volatility, python) -def runvolshell(img, volshell, python, volshellargs=[], globalargs=[]): +def runvolshell(img, volshell, python, volshellargs=None, globalargs=None): + volshellargs = volshellargs or [] + globalargs = globalargs or [] args = ( globalargs + [ From ef977efe62af8e4cc5baa96e8a20fb713ee7494f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 14:50:35 +1100 Subject: [PATCH 230/348] testcases: Improve volshell basic testcase calling ps() on each of them --- test/test_volatility.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 47ef9769f..bb7c9a851 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -80,12 +80,18 @@ def runvolshell(img, volshell, python, volshellargs=None, globalargs=None): def basic_volshell_test(image, volatility, python, globalargs): # Basic VolShell test to verify requirements and ensure VolShell runs without crashing + volshell_commands = [ + "print(ps())", + "exit()", + ] + # FIXME: When the minimum Python version includes 3.12, replace the following with: # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... fd, filename = tempfile.mkstemp(suffix=".txt") try: + volshell_script = "\n".join(volshell_commands) with os.fdopen(fd, "w") as f: - f.write("exit()") + f.write(volshell_script) rc, out, _err = runvolshell( img=image, @@ -101,12 +107,15 @@ def basic_volshell_test(image, volatility, python, globalargs): assert rc == 0 assert out.count(b"\n") >= 4 + return out + # WINDOWS def test_windows_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python, globalargs=["-w"]) + out = basic_volshell_test(image, volatility, python, globalargs=["-w"]) + assert out.count(b" 40 def test_windows_pslist(image, volatility, python): @@ -381,7 +390,8 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): def test_linux_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python, globalargs=["-l"]) + out = basic_volshell_test(image, volatility, python, globalargs=["-l"]) + assert out.count(b" 100 def test_linux_pslist(image, volatility, python): From 89564cca66a14f9a1707a6d3e056ef56e605b4f0 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 20 Dec 2024 05:48:42 +0000 Subject: [PATCH 231/348] Revert one f-string As part of the ruff linting we did recently all vollog messages should have explicitly been reverted back to %-formatting. --- volatility3/framework/plugins/windows/dlllist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 65e337dfc..1dafb6bf5 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -117,7 +117,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mod_re = re.compile(self.config["name"], flags) except re.error: vollog.debug( - f'Error parsing regular expression: {self.config["name"]}' + "Error parsing regular expression: %s", self.config["name"] ) return None From 15eb80b600e6a2114e232355a04cb1bbf5ce8972 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 20 Dec 2024 13:11:35 +0100 Subject: [PATCH 232/348] fix unbound page variable access --- volatility3/framework/plugins/windows/malfind.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 510719352..14362776b 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -120,8 +120,7 @@ class Malfind(interfaces.plugins.PluginInterface): vadinfo.winnt_protections, ) write_exec = "EXECUTE" in protection_string and "WRITE" in protection_string - dirty_page_check = False - + dirty_page = None if not write_exec: """ # Inspect "PAGE_EXECUTE_READ" VAD pages to detect @@ -135,12 +134,12 @@ class Malfind(interfaces.plugins.PluginInterface): try: # If we have a dirty page in a non writable "EXECUTE" region, it is suspicious. if proc_layer.is_dirty(page): - dirty_page_check = True + dirty_page = page break except exceptions.InvalidAddressException: # Abort as it is likely that other addresses in the same range will also fail. break - if not dirty_page_check: + if dirty_page is None: continue else: continue @@ -152,10 +151,10 @@ class Malfind(interfaces.plugins.PluginInterface): if cls.is_vad_empty(proc_layer, vad): continue - if dirty_page_check: + if dirty_page is not None: # Useful information to investigate the page content with volshell afterwards. vollog.warning( - f"[proc_id {proc_id}] Found suspicious DIRTY + {protection_string} page at {hex(page)}", + f"[proc_id {proc_id}] Found suspicious DIRTY + {protection_string} page at {hex(dirty_page)}", ) data = proc_layer.read(vad.get_start(), 64, pad=True) yield vad, data From 675ef6deea00cf4659bca50e0004dea3be43e8c1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:01:09 +0000 Subject: [PATCH 233/348] Generic: Fix up potential issue with isfinfo Fixes #1436 --- volatility3/framework/plugins/isfinfo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 78e78fb9e..1c2ac52e9 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -132,6 +132,7 @@ class IsfInfo(plugins.PluginInterface): valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {entry}") + continue yield ( 0, ( From f6f4c7b986c8c2adf7a73211f7aeb1375b00f255 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:11:42 +0000 Subject: [PATCH 234/348] Layers: Fix MSF page len on a possibly uninitialized variable Fixes #1441 --- volatility3/framework/layers/msf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 03e144e25..2b4fae963 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -194,7 +194,7 @@ class PdbMSFStream(linear.LinearlyMappedLayer): ) -> None: super().__init__(context, config_path, name, metadata) self._base_layer = self.config["base_layer"] - self._pages = self.config.get("pages", None) + self._pages = self.config.get("pages", []) self._pages_len = len(self._pages) if not self._pages: raise PDBFormatException(name, "Invalid/no pages specified") From eaca7b31bba1c9d8f934ad3e607fced26797abfd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:16:17 +0000 Subject: [PATCH 235/348] Layers: Fix vmware layer without a suitable meta Fixes #1442 --- volatility3/framework/layers/vmware.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 622ff0250..39fb21b63 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -57,6 +57,10 @@ class VmwareLayer(segmented.SegmentedLayer): ) meta_layer = self.context.layers.get(self._meta_layer, None) + if meta_layer is None: + raise exceptions.LayerException( + self._meta_layer, "VMware: Meta layer not found" + ) header_size = struct.calcsize(self.header_structure) data = meta_layer.read(0, header_size) magic, unknown, groupCount = struct.unpack(self.header_structure, data) From cc8fbd5b6824231d5f7bfb53a66560cb2e3fe3b1 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:34:52 +0000 Subject: [PATCH 236/348] Tweak a comment --- volatility3/framework/plugins/windows/svclist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index ea73247ce..a5825e1fe 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -41,7 +41,7 @@ class SvcList(svcscan.SvcScan): @classmethod def _get_exe_range(cls, proc) -> Optional[Tuple[int, int]]: """ - Returns a tuple of starting,ending address for + Returns a tuple of starting address and size of the the VAD containing services.exe """ From bed03dfbc7024d97c289b0c26f98d0627e105eee Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 21 Dec 2024 06:15:35 +0000 Subject: [PATCH 237/348] Refactor check of BasicType The intention is either a BasicType or a list where each element is only a BasicType (and not a list). --- volatility3/cli/volshell/generic.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 08132608b..a4b141c2d 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -554,11 +554,15 @@ class Volshell(interfaces.plugins.PluginInterface): del kwargs[argname] for keyword, val in kwargs.items(): - if not isinstance(val, (interfaces.configuration.BasicTypes, list)): - if all(isinstance(x, interfaces.configuration.BasicTypes) for x in val): - raise TypeError( - "Configurable values must be simple types (int, bool, str, bytes)" - ) + BasicType_or_list_of_BasicType = False # excludes list of lists + if isinstance(val, interfaces.configuration.BasicTypes): + BasicType_or_list_of_BasicType = True + if all(isinstance(x, interfaces.configuration.BasicTypes) for x in val): + BasicType_or_list_of_BasicType = True + if not BasicType_or_list_of_BasicType: + raise TypeError( + "Configurable values must be simple types (int, bool, str, bytes)" + ) self.context.config[config_path + "." + keyword] = val constructed = clazz(self.context, config_path, **constructor_args) From 88eb2aa88648180439e4d5311c9bf49ef2ae9363 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 21 Dec 2024 13:22:04 +0100 Subject: [PATCH 238/348] switch pillow to pyproject.toml --- pyproject.toml | 78 ++++++++++++++++++++++++++++++++++++++++++++---- requirements.txt | 29 ------------------ 2 files changed, 72 insertions(+), 35 deletions(-) delete mode 100644 requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 2e1636a43..c8b2971e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,13 +8,55 @@ authors = [ ] requires-python = ">=3.8.0" license = { text = "VSL" } -dynamic = ["dependencies", "optional-dependencies", "version"] +dynamic = ["version"] + +dependencies = [ + "pefile>=2024.8.26", +] + +[project.optional-dependencies] +full = [ + "yara-python>=4.5.1,<5", + "capstone>=5.0.3,<6", + "pycryptodome>=3.21.0,<4", + "leechcorepyc>=2.19.2,<3; sys_platform != 'darwin'", + # https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst + # 10.0.0 dropped support for Python3.7 + # 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 + "pillow>=10.0.0,<11.0.0", +] + +cloud = [ + "gcsfs>=2024.10.0", + "s3fs>=2024.10.0", +] + +dev = [ + "volatility3[full,cloud]", + "jsonschema>=4.23.0,<5", + "pyinstaller>=6.11.0,<7", + "pyinstaller-hooks-contrib>=2024.9", +] + +test = [ + "volatility3[dev]", + "pytest>=8.3.3,<9", + "capstone>=5.0.3,<6", + "yara-x>=0.10.0,<1", +] + +docs = [ + "volatility3[dev]", + "sphinx>=8.0.0,<7", + "sphinx-autodoc-typehints>=2.5.0,<3", + "sphinx-rtd-theme>=3.0.1,<4", +] [project.urls] -Homepage = "https://github.com/volatilityfoundation/volatility3/" -"Bug Tracker" = "https://github.com/volatilityfoundation/volatility3/issues" -Documentation = "https://volatility3.readthedocs.io/" -"Source Code" = "https://github.com/volatilityfoundation/volatility3" +homepage = "https://github.com/volatilityfoundation/volatility3/" +documentation = "https://volatility3.readthedocs.io/" +repository = "https://github.com/volatilityfoundation/volatility3" +issues = "https://github.com/volatilityfoundation/volatility3/issues" [project.scripts] vol = "volatility3.cli:main" @@ -22,11 +64,35 @@ volshell = "volatility3.cli.volshell:main" [tool.setuptools.dynamic] version = { attr = "volatility3.framework.constants._version.PACKAGE_VERSION" } -dependencies = { file = "requirements-minimal.txt" } [tool.setuptools.packages.find] include = ["volatility3*"] +[tool.mypy] +mypy_path = "./stubs" +show_traceback = true + +[tool.mypy.overrides] +ignore_missing_imports = true + +[tool.ruff] +line-length = 88 +target-version = "py38" + +[tool.ruff.lint] +select = [ + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade +] + +ignore = [ + "E501", # ignore due to conflict with formatter +] + [build-system] requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 21c8f9a76..000000000 --- a/requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# Include the minimal requirements --r requirements-minimal.txt - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 - -# This is required for several plugins that perform malware analysis and disassemble code. -# It can also improve accuracy of Windows 8 and later memory samples. -# FIXME: Version 6.0.0 is incompatible (#1336) so we'll need an adaptor at some point -capstone>=3.0.5,<6.0.0 - -# This is required by plugins that decrypt passwords, password hashes, etc. -pycryptodome - -# This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0; sys_platform != 'darwin' - -# This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage -gcsfs>=2023.1.0 -s3fs>=2023.1.0 - -# This is required by plugins that manipulate pixels and images. -# https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst -# 10.0.0 dropped support for Python3.7 -# 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 -pillow>=10.0.0,<11.0.0 \ No newline at end of file From f6a54c5c48b6ba47a334956cb7994fecfcf14f0c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 21 Dec 2024 13:31:29 +0100 Subject: [PATCH 239/348] ruff fix --- volatility3/framework/plugins/linux/graphics/fbdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 60e00d033..7144e081a 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -218,7 +218,7 @@ class Fbdev(interfaces.plugins.PluginInterface): fourcc = linux.LinuxUtilities.convert_fourcc_code(fb_info.var.grayscale) warn_msg = f"""Framebuffer "{id}" uses a FOURCC pixel format "{fourcc}" that isn't natively supported. You can try using ffmpeg to decode the raw buffer. Example usage: -"ffmpeg -pix_fmts" to list supported formats, then +"ffmpeg -pix_fmts" to list supported formats, then "ffmpeg -f rawvideo -video_size {fb_info.var.xres_virtual}x{fb_info.var.yres_virtual} -i .raw -pix_fmt output.png".""" vollog.warning(warn_msg) From a84d3611130b1ba42e1e3c317bc6f633adc669ab Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 21 Dec 2024 12:18:23 -0600 Subject: [PATCH 240/348] Add the suspended threads plugin from DEF CON 2024 --- .../plugins/windows/suspended_threads.py | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 volatility3/framework/plugins/windows/suspended_threads.py diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py new file mode 100644 index 000000000..2cc0673d7 --- /dev/null +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -0,0 +1,147 @@ +import logging + +from typing import Dict +from functools import partial + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +import volatility3.plugins.windows.pslist as pslist +import volatility3.plugins.windows.threads as threads +import volatility3.plugins.windows.pe_symbols as pe_symbols + +from volatility3.framework.objects import utility + +vollog = logging.getLogger(__name__) + + +class SuspendedThreads(interfaces.plugins.PluginInterface): + """Enumerates suspended threads.""" + + _required_framework_version = (2, 13, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="threads", component=threads.Threads, version=(1, 0, 0) + ), + ] + + def _generator(self): + """ + The goal of this plugin is to report on threads that are suspended + + Legitimate programs can start threads suspended but then will later resume them + + Subsets of malware techniques, such as EDR evasion and process hollowing, + create suspended threads and do not resume them. These are the threads that this + plugin is designed to catch. + + See the whitepaper from our DEF CON 2024 presentation for more details: + + https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + """ + kernel = self.context.modules[self.config["kernel"]] + + vads_cache: Dict[int, pe_symbols.PESymbols.ranges_type] = {} + + proc_modules = None + + # walk the threads of each process checking for suspended threads + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ): + for thread in threads.Threads.list_threads(kernel, proc): + try: + # we only care if the thread is suspended + if thread.Tcb.SuspendCount == 0: + continue + + # 4 == terminated + if thread.Tcb.State == 4: + continue + + owner_proc = thread.owning_process() + owner_proc_pid = thread.Cid.UniqueProcess + owner_proc_name = utility.array_to_string(owner_proc.ImageFileName) + thread_tid = thread.Cid.UniqueThread + thread_start_addr = thread.StartAddress + thread_win32_addr = thread.Win32StartAddress + except exceptions.InvalidAddressException: + continue + + # Nothing useful to report if a process doesn't have VADs.. Also a sign of smear/terminated + vads = pe_symbols.PESymbols.get_vads_for_process_cache(vads_cache, owner_proc) + if not vads: + continue + + # Only compute this if needed as its expensive and 99.9% of samples + # will not have suspended threads + if not proc_modules: + proc_modules = pe_symbols.PESymbols.get_process_modules( + self.context, kernel.layer_name, kernel.symbol_table_name, None + ) + + path_and_symbol = partial( + pe_symbols.PESymbols.path_and_symbol_for_address, + self.context, + self.config_path, + proc_modules, + ) + + start_file, start_sym = path_and_symbol(vads, thread_start_addr) + win32_file, win32_sym = path_and_symbol(vads, thread_win32_addr) + + # the only false positive found in mass scanning of samples + if start_file and start_file.endswith("\\WorkFoldersShell.dll"): + continue + + if win32_file and win32_file.endswith("\\WorkFoldersShell.dll"): + continue + + yield ( + 0, + ( + owner_proc_name, + owner_proc_pid, + thread_tid, + start_file or renderers.NotAvailableValue(), + start_sym or renderers.NotAvailableValue(), + format_hints.Hex(thread_start_addr), + win32_file or renderers.NotAvailableValue(), + win32_sym or renderers.NotAvailableValue(), + format_hints.Hex(thread_win32_addr), + ), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("TID", int), + ("StartFile", str), + ("StartSymbol", str), + ("StartAddress", format_hints.Hex), + ("Win32StartFile", str), + ("Win32StartSymbol", str), + ("Win32StartAddress", format_hints.Hex), + ], + self._generator(), + ) + From d31ac276edb29721847f248c13953696c4a98a9c Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 21 Dec 2024 12:22:33 -0600 Subject: [PATCH 241/348] Add the suspended threads plugin from DEF CON 2024 --- volatility3/framework/plugins/windows/suspended_threads.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py index 2cc0673d7..6ddfecca7 100644 --- a/volatility3/framework/plugins/windows/suspended_threads.py +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -86,7 +86,9 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): continue # Nothing useful to report if a process doesn't have VADs.. Also a sign of smear/terminated - vads = pe_symbols.PESymbols.get_vads_for_process_cache(vads_cache, owner_proc) + vads = pe_symbols.PESymbols.get_vads_for_process_cache( + vads_cache, owner_proc + ) if not vads: continue @@ -144,4 +146,3 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): ], self._generator(), ) - From b9fa217d7980042aa7264efbdafee6ce4daa930f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 21 Dec 2024 18:48:12 +0000 Subject: [PATCH 242/348] Add a required framework version Added _required_framework_version and set it to the same value (2, 0, 0) as its plugin requirement of svcscan. Also tweaked one comment. --- volatility3/framework/plugins/windows/svclist.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index a5825e1fe..8a64084c5 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -18,6 +18,7 @@ vollog = logging.getLogger(__name__) class SvcList(svcscan.SvcScan): """Lists services contained with the services.exe doubly linked list of services""" + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) def __init__(self, *args, **kwargs): @@ -41,7 +42,7 @@ class SvcList(svcscan.SvcScan): @classmethod def _get_exe_range(cls, proc) -> Optional[Tuple[int, int]]: """ - Returns a tuple of starting address and size of the + Returns a tuple of starting address and size of the VAD containing services.exe """ From ea273fe878fd5724f1801fd709a805bfa92d7ce0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 21 Dec 2024 23:23:37 +0000 Subject: [PATCH 243/348] Volshell: Fix up shuffled imports --- volatility3/cli/volshell/generic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 90d73b4ac..2321408fe 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -11,6 +11,11 @@ import sys from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union from urllib import parse, request +from volatility3.cli import text_renderer, volshell +from volatility3.framework import exceptions, interfaces, objects, plugins, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import intel, physical, resources, scanners + try: import capstone @@ -18,11 +23,6 @@ try: except ImportError: has_capstone = False -from volatility3.cli import text_renderer, volshell -from volatility3.framework import exceptions, interfaces, objects, plugins, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.layers import intel, physical, resources, scanners - class Volshell(interfaces.plugins.PluginInterface): """Shell environment to directly interact with a memory image.""" From 7caf8c572a4629a2a235a974e6dfff112ccabecd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:03:26 +0100 Subject: [PATCH 244/348] PIL import graceful exit --- .../framework/plugins/linux/graphics/fbdev.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 7144e081a..28cae8000 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -4,9 +4,6 @@ import logging import io -# Image manipulation functions are kept in the plugin, -# to prevent a general exit on missing PIL (pillow) dependency. -from PIL import Image from dataclasses import dataclass from typing import Type, List, Dict, Tuple from volatility3.framework import constants, exceptions, interfaces @@ -16,6 +13,15 @@ from volatility3.framework.objects import utility from volatility3.framework.constants import architectures from volatility3.framework.symbols import linux +# Image manipulation functions are kept in the plugin, +# to prevent a general exit on missing PIL (pillow) dependency. +try: + from PIL import Image + + has_pil = True +except ImportError: + has_pil = False + vollog = logging.getLogger(__name__) @@ -101,7 +107,7 @@ class Fbdev(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, kernel_name: str, fb: Framebuffer, - ) -> Image.Image: + ): """Convert raw framebuffer pixels to an image. Args: @@ -238,6 +244,13 @@ You can try using ffmpeg to decode the raw buffer. Example usage: return fb def _generator(self): + + if not has_pil: + vollog.error( + "PIL (pillow) module is required to use this plugin. Please install it manually or through pyproject.toml." + ) + return None + kernel_name = self.config["kernel"] kernel = self.context.modules[kernel_name] From 1fef570022eb0ae13924ed321e5c09a24fe72563 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:15:45 +0100 Subject: [PATCH 245/348] restrict output to PNG, unify file handling --- .../framework/plugins/linux/graphics/fbdev.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 28cae8000..e82827944 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -160,15 +160,13 @@ class Fbdev(interfaces.plugins.PluginInterface): kernel_name: str, open_method: Type[interfaces.plugins.FileHandlerInterface], fb: Framebuffer, - convert_to_image: bool, - image_format: str = "PNG", + convert_to_png_image: bool, ) -> str: - """Dump a Framebuffer raw buffer to disk. + """Dump a Framebuffer buffer to disk. Args: fb: the relevant Framebuffer object convert_to_image: a boolean specifying if the buffer should be converted to an image - image_format: the target PIL image format (defaults to PNG) Returns: The filename of the dumped buffer. @@ -176,19 +174,19 @@ class Fbdev(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_name] kernel_layer = context.layers[kernel.layer_name] base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" - if convert_to_image: - image = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) - output = io.BytesIO() - image.save(output, image_format) - file_handle = open_method(f"{base_filename}.{image_format.lower()}") - file_handle.write(output.getvalue()) + if convert_to_png_image: + image_object = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) + raw_io_output = io.BytesIO() + image_object.save(raw_io_output, "PNG") + final_fb_buffer = raw_io_output.getvalue() + filename = f"{base_filename}.png" else: - raw_pixels = kernel_layer.read(fb.fb_info.screen_base, fb.size) - file_handle = open_method(f"{base_filename}.raw") - file_handle.write(raw_pixels) + final_fb_buffer = kernel_layer.read(fb.fb_info.screen_base, fb.size) + filename = f"{base_filename}.raw" - file_handle.close() - return file_handle.preferred_filename + with open_method(filename) as f: + f.write(final_fb_buffer) + return f.preferred_filename @classmethod def parse_fb_info( From 8d213284e642c545f44502d0fab3f026bc4fa0ff Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:23:04 +0100 Subject: [PATCH 246/348] handle NotAvailableValue in filename --- volatility3/framework/plugins/linux/graphics/fbdev.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index e82827944..7f7deee4e 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -173,7 +173,8 @@ class Fbdev(interfaces.plugins.PluginInterface): """ kernel = context.modules[kernel_name] kernel_layer = context.layers[kernel.layer_name] - base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" + id = "N-A" if isinstance(fb.id, NotAvailableValue) else fb.id + base_filename = f"{id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" if convert_to_png_image: image_object = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) raw_io_output = io.BytesIO() @@ -207,8 +208,7 @@ class Fbdev(interfaces.plugins.PluginInterface): - struct fb_var_screeninfo stores device independent changeable information about a frame buffer device, its current format and video mode, as well as other miscellaneous parameters. """ - # NotAvailableValue() messes with the filename output on disk - id = utility.array_to_string(fb_info.fix.id) or "N-A" + id = utility.array_to_string(fb_info.fix.id) or NotAvailableValue() color_fields = None # 0 = color, 1 = grayscale, >1 = FOURCC From f3d7647433a727a5bb7bc8c91fa3803ad44a6bf4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 15:48:29 +0100 Subject: [PATCH 247/348] unify Tainting parsing capabilities --- .../framework/symbols/linux/__init__.py | 121 ++++++++++++++++++ .../symbols/linux/extensions/__init__.py | 74 ++--------- 2 files changed, 131 insertions(+), 64 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0230a9c48..832b1de9b 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -11,6 +11,7 @@ from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions +from volatility3.framework.constants import linux as linux_constants class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -830,3 +831,123 @@ class PageCache: page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page + + +class Tainting: + """Tainted kernel and modules parsing capabilities. + + Relevant kernel functions: + - modules: module_flags_taint + - kernel: print_tainted + """ + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.kernel = context.modules[kernel_module_name] + + @property + def kernel_taint_flags_list( + self, + ) -> Optional[List[interfaces.objects.ObjectInterface]]: + if self.kernel.has_symbol("taint_flags"): + return list(self.kernel.object_from_symbol("taint_flags")) + return None + + def _module_flags_taint_pre_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if is_module and is_module != taint_flag.module: + continue + + if taints & taint_flag.shift: + taints_string += char + + return taints_string + + def _module_flags_taint_post_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self.kernel_taint_flags_list): + if is_module and is_module != taint_flag.module: + continue + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taints & (1 << i): + taints_string += c_true + elif c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + """Convert the taints value to a 1-1 character mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + s + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self.kernel_taint_flags_list: + return self._module_flags_taint_post_4_10_rc1(taints, is_module) + return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + + def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + """Convert the taints string to a 1-1 descriptor mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for character in self.get_taints_as_plain_string(taints, is_module): + taint_flag = linux_constants.TAINT_FLAGS.get(character) + if not taint_flag: + comprehensive_taints.append(f"") + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) + + return comprehensive_taints diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0ecf731f4..075a83ae8 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -279,76 +279,29 @@ class module(generic.GenericIntelProcess): return None - def _module_flags_taints_pre_4_10_rc1(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on statically defined taints mappings in the framework. - - Returns: - The raw taints string. - """ - taints_string = "" - for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if taint_flag.module and self.taints & taint_flag.shift: - taints_string += char - - return taints_string - - def _module_flags_taints_post_4_10_rc1(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on kernel symbol embedded taints definitions. - - struct taint_flag { - char c_true; /* character printed when tainted */ - char c_false; /* character printed when not tainted */ - bool module; /* also show as a per-module taint flag */ - }; - - Returns: - The raw taints string. - """ - taints_string = "" - for i, taint_flag in enumerate(self.taint_flags_list): - c_true = chr(taint_flag.c_true) - c_false = chr(taint_flag.c_false) - if taint_flag.module and (self.taints & (1 << i)): - taints_string += c_true - elif taint_flag.module and c_false != " ": - taints_string += c_false - - return taints_string - def get_taints_as_plain_string(self) -> str: """Convert the module's taints value to a 1-1 character mapping. + Convenient wrapper around framework's Tainting capabilities. Returns: The raw taints string. - - Documentation: - - module_flags_taint kernel function """ - - if self.taint_flags_list: - return self._module_flags_taints_post_4_10_rc1() - return self._module_flags_taints_pre_4_10_rc1() + return linux.Tainting( + self._context, + linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, + ).get_taints_as_plain_string(self.taints, True) def get_taints_parsed(self) -> List[str]: """Convert the module's taints string to a 1-1 descriptor mapping. + Convenient wrapper around framework's Tainting capabilities. Returns: A comprehensive (user-friendly) taint descriptor list. - - Documentation: - - module_flags_taint kernel function """ - comprehensive_taints = [] - for character in self.get_taints_as_plain_string(): - taint_flag = linux_constants.TAINT_FLAGS.get(character) - if not taint_flag: - comprehensive_taints.append(f"") - elif taint_flag.when_present: - comprehensive_taints.append(taint_flag.desc) - - return comprehensive_taints + return linux.Tainting( + self._context, + linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, + ).get_taints_parsed(self.taints, True) @property def section_symtab(self): @@ -376,13 +329,6 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("Unable to get strtab") - @property - def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: - kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - if kernel.has_symbol("taint_flags"): - return list(kernel.object_from_symbol("taint_flags")) - return None - class task_struct(generic.GenericIntelProcess): def add_process_layer( From 9d4dd010a7eb6212effc43dd8d57f86e127684f2 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:28:40 +0000 Subject: [PATCH 248/348] Reformat how to write a simple plugin --- doc/source/simple-plugin.rst | 93 ++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 84b921114..aa8ec3a7e 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -41,15 +41,24 @@ to be able to run properly. Any that are defined as optional need not necessari @classmethod def get_requirements(cls): - return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True), - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (2, 0, 0))] + return [ + requirements.ModuleRequirement( + name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"] + ), + requirements.ListRequirement( + name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True + ), + requirements.PluginRequirement( + name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0) + ), + ] This is a classmethod, so it can be called before the specific plugin object has been instantiated (in order to know how @@ -57,8 +66,11 @@ to instantiate the plugin). At the moment these requirements are fairly straigh :: - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"] + ), This requirement specifies the need for a particular submodule. Each module requires a :py:class:`TranslationLayer ` and a @@ -85,9 +97,11 @@ not be requested directly from the user. :: - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.TranslationLayerRequirement( + name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"] + ), This requirement indicates that the plugin will operate on a single :py:class:`TranslationLayer `. The name of the @@ -110,8 +124,10 @@ not be requested directly from the user. :: - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + requirements.SymbolTableRequirement( + name = "nt_symbols", + description = "Windows kernel symbols" + ), This requirement specifies the need for a particular :py:class:`SymbolTable ` @@ -127,10 +143,12 @@ not be requested directly from the user. :: - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True), + requirements.ListRequirement( + name = 'pid', + description = 'Filter on specific process IDs', + element_type = int, + optional = True + ), The next requirement is a List Requirement, populated by integers. The description will be presented to the user to describe what the value represents. The optional flag indicates that the plugin can function without the ``pid`` value @@ -138,9 +156,11 @@ being defined within the configuration tree at all. :: - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (2, 0, 0))] + requirements.PluginRequirement( + name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0) + ) This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements on that plugin. The version is specified in terms of Semantic Versioning meaning that, to be compatible, the major @@ -180,16 +200,21 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) kernel = self.context.modules[self.config['kernel']] - return renderers.TreeGrid([("PID", int), - ("Process", str), - ("Base", format_hints.Hex), - ("Size", format_hints.Hex), - ("Name", str), - ("Path", str)], - self._generator(pslist.PsList.list_processes(self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Base", format_hints.Hex), + ("Size", format_hints.Hex), + ("Name", str), + ("Path", str), + ], + self._generator( + pslist.PsList.list_processes( + self.context, kernel.layer_name, kernel.symbol_table_name, filter_func = filter_func + ) + ) + ) In this instance, the plugin constructs a filter (using the PsList plugin's *classmethod* for creating filters). It checks the plugin's configuration for the ``pid`` value, and passes it in as a list if it finds it, or None if @@ -281,5 +306,3 @@ such as ``!_UNICODE``) and the parameters to that type. Since the cast value must populate a string typed column, it had to be a Python string (such as being cast to the native type string) and could not have been a special Structure such as ``_UNICODE``. For the format hint columns, the format hint type must be used to ensure the error checking does not fail. - - From df37f0a909255410bd5df31cd4d16bdabb1aa9e8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:34:17 +0000 Subject: [PATCH 249/348] Reformat how to write a simple plugin --- doc/source/simple-plugin.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index aa8ec3a7e..07d9e1467 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -211,7 +211,10 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. ], self._generator( pslist.PsList.list_processes( - self.context, kernel.layer_name, kernel.symbol_table_name, filter_func = filter_func + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func = filter_func ) ) ) From 0bb09191aae08b2a1b481fef4fbd565e07a7d91f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 23 Dec 2024 21:28:58 +0100 Subject: [PATCH 250/348] file output failure results in UnreadableValue --- .../framework/plugins/linux/graphics/fbdev.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 7f7deee4e..ab4289cf1 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -8,7 +8,12 @@ from dataclasses import dataclass from typing import Type, List, Dict, Tuple from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.renderers import ( + format_hints, + TreeGrid, + NotAvailableValue, + UnreadableValue, +) from volatility3.framework.objects import utility from volatility3.framework.constants import architectures from volatility3.framework.symbols import linux @@ -280,11 +285,12 @@ You can try using ffmpeg to decode the raw buffer. Example usage: file_output = self.dump_fb( self.context, kernel_name, self.open, fb, bool(fb.color_fields) ) + file_output = str(file_output) except exceptions.InvalidAddressException as excp: vollog.error( f'Layer {excp.layer_name} failed to read address {hex(excp.invalid_address)} when dumping framebuffer "{fb.id}".' ) - file_output = "Error" + file_output = UnreadableValue() try: fb_device_name = utility.pointer_to_string( @@ -303,7 +309,7 @@ You can try using ffmpeg to decode the raw buffer. Example usage: f"{fb.xres_virtual}x{fb.yres_virtual}", fb.bpp, "RUNNING" if fb.fb_info.state == 0 else "SUSPENDED", - str(file_output), + file_output, ), ) From ea2757c06ce23d9a24e01e2ce7823e4980889ee8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 24 Dec 2024 11:45:23 +0100 Subject: [PATCH 251/348] minor version bump --- volatility3/framework/constants/_version.py | 2 +- volatility3/framework/plugins/linux/graphics/fbdev.py | 3 +++ volatility3/framework/symbols/linux/__init__.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 11edc07d8..9ca2d0a5b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 13 # Number of changes that only add to the interface +VERSION_MINOR = 14 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index ab4289cf1..7b644eccf 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -60,6 +60,9 @@ class Fbdev(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=architectures.LINUX_ARCHS, ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + ), requirements.BooleanRequirement( name="dump", description="Dump framebuffers", diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index ba223f979..5aa27b964 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -76,7 +76,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 1) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 79fe1c50dfcda812cd9d4b307b271ddcc3c26bd9 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 24 Dec 2024 19:42:05 +0000 Subject: [PATCH 252/348] Tweak configuration.py Create a tuple directly and replace random.choice by random.choices. --- volatility3/framework/interfaces/configuration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 2e4f580a7..a376fa813 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -53,7 +53,7 @@ ConfigSimpleType = Optional[Union[SimpleTypes, List[SimpleTypes]]] def path_join(*args) -> str: """Joins configuration paths together.""" # If a path element (particularly the first) is empty, then remove it from the list - args = tuple([arg for arg in args if arg]) + args = tuple(arg for arg in args if arg) return CONFIG_SEPARATOR.join(args) @@ -772,8 +772,7 @@ class ConfigurableInterface(metaclass=ABCMeta): str: The newly generated full configuration path """ random_config_dict = "".join( - random.SystemRandom().choice(string.ascii_uppercase + string.digits) - for _ in range(8) + random.SystemRandom().choices(string.ascii_uppercase + string.digits, k=8) ) new_config_path = path_join(base_config_path, random_config_dict) # TODO: Check that the new_config_path is empty, although it's not critical if it's not since the values are merged in From 4c430d2ec464b3e1fdf8ddd9b51fdf4525078814 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 26 Dec 2024 07:27:45 +0000 Subject: [PATCH 253/348] Use BasicTypes variable This removes a "float", which should be excluded. --- volatility3/framework/interfaces/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index a376fa813..b6f4f889c 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -779,9 +779,9 @@ class ConfigurableInterface(metaclass=ABCMeta): # This should check that each k corresponds to a requirement and each v is of the appropriate type # This would require knowledge of the new configurable itself to verify, and they should do validation in the - # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a simple type + # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a basic type for k, v in kwargs.items(): - if not isinstance(v, (int, str, bool, float, bytes)): + if not isinstance(v, BasicTypes): raise TypeError( "Config values passed to make_subconfig can only be simple types" ) From 834b7d0f072984f232dfc7880c0214b40df424fb Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 28 Dec 2024 14:58:12 +0000 Subject: [PATCH 254/348] Make ETHREAD year check dynamic Change upper bound year check of ETHREAD to be a decade from now. Makes consistent with EPROCESS. --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d63f138b6..5ec84f95f 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -519,7 +519,8 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False - if not (1998 < ctime.year < 2030): + current_year = datetime.datetime.now().year + if not (1998 < ctime.year < current_year + 10): return False except exceptions.InvalidAddressException: From 37873f9e1593fad055b0319085694d67a9568f4b Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 28 Dec 2024 16:58:24 +0000 Subject: [PATCH 255/348] Remove superfluous spaces in intermed.py --- volatility3/framework/symbols/intermed.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 5b4aa22b8..6802af7d6 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -101,7 +101,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): Args: context: The volatility context for the symbol table config_path: The configuration path for the symbol table - name: The name for the symbol table (this is used in symbols e.g. table!symbol ) + name: The name for the symbol table (this is used in symbols e.g. table!symbol) isf_url: The URL pointing to the ISF file location native_types: The NativeSymbolTable that contains the native types for this symbol table table_mapping: A dictionary linking names referenced in the file with symbol tables in the context @@ -111,7 +111,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): """ # Check there are no obvious errors # Open the file and test the version - self._versions = dict([(x.version, x) for x in class_subclasses(ISFormatTable)]) + self._versions = dict((x.version, x) for x in class_subclasses(ISFormatTable)) with resources.ResourceAccessor().open(isf_url) as fp: reader = codecs.getreader("utf-8") json_object = json.load(reader(fp)) # type: ignore @@ -166,9 +166,9 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): format. An interface version such as Major.Minor.Patch means that Major - of the provider must be equal to that of the consumer, and the + of the provider must be equal to that of the consumer, and the provider (the JSON in this instance) must have a greater minor - (indicating that only additive changes have been made) than + (indicating that only additive changes have been made) than the consumer (in this case, the file reader). """ major, minor, patch = (int(x) for x in version.split(".")) From 1bd031b9a86677f6b234e13f93ab2ad9feeb2cc8 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:42:25 +0000 Subject: [PATCH 256/348] Prevent infinite loops in device enumeration extensions #1483 --- .../symbols/windows/extensions/__init__.py | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d63f138b6..6dec08c38 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -405,11 +405,24 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the attached device's objects""" - device = self.AttachedDevice.dereference() - while device: - yield device - device = device.AttachedDevice.dereference() + seen = set() + try: + device = self.AttachedDevice.dereference() + except exceptions.InvalidAddressException: + return + + while device: + if device.vol.offset in seen: + break + seen.add(device.vol.offset) + + yield device + + try: + device = device.AttachedDevice.dereference() + except exceptions.InvalidAddressException: + return class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" @@ -421,10 +434,24 @@ class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" - device = self.DeviceObject.dereference() + seen = set() + + try: + device = self.DeviceObject.dereference() + except exceptions.InvalidAddressException: + return + while device: + if device.vol.offset in seen: + return + seen.add(device.vol.offset) + yield device - device = device.NextDevice.dereference() + + try: + device = device.NextDevice.dereference() + except exceptions.InvalidAddressException: + return def is_valid(self) -> bool: """Determine if the object is valid.""" From bf7f1ca91ed88bf482b19bd2558d79aa70bb5c0e Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:43:56 +0000 Subject: [PATCH 257/348] Prevent infinite loops in device enumeration extensions #1483 --- volatility3/framework/symbols/windows/extensions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 6dec08c38..c38d47f73 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -424,6 +424,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): except exceptions.InvalidAddressException: return + class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" From e64af61efa1a37f6d5c91e34d5375219b1544ee3 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:13:18 +0000 Subject: [PATCH 258/348] Do not analyze processes without VADs #1470 --- volatility3/framework/plugins/windows/direct_system_calls.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index b0c162f46..183e4095c 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -433,6 +433,8 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] vads = self.get_vad_maps(proc) + if not vads: + continue # for each valid process, look for malicious syscall invocations for address, vad_path in self._get_rule_hits( From 33855cf920a8ef76d2bfa414779b7cf96c8c8def Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:20:02 +0000 Subject: [PATCH 259/348] Significantly improve the smear/error handling in the netstat plugin --- .../framework/plugins/windows/netstat.py | 157 +++++++++++++----- .../symbols/windows/extensions/network.py | 8 +- 2 files changed, 120 insertions(+), 45 deletions(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index a1521a8c6..3408c0a3a 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -111,8 +111,21 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The list of indices at which a 1 was found. """ ret = [] + # This value is broken in many samples and was causing essentially infinite loops + # Testing showed that 8192 is the current size across all Windows versions + # We give some leeway in case it increases in later versions, while still keeping it sane + # The problematic samples had values that looked like addresses, so in the billions + if bitmap_size_in_byte > 8192 * 10: + return ret + for idx in range(bitmap_size_in_byte): - current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[0] + try: + current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[ + 0 + ] + except exceptions.InvalidAddressException: + continue + current_offs = idx * 8 for bit in range(8): if current_byte & (1 << bit) != 0: @@ -154,32 +167,37 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) else: # invalid argument. - return None + return vollog.debug(f"Current Port: {port}") # the given port serves as a shifted index into the port pool lists list_index = port >> 8 truncated_port = port & 0xFF - # constructing port_pool object here so callers don't have to - port_pool = context.object( - net_symbol_table + constants.BANG + "_INET_PORT_POOL", - layer_name=layer_name, - offset=port_pool_addr, - ) + try: + # constructing port_pool object here so callers don't have to + port_pool = context.object( + net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name=layer_name, + offset=port_pool_addr, + ) + # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) + inpa = port_pool.PortAssignments[list_index] - # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) - inpa = port_pool.PortAssignments[list_index] - - # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry - assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry + assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + except exceptions.InvalidAddressException: + return if not assignment: - return None + return # the value within assignment.Entry is a) masked and b) points inside of the network object # first decode the pointer - netw_inside = cls._decode_pointer(assignment.Entry) + try: + netw_inside = cls._decode_pointer(assignment.Entry) + except exceptions.InvalidAddressException: + return if netw_inside: # if the value is valid, calculate the actual object address by subtracting the offset @@ -188,16 +206,30 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) yield curr_obj + try: + next_obj_address = cls._decode_pointer(curr_obj.Next) + except exceptions.InvalidAddressException: + return + # if the same port is used on different interfaces multiple objects are created # those can be found by following the pointer within the object's `Next` field until it is empty - while curr_obj.Next: - curr_obj = context.object( - obj_name, - layer_name=layer_name, - offset=cls._decode_pointer(curr_obj.Next) - ptr_offset, - ) + while next_obj_address: + try: + curr_obj = context.object( + obj_name, + layer_name=layer_name, + offset=next_obj_address - ptr_offset, + ) + except exceptions.InvalidAddressException: + return + yield curr_obj + try: + next_obj_address = cls._decode_pointer(curr_obj.Next) + except exceptions.InvalidAddressException: + return + @classmethod def get_tcpip_module( cls, @@ -243,16 +275,25 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The hash table entries which are _not_ empty """ # we are looking for entries whose values are not their own address + # smear sanity check from mass testing + if ht_length > 4096: + return + for index in range(ht_length): current_addr = ht_offset + index * alignment - current_pointer = context.object( - net_symbol_table + constants.BANG + "pointer", - layer_name=layer_name, - offset=current_addr, - ) + try: + current_pointer = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=current_addr, + ) + except exceptions.InvalidAddressException: + continue + # check if addr of pointer is equal to the value pointed to if current_pointer.vol.offset == current_pointer: continue + yield current_pointer @classmethod @@ -292,11 +333,15 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): tcpip_symbol_table + constants.BANG + "PartitionCount" ).address - part_table_addr = context.object( - net_symbol_table + constants.BANG + "pointer", - layer_name=layer_name, - offset=tcpip_module_offset + part_table_symbol, - ) + try: + part_table_addr = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=tcpip_module_offset + part_table_symbol, + ) + except exceptions.InvalidAddressException: + vollog.debug(f"`PartitionTable` not present in memory.") + return # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects part_table = context.object( @@ -304,10 +349,18 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): layer_name=layer_name, offset=part_table_addr, ) - part_count = int.from_bytes( - context.layers[layer_name].read(tcpip_module_offset + part_count_symbol, 1), - "little", - ) + + try: + part_count = int.from_bytes( + context.layers[layer_name].read( + tcpip_module_offset + part_count_symbol, 1 + ), + "little", + ) + except exceptions.InvalidAddressException: + vollog.debug(f"`PartitionCount` not present in memory.") + return + part_table.Partitions.count = part_count vollog.debug( @@ -316,9 +369,21 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset( "ListEntry" ) - for ctr, partition in enumerate(part_table.Partitions): + + try: + partitions = part_table.Partitions + except exceptions.InvalidAddressException: + vollog.debug("Partitions member not present in memory") + return + + for ctr, partition in enumerate(partitions): vollog.debug(f"Parsing partition {ctr}") - if partition.Endpoints.NumEntries > 0: + try: + num_entries = partition.Endpoints.NumEntries + except exceptions.InvalidAddressException: + continue + + if num_entries > 0: for endpoint_entry in cls.parse_hashtable( context, layer_name, @@ -402,6 +467,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): upp_symbol = context.symbol_space.get_symbol( tcpip_symbol_table + constants.BANG + "UdpPortPool" ).address + upp_addr = context.object( net_symbol_table + constants.BANG + "pointer", layer_name=layer_name, @@ -498,13 +564,16 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # then, towards the UDP and TCP port pools # first, find their addresses - upp_addr, tpp_addr = cls.find_port_pools( - context, - layer_name, - net_symbol_table, - tcpip_symbol_table, - tcpip_module_offset, - ) + try: + upp_addr, tpp_addr = cls.find_port_pools( + context, + layer_name, + net_symbol_table, + tcpip_symbol_table, + tcpip_module_offset, + ) + except (exceptions.SymbolError, exceptions.InvalidAddressException): + vollog.debug("Unable to reconstruct port pools") # create port pool objects at the detected address and parse the port bitmap upp_obj = context.object( diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 478deab6b..e41ac6a05 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -219,7 +219,13 @@ class _TCP_ENDPOINT(_TCP_LISTENER): return None def is_valid(self): - if self.State not in self.State.choices.values(): + # netstat calls this before validating the object itself + try: + state = self.State + except exceptions.InvalidAddressException: + return False + + if state not in state.choices.values(): vollog.debug( f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid tcp state {self.State}" ) From 61d6a92f8f32e4fe81d951fc35fd7f36e80a2146 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:25:17 +0000 Subject: [PATCH 260/348] Significantly improve the smear/error handling in the netstat plugin --- volatility3/framework/plugins/windows/netstat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 3408c0a3a..902be5fc8 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -340,7 +340,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset=tcpip_module_offset + part_table_symbol, ) except exceptions.InvalidAddressException: - vollog.debug(f"`PartitionTable` not present in memory.") + vollog.debug("`PartitionTable` not present in memory.") return # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects @@ -358,7 +358,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): "little", ) except exceptions.InvalidAddressException: - vollog.debug(f"`PartitionCount` not present in memory.") + vollog.debug("`PartitionCount` not present in memory.") return part_table.Partitions.count = part_count From 65f602965b14a76dba1596e35a71ab1762257ea7 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:47:48 +0000 Subject: [PATCH 261/348] Address feedback --- volatility3/framework/plugins/windows/suspended_threads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py index 6ddfecca7..cec51ed37 100644 --- a/volatility3/framework/plugins/windows/suspended_threads.py +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -1,7 +1,7 @@ import logging from typing import Dict -from functools import partial +import functools from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements @@ -99,7 +99,7 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, None ) - path_and_symbol = partial( + path_and_symbol = functools.partial( pe_symbols.PESymbols.path_and_symbol_for_address, self.context, self.config_path, From 52a643d5b7f6c57e67acf39eed7c1feb2a0e9dbe Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 29 Dec 2024 20:29:03 +0000 Subject: [PATCH 262/348] Use enumerate for readability --- volatility3/framework/renderers/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 112e93751..093edf8cc 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -83,8 +83,7 @@ class TreeNode(interfaces.renderers.TreeNode): raise TypeError( "Values must be a list of objects made up of simple types and number the same as the columns" ) - for index in range(len(self._treegrid.columns)): - column = self._treegrid.columns[index] + for index, column in enumerate(self._treegrid.columns): val = values[index] if not isinstance(val, (column.type, interfaces.renderers.BaseAbsentValue)): raise TypeError( @@ -413,8 +412,7 @@ class ColumnSortKey(interfaces.renderers.ColumnSortKey): _index = None self._type = None self.ascending = ascending - for i in range(len(treegrid.columns)): - column = treegrid.columns[i] + for i, column in enumerate(treegrid.columns): if column.name.lower() == column_name.lower(): _index = i self._type = column.type From 28ff910d6280edc0dfa21b3b0585a1ab07de9279 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 30 Dec 2024 10:58:38 +0000 Subject: [PATCH 263/348] Use rsplit instead of split Since we want to split rightmost only. --- volatility3/plugins/windows/registry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/__init__.py b/volatility3/plugins/windows/registry/__init__.py index 8915cdfad..aeeaa87f2 100644 --- a/volatility3/plugins/windows/registry/__init__.py +++ b/volatility3/plugins/windows/registry/__init__.py @@ -15,5 +15,5 @@ import os import sys # This is necessary to ensure the core plugins are available, whilst still be overridable -parent_module, module_name = ".".join(__name__.split(".")[:-1]), __name__.split(".")[-1] +parent_module, module_name = __name__.rsplit(".", maxsplit=1) __path__ = [os.path.join(x, module_name) for x in sys.modules[parent_module].__path__] From e9d9345cef488067e7035aa485ff11ed665c4414 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 09:37:40 -0600 Subject: [PATCH 264/348] Windows Cachedump: Handle uncaught InvalidAddressException --- volatility3/framework/plugins/windows/cachedump.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index 6e667984a..6c730e6ae 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -8,7 +8,7 @@ from typing import Tuple from Crypto.Cipher import ARC4, AES from Crypto.Hash import HMAC -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions @@ -140,9 +140,14 @@ class Cachedump(interfaces.plugins.PluginInterface): if cache_item.Name == "NL$Control": continue - data = sechive.read(cache_item.Data + 4, cache_item.DataLength) - if data is None: + try: + data = sechive.read(cache_item.Data + 4, cache_item.DataLength) + except exceptions.InvalidAddressException: continue + + if not data: + continue + ( uname_len, domain_len, From 2153b742a1dbde57b369adb34fedc5e05e5eb40c Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 30 Dec 2024 18:06:04 +0000 Subject: [PATCH 265/348] Fix uncheck read() call and remove variable that would not be definied if exception triggers --- .../plugins/windows/skeleton_key_check.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index f5d7e1b3a..b103bc831 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -289,7 +289,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException as excp: vollog.debug( - f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" + f"Invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None, None @@ -431,15 +431,20 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): # we do not want to fail just because the count is not in memory # 16 was the size on samples I tested, so I chose it as the default + count = 16 + if target_address: - count = int.from_bytes( - self.context.layers[proc_layer_name].read( - target_address, 4 - ), - "little", - ) - else: - count = 16 + try: + count = int.from_bytes( + self.context.layers[proc_layer_name].read( + target_address, 4 + ), + "little", + ) + except exceptions.InvalidAddressException: + vollog.debug( + "Unable to read `cCsystems`. Defaulting to 16." + ) found_count = True From a3844e8bc54f9d597d459005830e733bd73d7256 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 30 Dec 2024 18:08:30 +0000 Subject: [PATCH 266/348] Fix uncheck read() call and remove variable that would not be definied if exception triggers --- volatility3/framework/plugins/windows/skeleton_key_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index b103bc831..6ae07381a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -282,7 +282,6 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): for proc in proc_list: try: - proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() return proc, proc_layer_name From 5af5363c461eab6ae1661665429a1794a2a712fb Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 13:50:05 -0600 Subject: [PATCH 267/348] Windows Handles: Work in fixes from @attrc These changes fix bugs encountered during regression testing related to virtual offset validation and string length checks. --- volatility3/framework/plugins/windows/handles.py | 8 ++++++++ .../framework/symbols/windows/extensions/pool.py | 11 ++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 62eceb973..e3845b376 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -226,6 +226,14 @@ class Handles(interfaces.plugins.PluginInterface): masked_offset = offset & layer_object.maximum_address for entry in table: + # This triggered a backtrace in many testing samples + # in the level == 0 path + # The code above this calls `is_valid` on the `offset` + # It is sent but then does not validate `entry` before + # sending it to `_get_item` + if not self.context.layers[virtual].is_valid(entry.vol.offset): + continue + if level > 0: yield from self._make_handle_array(entry, level - 1, depth) depth += 1 diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index de5c8271b..ff65acdeb 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -376,7 +376,16 @@ class OBJECT_HEADER(objects.StructType): try: # vista and earlier have a Type member - self._vol["object_header_object_type"] = self.Type.Name.String + length = self.Type.member("Name").Length + if length == 0 or length > 128: + string = None + else: + string = self.Type.Name.String + if len(string) == 0 or len(string) > 128: + string = None + + self._vol["object_header_object_type"] = string + except AttributeError: # windows 7 and later have a TypeIndex, but windows 10 # further encodes the index value with nt1!ObHeaderCookie From 3eeb10be2916bb7988d296c7a85785ffb5a7f25e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 10:10:03 -0600 Subject: [PATCH 268/348] Windows Registry: Handle uncaught exceptions A number of calls to `get_key` across multiple plugins are not made within a `try/except` block that handles `registry.RegistryFormatException` - the calls are either unprotected or only check for `KeyError`. This adds the required `try/except` blocks, or updates the existing ones as needed. --- volatility3/framework/plugins/windows/amcache.py | 10 +++++----- volatility3/framework/plugins/windows/hashdump.py | 2 +- volatility3/framework/plugins/windows/lsadump.py | 7 +++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/amcache.py b/volatility3/framework/plugins/windows/amcache.py index 1e918d61c..46a742233 100644 --- a/volatility3/framework/plugins/windows/amcache.py +++ b/volatility3/framework/plugins/windows/amcache.py @@ -543,7 +543,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\InventoryDriverBinary") # type: ignore ) ) - except KeyError: + except (KeyError, registry.RegistryFormatException): # Registry key not found pass @@ -554,7 +554,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\Programs") ) # type: ignore } - except KeyError: + except (KeyError, registry.RegistryFormatException): programs = {} try: @@ -564,7 +564,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), key=_entry_sort_key, ) - except KeyError: + except (KeyError, registry.RegistryFormatException): files = [] for program_id, file_entries in itertools.groupby( @@ -593,7 +593,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\InventoryApplication") # type: ignore ) ) - except KeyError: + except (KeyError, registry.RegistryFormatException): programs = {} try: @@ -603,7 +603,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), key=_entry_sort_key, ) - except KeyError: + except (KeyError, registry.RegistryFormatException): files = [] for program_id, file_entries in itertools.groupby( diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index 0c98ab8ca..621b0ae53 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -332,7 +332,7 @@ class Hashdump(interfaces.plugins.PluginInterface): try: if hive: result = hive.get_key(key) - except KeyError: + except (KeyError, registry.RegistryFormatException): vollog.info( f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image" ) diff --git a/volatility3/framework/plugins/windows/lsadump.py b/volatility3/framework/plugins/windows/lsadump.py index da8dee325..f3925f2a2 100644 --- a/volatility3/framework/plugins/windows/lsadump.py +++ b/volatility3/framework/plugins/windows/lsadump.py @@ -8,7 +8,7 @@ from typing import Optional from Crypto.Cipher import ARC4, DES, AES from Crypto.Hash import MD5, SHA256 -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions @@ -81,7 +81,10 @@ class Lsadump(interfaces.plugins.PluginInterface): if not enc_reg_value: return None - obf_lsa_key = sechive.read(enc_reg_value.Data + 4, enc_reg_value.DataLength) + try: + obf_lsa_key = sechive.read(enc_reg_value.Data + 4, enc_reg_value.DataLength) + except exceptions.InvalidAddressException: + return None if not obf_lsa_key: return None From f3294ef5f12a6b036989585b88105fde62634ce2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 12:58:44 -0600 Subject: [PATCH 269/348] Windows Registry: Handle possible exception in get_node Encountered a `SwappedInvalidAddressException` within the call to `cast` due to an underlying call to `read`. --- volatility3/framework/layers/registry.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc364ad50..ee7286e1e 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -140,7 +140,14 @@ class RegistryHive(linear.LinearlyMappedLayer): """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast("string", max_length=2, encoding="latin-1") + try: + signature = cell.cast("string", max_length=2, encoding="latin-1") + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read cell signature for cell at {cell.vol.offset:x}" + ) + return cell + if signature == "nk": return cell.u.KeyNode elif signature == "sk": From 21077f909f6bda2dec6a3900c7e9ec53268b8213 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 30 Dec 2024 22:28:47 +0000 Subject: [PATCH 270/348] Sort imports and swap two assignments --- volatility3/cli/text_filter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 6bd6878a5..b6f019da9 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -1,7 +1,8 @@ import logging -from typing import Any, List, Optional -from volatility3.framework import constants, interfaces import re +from typing import Any, List, Optional + +from volatility3.framework import constants, interfaces vollog = logging.getLogger(__name__) @@ -67,8 +68,8 @@ class ColumnFilter: ) -> None: self.column_num = column_num self.pattern = pattern - self.exclude = exclude self.regex = regex + self.exclude = exclude def find(self, item) -> bool: """Identifies whether an item is found in the appropriate column""" From 3288ac971397500f61501b86a99678592cbd4128 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 14:01:40 -0600 Subject: [PATCH 271/348] Windows Handles: Handle possibly invalid memory accesses Any number of member accesses here can raise an `InvalidAddressException`; each is now checked, and `None` returned if any `InvalidAddressException` occurs. --- .../framework/plugins/windows/handles.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index e3845b376..85b16d2d4 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -68,7 +68,12 @@ class Handles(interfaces.plugins.PluginInterface): if not self.context.layers[virtual].is_valid(handle_table_entry.Object): return None fast_ref = handle_table_entry.Object.cast("_EX_FAST_REF") - object_header = fast_ref.dereference().cast("_OBJECT_HEADER") + + try: + object_header = fast_ref.dereference().cast("_OBJECT_HEADER") + except exceptions.InvalidAddressException: + return None + object_header.GrantedAccess = handle_table_entry.GrantedAccess except AttributeError: # starting with windows 8 @@ -77,16 +82,26 @@ class Handles(interfaces.plugins.PluginInterface): ) if is_64bit: - if handle_table_entry.ObjectPointerBits == 0: + try: + pointer_bits = handle_table_entry.ObjectPointerBits + except exceptions.InvalidAddressException: return None - offset = handle_table_entry.ObjectPointerBits << 4 + if pointer_bits == 0: + return None + + offset = pointer_bits << 4 else: - if handle_table_entry.InfoTable == 0: + try: + info_table = handle_table_entry.InfoTable + except exceptions.InvalidAddressException: return None - offset = handle_table_entry.InfoTable & ~7 + if info_table == 0: + return None + + offset = info_table & ~7 # print("LowValue: {0:#x} Magic: {1:#x} Offset: {2:#x}".format(handle_table_entry.InfoTable, magic, offset)) object_header = self.context.object( @@ -94,7 +109,10 @@ class Handles(interfaces.plugins.PluginInterface): virtual, offset=offset, ) - object_header.GrantedAccess = handle_table_entry.GrantedAccessBits + try: + object_header.GrantedAccess = handle_table_entry.GrantedAccessBits + except exceptions.InvalidAddressException: + return None object_header.HandleValue = handle_value return object_header From 9a5365e681e971f0e58b23ed42195df09dced6e3 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 16:59:40 -0600 Subject: [PATCH 272/348] Windows Handles: Fix unbound local in exception handler This fixes an unbound local used in a debug message; If the exception is raised during the dereference operation, the `objct` variable may be uninitialized. This uses the offset of `ptr` instead. --- volatility3/framework/plugins/windows/handles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 85b16d2d4..38ccfbfbc 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -178,7 +178,7 @@ class Handles(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, - f"Cannot access _OBJECT_HEADER Name at {objt.vol.offset:#x}", + f"Cannot access _OBJECT_HEADER Name at {ptr.vol.offset:#x}", ) continue From 263c87611b51f1cb9710c2ec71d1f41eb98e9c77 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 31 Dec 2024 10:43:27 -0600 Subject: [PATCH 273/348] Windows Registry: Handle exceptions in read calls These calls to `.read()` can raise an `InvalidAddressException`. Instead of propagating this exception to the caller, this adds debug logging, and pads the data will null bytes. Also updates the docstring for `decode_data()` to indicate that it can raise `TypeError` and `ValueError`. --- .../symbols/windows/extensions/registry.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 9e2f8df3b..97dd7390d 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -276,7 +276,16 @@ class CM_KEY_VALUE(objects.StructType): return RegValueTypes(self.Type) def decode_data(self) -> Union[int, bytes]: - """Properly decodes the data associated with the value node""" + """ + Properly decodes the data associated with the value node. + + If an InvalidAddressException occurs when reading data from the + underlying RegistryHive layer, the data will be padded with null bytes + of the same length. + + Raises ValueError if the data cannot be read + Raises TypeError if the class was not instantiated on a RegistryHive layer + """ # Determine if the data is stored inline datalen = self.DataLength data = b"" @@ -310,14 +319,26 @@ class CM_KEY_VALUE(objects.StructType): and block_offset < layer.maximum_address ): amount = min(BIG_DATA_MAXLEN, datalen) - data += layer.read( - offset=layer.get_cell(block_offset).vol.offset, length=amount - ) + try: + data += layer.read( + offset=layer.get_cell(block_offset).vol.offset, + length=amount, + ) + except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {amount:x} bytes of data, padding with {amount:x}" + ) datalen -= amount else: # Suspect Data actually points to a Cell, # but the length at the start could be negative so just adding 4 to jump past it - data = layer.read(self.Data + 4, datalen) + try: + data = layer.read(self.Data + 4, datalen) + except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {datalen:x} bytes of data, returning {datalen:x} null bytes" + ) + data = b"\x00" * datalen if self.get_type() == RegValueTypes.REG_DWORD: if len(data) != struct.calcsize(" Date: Tue, 31 Dec 2024 11:11:57 -0600 Subject: [PATCH 274/348] Windows Registry: Update docstrings + exceptions This updates the docstrings on several methods to indicate that they may raise an exception. --- .../symbols/windows/extensions/registry.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 97dd7390d..e53338855 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -159,6 +159,11 @@ class CM_KEY_NODE(objects.StructType): """Extension to allow traversal of registry keys.""" def get_volatile(self) -> bool: + """ + Returns a bool indicating whether or not the key is volatile. + + Raises ValueError if the key was not instantiated on a RegistryHive layer + """ if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): raise ValueError( "Cannot determine volatility of registry key without an offset in a RegistryHive layer" @@ -166,7 +171,10 @@ class CM_KEY_NODE(objects.StructType): return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: - """Returns a list of the key nodes.""" + """Returns a list of the key nodes. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") @@ -222,7 +230,10 @@ class CM_KEY_NODE(objects.StructType): yield from self._get_subkeys_recursive(hive, subnode) def get_values(self) -> Iterator["CM_KEY_VALUE"]: - """Returns a list of the Value nodes for a key.""" + """Returns a list of the Value nodes for a key. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") @@ -251,6 +262,11 @@ class CM_KEY_NODE(objects.StructType): return self.Name.cast("string", max_length=namelength, encoding="latin-1") def get_key_path(self) -> str: + """ + Returns the full path to this registry key. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ reg = self._context.layers[self.vol.layer_name] if not isinstance(reg, RegistryHive): raise TypeError("Key was not instantiated on a RegistryHive layer") From fa67f10d3183cd743ca7e44b66d152141a34b2fb Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 31 Dec 2024 11:40:41 -0600 Subject: [PATCH 275/348] Windows Registry: Catch RegistryInvalidIndex refs #1484 This catches uncaught exceptions when casting the cell to a string in `get_node`. --- volatility3/framework/layers/registry.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc364ad50..c684ccd40 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -140,7 +140,13 @@ class RegistryHive(linear.LinearlyMappedLayer): """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast("string", max_length=2, encoding="latin-1") + try: + signature = cell.cast("string", max_length=2, encoding="latin-1") + except (RegistryInvalidIndex, exceptions.InvalidAddressException): + vollog.debug( + f"Failed to get cell signature for cell (0x{cell.vol.offset:x})" + ) + return cell if signature == "nk": return cell.u.KeyNode elif signature == "sk": From 9c02f0d12a13fb77db8fb326f6f68dc31fceec1f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 15:22:09 +0000 Subject: [PATCH 276/348] Linux: Fix kmsf f-strings Closes #1496 --- volatility3/framework/plugins/linux/kmsg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d66e3b9ca..c1d09aff8 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return f"{nsec / 1000000000:lu}.{(nsec % 1000000000) / 1000:06lu}" + return f"{nsec / 1000000000}.{(nsec % 1000000000) / 1000:06}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = f"{caller_name}({caller_id & ~0x80000000:u})" + caller = f"{caller_name}({int(caller_id & ~0x80000000)})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: From ac3e76665b7a44b6c5dbc18e633814bd2371ff75 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 15:29:07 +0000 Subject: [PATCH 277/348] Linux: Fix kmsg unguarded read of msg.len --- volatility3/framework/plugins/linux/kmsg.py | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d66e3b9ca..67114c087 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -317,23 +317,27 @@ class Kmsg_3_5_to_3_11(ABCKmsg): while cur_idx < end_idx: msg_offset = log_buf_ptr + cur_idx # type: ignore msg = self.vmlinux.object(object_type=log_struct_name, offset=msg_offset) - if msg.len == 0: - # As per kernel/printk.c: - # A length == 0 for the next message indicates a wrap-around to - # the beginning of the buffer. - cur_idx = 0 - end_idx = log_next_idx - else: - facility, level, timestamp, caller = self.get_prefix(msg) - level_txt = self.get_level_text(level) - facility_txt = self.get_facility_text(facility) + try: + if msg.len == 0: + # As per kernel/printk.c: + # A length == 0 for the next message indicates a wrap-around to + # the beginning of the buffer. + cur_idx = 0 + end_idx = log_next_idx + else: + facility, level, timestamp, caller = self.get_prefix(msg) + level_txt = self.get_level_text(level) + facility_txt = self.get_facility_text(facility) - for line in self.get_log_lines(msg): - yield facility_txt, level_txt, timestamp, caller, line - for line in self.get_dict_lines(msg): - yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_log_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_dict_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line - cur_idx += msg.len + cur_idx += msg.len + except exceptions.InvalidAddressException: + vollog.warning("Kmsg buffer msg length could not be read") + return class Kmsg_3_11_to_5_10(Kmsg_3_5_to_3_11): From c8e67e526a831dcd05b59fa0adeeda8937c4f81a Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 1 Jan 2025 22:33:25 -0600 Subject: [PATCH 278/348] Convert ValueError to TypeError All other methods in this class raise a `TypeError` if the hive was not instantiated on a registry layer; this changes makes this method consistent with the convention used in the others. All `except` blocks checking for `ValueError` have been audited to ensure that this doesn't break exception handling in existing code within the framework. This also includes a minor version bump because: 1. RegistryHives are currently only instantiated one way, which is through the `hivelist` plugin. `hivelist` uses the correct layers when instantiating the hives. 2. Because there is currently a single source for registry hives, and it's unlikely that a hive from that source will ever be created on the wrong layer, it's unlikely that the existing `ValueError` is being raised anywhere within the framework's code. 3. It seems unlikely that consumers of this framework would be instantiating registry hives independent of the `hivelist` plugin, given that they would effectively have to duplicate the `hivelist` code to do so. For these reasons, we're going to do a minor version bump, even though an argument can be made that this warrants a major version bump according to the SemVer rules. This is a one-off and does not indicate any change in the way that we typically update version numbers. --- volatility3/framework/constants/_version.py | 2 +- .../framework/symbols/windows/extensions/registry.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 9ca2d0a5b..2f0c53093 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 14 # Number of changes that only add to the interface +VERSION_MINOR = 15 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index e53338855..c9544a8ba 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -162,12 +162,10 @@ class CM_KEY_NODE(objects.StructType): """ Returns a bool indicating whether or not the key is volatile. - Raises ValueError if the key was not instantiated on a RegistryHive layer + Raises TypeError if the key was not instantiated on a RegistryHive layer """ if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): - raise ValueError( - "Cannot determine volatility of registry key without an offset in a RegistryHive layer" - ) + raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: From 97b93abe438bf32b32067bd962a19e07d9917406 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 2 Jan 2025 11:26:21 +0000 Subject: [PATCH 279/348] Linux: Remove unnecessary int cast --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index c1d09aff8..894ca575f 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = f"{caller_name}({int(caller_id & ~0x80000000)})" + caller = f"{caller_name}({caller_id & ~0x80000000})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: From 7278bb244f58f36b346d254512e393cde6f63871 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 14:47:18 +0100 Subject: [PATCH 280/348] move get_flags_list at bottom --- .../framework/symbols/linux/extensions/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 34d0fcba9..9546fcf82 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2628,6 +2628,19 @@ class page(objects.StructType): page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) return page_data + def get_flags_list(self) -> List[str]: + """Returns a list of page flags + + Returns: + List of page flags + """ + flags = [] + for name, value in self.pageflags_enum.items(): + if self.flags & (1 << value) != 0: + flags.append(name) + + return flags + class IDR(objects.StructType): IDR_BITS = 8 From b61ba66223a866a29a119eb17f44fba250ecc01e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 14:51:23 +0100 Subject: [PATCH 281/348] multi-architecture vmemmap_start calculation --- .../symbols/linux/extensions/__init__.py | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9546fcf82..b02f80433 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,7 +15,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion from volatility3.framework.constants import linux as linux_constants -from volatility3.framework.layers import linear +from volatility3.framework.layers import linear, intel from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf @@ -2525,16 +2525,13 @@ class address_space(objects.StructType): class page(objects.StructType): - @property - @functools.lru_cache + @functools.cached_property def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values Returns: A dictionary with the pageflags enumeration key/values """ - # FIXME: It would be even better to use @functools.cached_property instead, - # however, this requires Python +3.8 try: pageflags_enum = self._context.symbol_space.get_enumeration( self.get_symbol_table_name() + constants.BANG + "pageflags" @@ -2548,24 +2545,12 @@ class page(objects.StructType): return pageflags_enum - def get_flags_list(self) -> List[str]: - """Returns a list of page flags + @functools.cached_property + def _intel_vmemmap_start(self) -> int: + """Determine the start of the struct page array, for Intel systems. Returns: - List of page flags - """ - flags = [] - for name, value in self.pageflags_enum.items(): - if self.flags & (1 << value) != 0: - flags.append(name) - - return flags - - def to_paddr(self) -> int: - """Converts a page's virtual address to its physical address using the current physical memory model. - - Returns: - int: page physical address + int: vmemmap_start address """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] @@ -2605,13 +2590,39 @@ class page(objects.StructType): "Something went wrong, we shouldn't be here" ) - page_type_size = vmlinux.get_type("page").size + return vmemmap_start + + def _intel_to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current Intel memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] pagec = vmlinux_layer.canonicalize(self.vol.offset) - pfn = (pagec - vmemmap_start) // page_type_size + pfn = (pagec - self._intel_vmemmap_start) // vmlinux.get_type("page").size page_paddr = pfn * vmlinux_layer.page_size return page_paddr + def to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current CPU memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + if isinstance(vmlinux_layer, intel.Intel): + page_paddr = self._intel_to_paddr() + else: + raise exceptions.LayerException( + f"Architecture {type(vmlinux_layer)} vmemmap_start calculation isn't currently supported." + ) + + return page_paddr + def get_content(self) -> Union[str, None]: """Returns the page content @@ -2620,7 +2631,10 @@ class page(objects.StructType): """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - physical_layer = vmlinux.context.layers["memory_layer"] + physical_layer_name = self._context.layers[self.vol.layer_name].config.get( + "memory_layer", self.vol.layer_name + ) + physical_layer = self._context.layers[physical_layer_name] page_paddr = self.to_paddr() if not page_paddr: return None From dda104bd62b9f5f7b9c0208832c6d788c0ebd2ea Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:37:14 +0100 Subject: [PATCH 282/348] move out Tainting capabilities --- .../framework/symbols/linux/__init__.py | 121 ------------------ 1 file changed, 121 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 832b1de9b..0230a9c48 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -11,7 +11,6 @@ from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions -from volatility3.framework.constants import linux as linux_constants class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -831,123 +830,3 @@ class PageCache: page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page - - -class Tainting: - """Tainted kernel and modules parsing capabilities. - - Relevant kernel functions: - - modules: module_flags_taint - - kernel: print_tainted - """ - - def __init__( - self, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - ): - self.kernel = context.modules[kernel_module_name] - - @property - def kernel_taint_flags_list( - self, - ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self.kernel.has_symbol("taint_flags"): - return list(self.kernel.object_from_symbol("taint_flags")) - return None - - def _module_flags_taint_pre_4_10_rc1( - self, taints: int, is_module: bool = False - ) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on statically defined taints mappings in the framework. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - The raw taints string. - """ - taints_string = "" - for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if is_module and is_module != taint_flag.module: - continue - - if taints & taint_flag.shift: - taints_string += char - - return taints_string - - def _module_flags_taint_post_4_10_rc1( - self, taints: int, is_module: bool = False - ) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on kernel symbol embedded taints definitions. - - struct taint_flag { - char c_true; /* character printed when tainted */ - char c_false; /* character printed when not tainted */ - bool module; /* also show as a per-module taint flag */ - }; - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - The raw taints string. - """ - taints_string = "" - for i, taint_flag in enumerate(self.kernel_taint_flags_list): - if is_module and is_module != taint_flag.module: - continue - c_true = chr(taint_flag.c_true) - c_false = chr(taint_flag.c_false) - if taints & (1 << i): - taints_string += c_true - elif c_false != " ": - taints_string += c_false - - return taints_string - - def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: - """Convert the taints value to a 1-1 character mapping. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - s - Returns: - The raw taints string. - - Documentation: - - module_flags_taint kernel function - """ - - if self.kernel_taint_flags_list: - return self._module_flags_taint_post_4_10_rc1(taints, is_module) - return self._module_flags_taint_pre_4_10_rc1(taints, is_module) - - def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: - """Convert the taints string to a 1-1 descriptor mapping. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - A comprehensive (user-friendly) taint descriptor list. - - Documentation: - - module_flags_taint kernel function - """ - comprehensive_taints = [] - for character in self.get_taints_as_plain_string(taints, is_module): - taint_flag = linux_constants.TAINT_FLAGS.get(character) - if not taint_flag: - comprehensive_taints.append(f"") - elif taint_flag.when_present: - comprehensive_taints.append(taint_flag.desc) - - return comprehensive_taints From 2a5f38ebad48e0d729b3b22caac84bd4209f20a2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:39:07 +0100 Subject: [PATCH 283/348] introduce versioned Linux utilities --- .../framework/symbols/linux/utilities/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 volatility3/framework/symbols/linux/utilities/__init__.py diff --git a/volatility3/framework/symbols/linux/utilities/__init__.py b/volatility3/framework/symbols/linux/utilities/__init__.py new file mode 100644 index 000000000..4225d444b --- /dev/null +++ b/volatility3/framework/symbols/linux/utilities/__init__.py @@ -0,0 +1,11 @@ +from volatility3 import framework +from volatility3.framework import interfaces + + +class LinuxUtilityInterface(interfaces.configuration.VersionableInterface): + """Class with multiple useful Linux functions surrounding a specific piece of functionality.""" + + _version = (2, 1, 1) + _required_framework_version = (2, 0, 0) + + framework.require_interface_version(*_required_framework_version) From 8bc62598f4530bdf2fb99aeb725e5b8f3e0d8cd5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:39:57 +0100 Subject: [PATCH 284/348] initial tainting utilities --- .../symbols/linux/utilities/tainting.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 volatility3/framework/symbols/linux/utilities/tainting.py diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py new file mode 100644 index 000000000..e6d75a963 --- /dev/null +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -0,0 +1,130 @@ +from volatility3 import framework +from volatility3.framework import interfaces +from volatility3.framework.symbols.linux.utilities import LinuxUtilityInterface +from volatility3.framework.constants import linux as linux_constants +from typing import List, Optional + + +class Tainting(LinuxUtilityInterface): + """Tainted kernel and modules parsing capabilities. + + Relevant Linux kernel functions: + - modules: module_flags_taint + - kernel: print_tainted + """ + + _version = (1, 0, 0) + _required_framework_version = (2, 14, 0) + + framework.require_interface_version(*_required_framework_version) + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.kernel = context.modules[kernel_module_name] + + @property + def _kernel_taint_flags_list( + self, + ) -> Optional[List[interfaces.objects.ObjectInterface]]: + if self.kernel.has_symbol("taint_flags"): + return list(self.kernel.object_from_symbol("taint_flags")) + return None + + def _module_flags_taint_pre_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if is_module and is_module != taint_flag.module: + continue + + if taints & taint_flag.shift: + taints_string += char + + return taints_string + + def _module_flags_taint_post_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self._kernel_taint_flags_list): + if is_module and is_module != taint_flag.module: + continue + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taints & (1 << i): + taints_string += c_true + elif c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + """Convert the taints value to a 1-1 character mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + s + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self._kernel_taint_flags_list: + return self._module_flags_taint_post_4_10_rc1(taints, is_module) + return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + + def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + """Convert the taints string to a 1-1 descriptor mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for character in self.get_taints_as_plain_string(taints, is_module): + taint_flag = linux_constants.TAINT_FLAGS.get(character) + if not taint_flag: + comprehensive_taints.append(f"") + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) + + return comprehensive_taints From 3105a31964a1a36420281bd995d983a81b161e97 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:40:49 +0100 Subject: [PATCH 285/348] leverage Tainting from separated Linux utilities --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 075a83ae8..ac07d2def 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -19,7 +19,7 @@ from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf - +from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -286,7 +286,7 @@ class module(generic.GenericIntelProcess): Returns: The raw taints string. """ - return linux.Tainting( + return tainting.Tainting( self._context, linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, ).get_taints_as_plain_string(self.taints, True) @@ -298,7 +298,7 @@ class module(generic.GenericIntelProcess): Returns: A comprehensive (user-friendly) taint descriptor list. """ - return linux.Tainting( + return tainting.Tainting( self._context, linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, ).get_taints_parsed(self.taints, True) From 6e4213e321b96dd8f0b35df6c87aa8426698a42c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:41:37 +0100 Subject: [PATCH 286/348] update tainting requirements to new versioned utilities --- volatility3/framework/plugins/linux/modxview.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index d79f5e7a9..c97864a87 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -9,6 +9,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue from volatility3.framework.symbols.linux import extensions from volatility3.framework.constants import architectures +from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -18,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 11, 0) + _required_framework_version = (2, 14, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -28,6 +29,9 @@ class Modxview(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=architectures.LINUX_ARCHS, ), + requirements.VersionRequirement( + name="linux-tainting", component=tainting.Tainting, version=(1, 0, 0) + ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), From 0a3502697cca4dac2ac2f39896ecfaaef507ac9b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:42:16 +0100 Subject: [PATCH 287/348] 2.13.0 -> 2.14.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 11edc07d8..9ca2d0a5b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 13 # Number of changes that only add to the interface +VERSION_MINOR = 14 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 89b8da8c39fe14f699d711c95a8311ec1e21331e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:48:22 +0100 Subject: [PATCH 288/348] make self.kernel private and call parent __init__ --- .../framework/symbols/linux/utilities/tainting.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index e6d75a963..603215961 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -22,15 +22,18 @@ class Tainting(LinuxUtilityInterface): self, context: interfaces.context.ContextInterface, kernel_module_name: str, + *args, + **kwargs, ): - self.kernel = context.modules[kernel_module_name] + super().__init__(*args, **kwargs) + self._kernel = context.modules[kernel_module_name] @property def _kernel_taint_flags_list( self, ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self.kernel.has_symbol("taint_flags"): - return list(self.kernel.object_from_symbol("taint_flags")) + if self._kernel.has_symbol("taint_flags"): + return list(self._kernel.object_from_symbol("taint_flags")) return None def _module_flags_taint_pre_4_10_rc1( From d2bb5c9f31d7f01fe2e343867c0c7c1926b3ac50 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 3 Jan 2025 10:01:37 +1100 Subject: [PATCH 289/348] linux: fix kmsg fstring bug introduced in #1502 --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 894ca575f..30f67b319 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return f"{nsec / 1000000000}.{(nsec % 1000000000) / 1000:06}" + return f"{nsec // 1000000000}.{(nsec % 1000000000) // 1000:06}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info From 4a34b988d1e4cb02e33e555c3e2ed63d808e5028 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 3 Jan 2025 13:21:09 +0100 Subject: [PATCH 290/348] minor readability adjustments --- .../framework/symbols/linux/utilities/tainting.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 603215961..f7f6c83ec 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -51,7 +51,7 @@ class Tainting(LinuxUtilityInterface): """ taints_string = "" for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if is_module and is_module != taint_flag.module: + if is_module and not taint_flag.module: continue if taints & taint_flag.shift: @@ -79,12 +79,12 @@ class Tainting(LinuxUtilityInterface): The raw taints string. """ taints_string = "" - for i, taint_flag in enumerate(self._kernel_taint_flags_list): - if is_module and is_module != taint_flag.module: + for taint_bit, taint_flag in enumerate(self._kernel_taint_flags_list): + if is_module and not taint_flag.module: continue c_true = chr(taint_flag.c_true) c_false = chr(taint_flag.c_false) - if taints & (1 << i): + if taints & (1 << taint_bit): taints_string += c_true elif c_false != " ": taints_string += c_false @@ -97,7 +97,6 @@ class Tainting(LinuxUtilityInterface): Args: taints: The taints value, represented by an integer is_module: Indicates if the taints value is associated with a built-in/LKM module - s Returns: The raw taints string. From 38c5cc168f93a4d1a5cab2a6c9b071cf32e22fc2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 3 Jan 2025 13:23:27 +0100 Subject: [PATCH 291/348] bump framework req to 2.16.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- volatility3/framework/symbols/linux/utilities/tainting.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index c97864a87..042930740 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 14, 0) + _required_framework_version = (2, 16, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index f7f6c83ec..fc2f94109 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -14,7 +14,7 @@ class Tainting(LinuxUtilityInterface): """ _version = (1, 0, 0) - _required_framework_version = (2, 14, 0) + _required_framework_version = (2, 16, 0) framework.require_interface_version(*_required_framework_version) From 32ca62bbb1205b11be0338e741e3046d503153a8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:20:35 +0000 Subject: [PATCH 292/348] Make f-string slightly more readable --- .../framework/plugins/windows/shimcachemem.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index b8e9b5bd7..9d968c30a 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -305,14 +305,14 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf If a number of validity checks are passed, this method will return the `SHIM_CACHE_HEAD` object. Otherwise, `None` is returned. """ - # print("checking RTL_AVL_TABLE at offset %s" % hex(offset)) + # Check RTL_AVL_TABLE at offset rtl_avl_table = context.object( symbol_table + constants.BANG + "_RTL_AVL_TABLE", layer_name, offset ) if not rtl_avl_table.is_valid(mod_page_start, mod_page_end): return None - vollog.debug(f"Candidate RTL_AVL_TABLE found at offset {hex(offset)}") + vollog.debug(f"Candidate RTL_AVL_TABLE found at offset {offset:#x}") ersrc_size = context.symbol_space.get_type( kernel_symbol_table + constants.BANG + "_ERESOURCE" @@ -324,13 +324,13 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf # 0x20 if context.symbol_space.get_type("pointer").size == 8 else 0x10 ) vollog.debug( - f"ERESOURCE size: {hex(ersrc_size)}, ERESOURCE alignment: {hex(ersrc_alignment)}" + f"ERESOURCE size: {ersrc_size:#x}, ERESOURCE alignment: {ersrc_alignment:#x}" ) eresource_rel_off = ersrc_size + ((offset - ersrc_size) % ersrc_alignment) eresource_offset = offset - eresource_rel_off - vollog.debug(f"Constructing ERESOURCE at {hex(eresource_offset)}") + vollog.debug(f"Constructing ERESOURCE at {eresource_offset:#x}") eresource = context.object( kernel_symbol_table + constants.BANG + "_ERESOURCE", layer_name, @@ -408,8 +408,8 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf # iterate over ahcache kernel module's .data section in search of *two* SHIM handles shim_heads = [] - vollog.debug(f"PAGE offset: {hex(mod_page_offset)}") - vollog.debug(f".data offset: {hex(data_sec_offset)}") + vollog.debug(f"PAGE offset: {mod_page_offset:#x}") + vollog.debug(f".data offset: {data_sec_offset:#x}") handle_type = context.symbol_space.get_type( shimcache_symbol_table + constants.BANG + "SHIM_CACHE_HANDLE" @@ -419,7 +419,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf data_sec_offset + data_sec_size, 8 if symbols.symbol_table_is_64bit(context, nt_symbol_table) else 4, ): - vollog.debug(f"Building shim handle pointer at {hex(offset)}") + vollog.debug(f"Building shim handle pointer at {offset:#x}") shim_handle = context.object( object_type=shimcache_symbol_table + constants.BANG + "pointer", layer_name=kernel_layer_name, @@ -430,7 +430,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf if shim_handle.is_valid(mod_page_offset, mod_page_offset + mod_page_size): if shim_handle.head is not None: vollog.debug( - f"Found valid shim handle @ {hex(shim_handle.vol.offset)}" + f"Found valid shim handle @ {shim_handle.vol.offset:#x}" ) shim_heads.append(shim_handle.head) if len(shim_heads) == 2: @@ -440,7 +440,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf vollog.debug("Failed to identify two valid SHIM_CACHE_HANDLE structures") return - # On Windows 8 x64, the frist cache contains the shim cache + # On Windows 8 x64, the first cache contains the shim cache. # On Windows 8 x86, 8.1 x86/x64, and 10, the second cache contains the shim cache. if ( not symbols.symbol_table_is_64bit(context, nt_symbol_table) From 03049f789559af5c4cdb56f343460178b52220f9 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 3 Jan 2025 18:40:52 +0000 Subject: [PATCH 293/348] Add missing exception handling in env var recovery. Prevent backtraces --- volatility3/framework/plugins/linux/envars.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 8cdbfe493..04b75c8a8 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -5,7 +5,7 @@ import logging from typing import Iterable, Tuple -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -58,10 +58,16 @@ class Envars(plugins.PluginInterface): Tuples of (key, value) representing each environment variable. """ - task_name = utility.array_to_string(task.comm) + # This ensures the `task` is valid as well as its + # memory mapping structures + try: + task_name = utility.array_to_string(task.comm) + env_start = task.mm.env_start + env_end = task.mm.env_end + except exceptions.InvalidAddressException: + return None + task_pid = task.pid - env_start = task.mm.env_start - env_end = task.mm.env_end env_area_size = env_end - env_start if not (0 < env_area_size <= env_area_max_size): vollog.debug( From 8ba60a2aaddf86e4cbd065c95d2553ce221db183 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 4 Jan 2025 16:49:02 +0000 Subject: [PATCH 294/348] Change add_process_layer to return None instead of throwing an exception as it was meant to be designed --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b02f80433..df1c00e3d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -324,9 +324,11 @@ class task_struct(generic.GenericIntelProcess): raise TypeError( "Parent layer is not a translation layer, unable to construct process layer" ) - dtb, layer_name = parent_layer.translate(pgd) - if not dtb: + try: + dtb, layer_name = parent_layer.translate(pgd) + except exceptions.InvalidAddressException: return None + if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.pid}" # Add the constructed layer and return the name From 5f1d318c715311ed12d67bde5a87a8a78e0d3bf0 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 13:39:00 +0000 Subject: [PATCH 295/348] Tiny comment changes --- volatility3/framework/plugins/windows/cmdscan.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 9645ee507..3dc70d649 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -67,6 +67,7 @@ class CmdScan(interfaces.plugins.PluginInterface): Args: conhost_proc: the process object for conhost.exe + size_filter: filter (keep) vads less than this size (bytes) Returns: A list of tuples of: @@ -100,7 +101,7 @@ class CmdScan(interfaces.plugins.PluginInterface): kernel_symbol_table_name: The name of the table containing the kernel symbols config_path: The config path where to find symbol files procs: list of process objects - max_history: an initial set of CommandHistorySize values + max_history: An initial set of CommandHistorySize values Returns: The conhost process object, the command history structure, a dictionary of properties for @@ -227,7 +228,6 @@ class CmdScan(interfaces.plugins.PluginInterface): "data": command_history.CommandCountMax, } ) - command_history_properties.append( { "level": 1, @@ -236,6 +236,7 @@ class CmdScan(interfaces.plugins.PluginInterface): "data": "", } ) + for ( cmd_index, bucket_cmd, @@ -352,7 +353,7 @@ class CmdScan(interfaces.plugins.PluginInterface): def _conhost_proc_filter(self, proc: interfaces.objects.ObjectInterface): """ - Used to filter to only conhost.exe processes + Used to filter only conhost.exe processes """ process_name = utility.array_to_string(proc.ImageFileName) From ab60add9933ee3863c3f2329d2c99af314b5b453 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 17:14:21 +0000 Subject: [PATCH 296/348] Update case insensitive check Update link and use casefold() instead of lower(). --- volatility3/framework/layers/registry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index c684ccd40..6d85da982 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -192,9 +192,9 @@ class RegistryHive(linear.LinearlyMappedLayer): while key_array and node_key: subkeys = node_key[-1].get_subkeys() for subkey in subkeys: - # registry keys are not case sensitive so compare lowercase - # https://msdn.microsoft.com/en-us/library/windows/desktop/ms724946(v=vs.85).aspx - if subkey.get_name().lower() == key_array[0].lower(): + # registry keys are not case sensitive so compare likewise + # https://learn.microsoft.com/en-gb/windows/win32/sysinfo/structure-of-the-registry + if subkey.get_name().casefold() == key_array[0].casefold(): node_key = node_key + [subkey] found_key, key_array = found_key + [key_array[0]], key_array[1:] break From 8f4f576e93a7594666f0e58f8ae73cce5538902c Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 21:09:15 +0000 Subject: [PATCH 297/348] Update case insensitive check Update link and use casefold() instead of lower(). --- volatility3/framework/layers/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 6d85da982..21e1a938e 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -193,7 +193,7 @@ class RegistryHive(linear.LinearlyMappedLayer): subkeys = node_key[-1].get_subkeys() for subkey in subkeys: # registry keys are not case sensitive so compare likewise - # https://learn.microsoft.com/en-gb/windows/win32/sysinfo/structure-of-the-registry + # https://learn.microsoft.com/en-us/windows/win32/sysinfo/structure-of-the-registry if subkey.get_name().casefold() == key_array[0].casefold(): node_key = node_key + [subkey] found_key, key_array = found_key + [key_array[0]], key_array[1:] From 94ec7d89c09b2a276e79fc4c7561828340d5712a Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 21:55:58 +0000 Subject: [PATCH 298/348] Tiny comment changes --- volatility3/framework/plugins/windows/cmdscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 3dc70d649..0cd0addb2 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -67,7 +67,7 @@ class CmdScan(interfaces.plugins.PluginInterface): Args: conhost_proc: the process object for conhost.exe - size_filter: filter (keep) vads less than this size (bytes) + size_filter: size above which vads will not be returned Returns: A list of tuples of: @@ -100,7 +100,7 @@ class CmdScan(interfaces.plugins.PluginInterface): kernel_layer_name: The name of the layer on which to operate kernel_symbol_table_name: The name of the table containing the kernel symbols config_path: The config path where to find symbol files - procs: list of process objects + procs: List of process objects max_history: An initial set of CommandHistorySize values Returns: From a7b4e2fb45bef981eb54c44a5e0cef87b879058f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:02:15 +0100 Subject: [PATCH 299/348] version check_modules --- volatility3/framework/plugins/linux/check_modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/check_modules.py b/volatility3/framework/plugins/linux/check_modules.py index 9b3594c5e..0ed638d9c 100644 --- a/volatility3/framework/plugins/linux/check_modules.py +++ b/volatility3/framework/plugins/linux/check_modules.py @@ -18,6 +18,7 @@ vollog = logging.getLogger(__name__) class Check_modules(plugins.PluginInterface): """Compares module list to sysfs info, if available""" + _version = (1, 0, 0) _required_framework_version = (2, 0, 0) @classmethod From 2d262e7acf5c9aabb32240c01cd57890b7d57647 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:20:56 +0100 Subject: [PATCH 300/348] cut unnecessary intermediate LinuxUtilityInterface --- .../framework/symbols/linux/utilities/__init__.py | 11 ----------- .../framework/symbols/linux/utilities/tainting.py | 5 ++--- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/__init__.py b/volatility3/framework/symbols/linux/utilities/__init__.py index 4225d444b..e69de29bb 100644 --- a/volatility3/framework/symbols/linux/utilities/__init__.py +++ b/volatility3/framework/symbols/linux/utilities/__init__.py @@ -1,11 +0,0 @@ -from volatility3 import framework -from volatility3.framework import interfaces - - -class LinuxUtilityInterface(interfaces.configuration.VersionableInterface): - """Class with multiple useful Linux functions surrounding a specific piece of functionality.""" - - _version = (2, 1, 1) - _required_framework_version = (2, 0, 0) - - framework.require_interface_version(*_required_framework_version) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index fc2f94109..29d7d2b5b 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -1,11 +1,10 @@ from volatility3 import framework from volatility3.framework import interfaces -from volatility3.framework.symbols.linux.utilities import LinuxUtilityInterface from volatility3.framework.constants import linux as linux_constants from typing import List, Optional -class Tainting(LinuxUtilityInterface): +class Tainting(interfaces.configuration.VersionableInterface): """Tainted kernel and modules parsing capabilities. Relevant Linux kernel functions: @@ -14,7 +13,7 @@ class Tainting(LinuxUtilityInterface): """ _version = (1, 0, 0) - _required_framework_version = (2, 16, 0) + _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 5c70356c27aedc931c03a8e633952b126ef5254b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:23:16 +0100 Subject: [PATCH 301/348] version check_modules requirement --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 042930740..b44f84c7d 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -38,7 +38,7 @@ class Modxview(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="check_modules", plugin=check_modules.Check_modules, - version=(0, 0, 0), + version=(1, 0, 0), ), requirements.PluginRequirement( name="hidden_modules", From 302f9fdf5ba1c24d07d2fce3d0f7c87c3e6bd1f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:24:18 +0100 Subject: [PATCH 302/348] cut unnecessary plugin runner functions --- .../framework/plugins/linux/modxview.py | 78 ++++++------------- 1 file changed, 25 insertions(+), 53 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index b44f84c7d..a247bc2cc 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -53,54 +53,6 @@ class Modxview(interfaces.plugins.PluginInterface): ), ] - @classmethod - def run_lsmod( - cls, context: interfaces.context.ContextInterface, kernel_name: str - ) -> List[extensions.module]: - """Wrapper for the lsmod plugin.""" - return list(lsmod.Lsmod.list_modules(context, kernel_name)) - - @classmethod - def run_check_modules( - cls, - context: interfaces.context.ContextInterface, - kernel_name: str, - ) -> List[extensions.module]: - """Wrapper for the check_modules plugin. - Here, we extract the /sys/module/ list.""" - kernel = context.modules[kernel_name] - sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( - context, kernel_name - ) - - # Convert get_kset_modules() offsets back to module objects - return [ - kernel.object(object_type="module", offset=m_offset, absolute=True) - for m_offset in sysfs_modules.values() - ] - - @classmethod - def run_hidden_modules( - cls, - context: interfaces.context.ContextInterface, - kernel_name: str, - known_modules_addresses: Set[int], - ) -> List[extensions.module]: - """Wrapper for the hidden_modules plugin.""" - modules_memory_boundaries = ( - hidden_modules.Hidden_modules.get_modules_memory_boundaries( - context, kernel_name - ) - ) - return list( - hidden_modules.Hidden_modules.get_hidden_modules( - context, - kernel_name, - known_modules_addresses, - modules_memory_boundaries, - ) - ) - @classmethod def flatten_run_modules_results( cls, run_results: Dict[str, List[extensions.module]], deduplicate: bool = True @@ -140,15 +92,35 @@ class Modxview(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_name] run_results = {} - run_results["lsmod"] = cls.run_lsmod(context, kernel_name) - run_results["check_modules"] = cls.run_check_modules(context, kernel_name) + # lsmod + run_results["lsmod"] = list(lsmod.Lsmod.list_modules(context, kernel_name)) + # check_modules + sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( + context, kernel_name + ) + ## Convert get_kset_modules() offsets back to module objects + run_results["check_modules"] = [ + kernel.object(object_type="module", offset=m_offset, absolute=True) + for m_offset in sysfs_modules.values() + ] + # hidden_modules if run_hidden_modules: - known_module_addresses = set( + known_modules_addresses = set( context.layers[kernel.layer_name].canonicalize(module.vol.offset) for module in run_results["lsmod"] + run_results["check_modules"] ) - run_results["hidden_modules"] = cls.run_hidden_modules( - context, kernel_name, known_module_addresses + modules_memory_boundaries = ( + hidden_modules.Hidden_modules.get_modules_memory_boundaries( + context, kernel_name + ) + ) + run_results["hidden_modules"] = list( + hidden_modules.Hidden_modules.get_hidden_modules( + context, + kernel_name, + known_modules_addresses, + modules_memory_boundaries, + ) ) return run_results From 4115c26e7cc119a68aa33fff7f5b8a730b5b2c69 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:24:44 +0100 Subject: [PATCH 303/348] bump framework req to 2.18.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index a247bc2cc..34f5bac8f 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 16, 0) + _required_framework_version = (2, 18, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From bd82f4f33d860cb067e379600da5bbc74f9e2247 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:25:07 +0100 Subject: [PATCH 304/348] 2.16.0 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 24f96fa89..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 16 # Number of changes that only add to the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From d956742db98610dfa94678ffb98d53c5b6bcd161 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:31:39 +0100 Subject: [PATCH 305/348] remove typing.Set import --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 34f5bac8f..3655200e8 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import List, Dict, Set, Iterator +from typing import List, Dict, Iterator from volatility3.plugins.linux import lsmod, check_modules, hidden_modules from volatility3.framework import interfaces from volatility3.framework.configuration import requirements From b5bc54cfaed91f4d615790305c80ce802658dafe Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:18:34 +0000 Subject: [PATCH 306/348] Use in-place subtraction Also tweak comments. --- volatility3/framework/renderers/conversion.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index e48684b31..f848b2dad 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -18,7 +18,7 @@ def wintime_to_datetime( unix_time = wintime // 10000000 if unix_time == 0: return renderers.NotApplicableValue() - unix_time = unix_time - 11644473600 + unix_time -= 11644473600 try: return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc) # Windows sometimes throws OSErrors rather than ValueError/OverflowError when it can't convert a value @@ -71,7 +71,7 @@ def round(addr: int, align: int, up: bool = False) -> int: Args: addr: the address align: the alignment value - up: Whether to round up or not + up: whether to round up or not Returns: The aligned address @@ -122,11 +122,12 @@ def convert_port(port_as_integer): def convert_network_four_tuple(family, four_tuple): - """Converts the connection four_tuple: (source ip, source port, dest ip, - dest port) + """Converts the connection four_tuple: + + (source ip, source port, dest ip, dest port) into their string equivalents. IP addresses are expected as a tuple - of unsigned shorts Ports are converted to proper endianness as well + of unsigned shorts. Ports are converted to proper endianness as well. """ if family == socket.AF_INET: From 43ac6c4d6271c928d9bcdaf6407e01e1c96d7cf9 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 7 Jan 2025 10:24:46 -0600 Subject: [PATCH 307/348] Fix copy-pasted module docstrings This updates the module docstrings for 5 modules that duplicate the docstring from the `proc` module. This was presumably the result of using the `proc` module as a template for the others. --- volatility3/framework/plugins/linux/bash.py | 4 ++-- volatility3/framework/plugins/linux/check_afinfo.py | 4 ++-- volatility3/framework/plugins/linux/check_syscall.py | 3 +-- volatility3/framework/plugins/linux/elfs.py | 4 ++-- volatility3/framework/plugins/linux/lsmod.py | 3 +-- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index 056e3cd51..8acfeb848 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that recovers bash command history +from bash process memory.""" import datetime import struct diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index 201a443f7..7aa3cbdd2 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that verifies the operation function +pointers of network protocols.""" import logging from typing import List diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 3537a9fa1..13d312f2f 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -1,8 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that checks the system call table for hooks.""" import contextlib import logging from typing import List diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 2fd740941..0d1c9c2dd 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin for enumerating memory-mapped +ELF files across all processes.""" import logging from typing import List, Optional, Type diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index 49e990e93..e9a2a7137 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -1,8 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that lists loaded kernel modules.""" import logging from typing import List, Iterable From 32cb6e11f6abe86ce5284e1a618bae9ab1cd4a5f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 7 Jan 2025 19:41:02 +0000 Subject: [PATCH 308/348] Change one letter of a typo --- volatility3/framework/plugins/windows/driverscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/driverscan.py b/volatility3/framework/plugins/windows/driverscan.py index 24d81c3d5..d388ffbb7 100644 --- a/volatility3/framework/plugins/windows/driverscan.py +++ b/volatility3/framework/plugins/windows/driverscan.py @@ -64,7 +64,7 @@ class DriverScan(interfaces.plugins.PluginInterface): names associated with a driver Args: - driver: A Eriver object + driver: A Driver object Returns: A tuple of strings of (driver name, service key, driver alt. name) From 585901105275a015a3c4326e486f4e2a52d8eb12 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 12:35:04 +0100 Subject: [PATCH 309/348] introduce customizable plugin arparse epilog --- volatility3/cli/__init__.py | 3 +++ volatility3/framework/interfaces/plugins.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 6172a17f3..87caaece6 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -368,6 +368,9 @@ class CommandLine: help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, ) + epilog = getattr(plugin_list[plugin], "_argparse_epilog", None) + if epilog is not None: + plugin_parser.epilog = epilog self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) ### diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index f763815a6..6cd72f02e 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -112,6 +112,8 @@ class PluginInterface( # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" + _argparse_epilog: str = None + """Display additional description of the plugin after the description of the arguments. See: https://docs.python.org/3/library/argparse.html#epilog""" def __init__( self, From 530617a700e259f69d53f62f08ccc3382bcdd057 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:52:54 +0000 Subject: [PATCH 310/348] Small readability improvements --- volatility3/framework/automagic/mac.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index f3679d160..a883028d2 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -101,7 +101,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVVV, - f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}", + f"Skipping invalid idlepml4_ptr: {idlepml4_ptr:#x}", ) continue @@ -112,7 +112,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if tmp_dtb % 4096: vollog.log( constants.LOGLEVEL_VVV, - f"Skipping non-page aligned DTB: 0x{tmp_dtb:0x}", + f"Skipping non-page aligned DTB: {tmp_dtb:#x}", ) continue @@ -136,7 +136,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): new_layer.config["kernel_virtual_offset"] = kaslr_shift if new_layer and dtb: - vollog.debug(f"DTB was found at: 0x{dtb:0x}") + vollog.debug(f"DTB was found at: {dtb:#x}") return new_layer vollog.debug("No suitable mac banner could be matched") return None @@ -182,7 +182,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) tmp_aslr_shift = offset - cls.virtual_to_physical_address( version_json_address @@ -208,7 +208,6 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): continue aslr_shift = tmp_aslr_shift & 0xFFFFFFFF - break vollog.log(constants.LOGLEVEL_VVVV, f"Mac find_aslr returned: {aslr_shift:0x}") @@ -219,9 +218,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): """Converts a virtual mac address to a physical one (does not account of ASLR)""" if addr > 0xFFFFFF8000000000: - addr = addr - 0xFFFFFF8000000000 + addr -= 0xFFFFFF8000000000 else: - addr = addr - 0xFF8000000000 + addr -= 0xFF8000000000 return addr From a7661d45e78b10bc736946425055755c9627d111 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 10 Jan 2025 09:21:21 -0600 Subject: [PATCH 311/348] Windows: Certificates - handle uncaught RegistryFormatException Changes variable import to module import, and catches an unhandled `RegistryFormatException` in certificates.py --- .../plugins/windows/registry/certificates.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 8587b3719..a83badb90 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,11 +1,11 @@ import contextlib import logging import struct -from typing import List, Iterator, Optional, Tuple, Type +from typing import Iterator, List, Optional, Tuple, Type from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes +from volatility3.framework.symbols.windows.extensions import registry from volatility3.plugins.windows.registry import hivelist, printkey vollog = logging.getLogger(__name__) @@ -81,7 +81,11 @@ class Certificates(interfaces.plugins.PluginInterface): "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - with contextlib.suppress(KeyError, exceptions.InvalidAddressException): + with contextlib.suppress( + KeyError, + registry.RegistryFormatException, + exceptions.InvalidAddressException, + ): # Walk it node_path = hive.get_key(top_key, return_list=True) for ( @@ -92,7 +96,11 @@ class Certificates(interfaces.plugins.PluginInterface): _volatility, node, ) in printkey.PrintKey.key_iterator(hive, node_path, recurse=True): - if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": + if ( + not is_key + and registry.RegValueTypes(node.Type) + == registry.RegValueTypes.REG_BINARY + ): name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = ( key_path.casefold().index(top_key.casefold()) From 96eca6e0162a77699c2befcce6df16f7deac4d23 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 19:38:07 +0100 Subject: [PATCH 312/348] more compact _argparse_epilog --- volatility3/cli/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 87caaece6..37923362a 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -368,9 +368,9 @@ class CommandLine: help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, ) - epilog = getattr(plugin_list[plugin], "_argparse_epilog", None) - if epilog is not None: - plugin_parser.epilog = epilog + plugin_parser.epilog = getattr( + plugin_list[plugin], "_argparse_epilog", None + ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) ### From 615d1d5a2e85dcd2f9d65493690a474c15f691cd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 19:49:25 +0100 Subject: [PATCH 313/348] more compact _argparse_epilog --- volatility3/cli/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 37923362a..fde4fcc6d 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -367,9 +367,7 @@ class CommandLine: plugin, help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, - ) - plugin_parser.epilog = getattr( - plugin_list[plugin], "_argparse_epilog", None + epilog=getattr(plugin_list[plugin], "_argparse_epilog", None), ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) From a26ff8fa6e6ba03a6ea3ebe6c5f3b38b3a4d8851 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 10 Jan 2025 19:03:51 +0000 Subject: [PATCH 314/348] Small readability improvements --- volatility3/framework/automagic/mac.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index a883028d2..3b16eb353 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -184,12 +184,12 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): for offset, banner in offset_generator: banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) - tmp_aslr_shift = offset - cls.virtual_to_physical_address( + aslr_shift = offset - cls.virtual_to_physical_address( version_json_address ) major_string = context.layers[layer_name].read( - version_major_phys_offset + tmp_aslr_shift, 4 + version_major_phys_offset + aslr_shift, 4 ) major = struct.unpack(" Date: Fri, 10 Jan 2025 19:08:56 +0000 Subject: [PATCH 315/348] Small readability improvements --- volatility3/framework/automagic/mac.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 3b16eb353..94c259463 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -184,9 +184,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): for offset, banner in offset_generator: banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) - aslr_shift = offset - cls.virtual_to_physical_address( - version_json_address - ) + aslr_shift = offset - cls.virtual_to_physical_address(version_json_address) major_string = context.layers[layer_name].read( version_major_phys_offset + aslr_shift, 4 From 1cf0232d25fa6dffa21ae3c281e1f568bbb280ab Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 21:19:31 +0100 Subject: [PATCH 316/348] less specific argparse epilog reference --- volatility3/cli/__init__.py | 2 +- volatility3/framework/interfaces/plugins.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index fde4fcc6d..82a2a4205 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -367,7 +367,7 @@ class CommandLine: plugin, help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, - epilog=getattr(plugin_list[plugin], "_argparse_epilog", None), + epilog=plugin_list[plugin].additional_description, ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 6cd72f02e..7ad78d0ba 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -112,7 +112,7 @@ class PluginInterface( # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" - _argparse_epilog: str = None + additional_description: str = None """Display additional description of the plugin after the description of the arguments. See: https://docs.python.org/3/library/argparse.html#epilog""" def __init__( From 7913fb2bb0aac4cc390ce6e42ad6621115f0ae7c Mon Sep 17 00:00:00 2001 From: ikelos Date: Fri, 10 Jan 2025 21:07:08 +0000 Subject: [PATCH 317/348] Revert "Small readability improvements" --- volatility3/framework/automagic/mac.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 94c259463..f3679d160 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -101,7 +101,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVVV, - f"Skipping invalid idlepml4_ptr: {idlepml4_ptr:#x}", + f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}", ) continue @@ -112,7 +112,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if tmp_dtb % 4096: vollog.log( constants.LOGLEVEL_VVV, - f"Skipping non-page aligned DTB: {tmp_dtb:#x}", + f"Skipping non-page aligned DTB: 0x{tmp_dtb:0x}", ) continue @@ -136,7 +136,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): new_layer.config["kernel_virtual_offset"] = kaslr_shift if new_layer and dtb: - vollog.debug(f"DTB was found at: {dtb:#x}") + vollog.debug(f"DTB was found at: 0x{dtb:0x}") return new_layer vollog.debug("No suitable mac banner could be matched") return None @@ -182,12 +182,14 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) - aslr_shift = offset - cls.virtual_to_physical_address(version_json_address) + tmp_aslr_shift = offset - cls.virtual_to_physical_address( + version_json_address + ) major_string = context.layers[layer_name].read( - version_major_phys_offset + aslr_shift, 4 + version_major_phys_offset + tmp_aslr_shift, 4 ) major = struct.unpack(" 0xFFFFFF8000000000: - addr -= 0xFFFFFF8000000000 + addr = addr - 0xFFFFFF8000000000 else: - addr -= 0xFF8000000000 + addr = addr - 0xFF8000000000 return addr From 884237534142ec10ba6e7386eedc06ef30d277d0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 23:28:02 +0100 Subject: [PATCH 318/348] 2.15.0 -> 2.16.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 2f0c53093..24f96fa89 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 15 # Number of changes that only add to the interface +VERSION_MINOR = 16 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 0e4e7518447837b9c7f0f30203155b3a3fee0c3a Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 11 Jan 2025 14:05:36 +0100 Subject: [PATCH 319/348] stateless classmethods --- .../symbols/linux/utilities/tainting.py | 67 +++++++++++++------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 29d7d2b5b..552f51b98 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -17,26 +17,22 @@ class Tainting(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) - def __init__( - self, + @classmethod + def _get_kernel_taint_flags_list( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self._kernel = context.modules[kernel_module_name] - - @property - def _kernel_taint_flags_list( - self, ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self._kernel.has_symbol("taint_flags"): - return list(self._kernel.object_from_symbol("taint_flags")) + kernel = context.modules[kernel_module_name] + if kernel.has_symbol("taint_flags"): + return list(kernel.object_from_symbol("taint_flags")) return None + @classmethod def _module_flags_taint_pre_4_10_rc1( - self, taints: int, is_module: bool = False + cls, + taints: int, + is_module: bool = False, ) -> str: """Convert the module's taints value to a 1-1 character mapping. Relies on statically defined taints mappings in the framework. @@ -58,8 +54,13 @@ class Tainting(interfaces.configuration.VersionableInterface): return taints_string + @classmethod def _module_flags_taint_post_4_10_rc1( - self, taints: int, is_module: bool = False + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, ) -> str: """Convert the module's taints value to a 1-1 character mapping. Relies on kernel symbol embedded taints definitions. @@ -78,7 +79,9 @@ class Tainting(interfaces.configuration.VersionableInterface): The raw taints string. """ taints_string = "" - for taint_bit, taint_flag in enumerate(self._kernel_taint_flags_list): + for taint_bit, taint_flag in enumerate( + cls._get_kernel_taint_flags_list(context, kernel_module_name) + ): if is_module and not taint_flag.module: continue c_true = chr(taint_flag.c_true) @@ -90,7 +93,14 @@ class Tainting(interfaces.configuration.VersionableInterface): return taints_string - def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + @classmethod + def get_taints_as_plain_string( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, + ) -> str: """Convert the taints value to a 1-1 character mapping. Args: @@ -103,11 +113,22 @@ class Tainting(interfaces.configuration.VersionableInterface): - module_flags_taint kernel function """ - if self._kernel_taint_flags_list: - return self._module_flags_taint_post_4_10_rc1(taints, is_module) - return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + if cls._get_kernel_taint_flags_list(context, kernel_module_name): + return cls._module_flags_taint_post_4_10_rc1( + context, kernel_module_name, taints, is_module + ) + return cls._module_flags_taint_pre_4_10_rc1( + context, kernel_module_name, taints, is_module + ) - def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + @classmethod + def get_taints_parsed( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, + ) -> List[str]: """Convert the taints string to a 1-1 descriptor mapping. Args: @@ -121,7 +142,9 @@ class Tainting(interfaces.configuration.VersionableInterface): - module_flags_taint kernel function """ comprehensive_taints = [] - for character in self.get_taints_as_plain_string(taints, is_module): + for character in cls.get_taints_as_plain_string( + context, kernel_module_name, taints, is_module + ): taint_flag = linux_constants.TAINT_FLAGS.get(character) if not taint_flag: comprehensive_taints.append(f"") From 6817d2c765fb5117a8ec6adb92cf343d37a92595 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:00:50 +1100 Subject: [PATCH 320/348] linux: ensure process listing functions yield only valid tasks --- volatility3/framework/plugins/linux/pslist.py | 5 ++- .../symbols/linux/extensions/__init__.py | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 37cf000fc..931acf29a 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -34,7 +34,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 13, 0) - _version = (4, 0, 0) + _version = (4, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -250,6 +250,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Note that the init_task itself is not yielded, since "ps" also never shows it. for task in init_task.tasks: + if not task.is_valid(): + continue + if filter_func(task): continue diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..a50b8ae09 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -307,6 +307,36 @@ class module(generic.GenericIntelProcess): class task_struct(generic.GenericIntelProcess): + def is_valid(self) -> bool: + layer = self._context.layers[self.vol.layer_name] + # Make sure the entire task content is readable + if not layer.is_valid(self.vol.offset, self.vol.size): + return False + + if self.pid < 0: + return False + + if not (self.signal and self.signal.is_readable()): + return False + + if not (self.nsproxy and self.nsproxy.is_readable()): + return False + + if not (self.real_parent and self.real_parent.is_readable()): + return False + + if self.active_mm and not self.active_mm.is_readable(): + return False + + if self.mm: + if not self.mm.is_readable(): + return False + + if self.mm != self.active_mm: + return False + + return True + def add_process_layer( self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: @@ -401,6 +431,8 @@ class task_struct(generic.GenericIntelProcess): tasks_iterable = self._get_tasks_iterable() threads_seen = set([self.vol.offset]) for task in tasks_iterable: + if not task.is_valid(): + continue if task.vol.offset not in threads_seen: threads_seen.add(task.vol.offset) yield task From 093b12b7cdf4a1623a5d534309f0673c0311cc6b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:52:45 +1100 Subject: [PATCH 321/348] Linux and Windows: Ensure linked list object extensions consistently yield valid entries --- .../symbols/linux/extensions/__init__.py | 42 ++++++++++------- .../symbols/windows/extensions/__init__.py | 47 +++++++++---------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..2065b3bb4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1209,35 +1209,43 @@ class list_head(objects.StructType, collections.abc.Iterable): Objects of the type specified via the "symbol_type" argument. """ - layer = layer or self.vol.layer_name + layer_name = layer or self.vol.layer_name + + trans_layer = self._context.layers[layer_name] + if not trans_layer.is_valid(self.vol.offset): + return None relative_offset = self._context.symbol_space.get_type( symbol_type ).relative_child_offset(member) - direction = "prev" - if forward: - direction = "next" - try: - link = getattr(self, direction).dereference() - except exceptions.InvalidAddressException: + direction = "next" if forward else "prev" + + link_ptr = getattr(self, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() + if not sentinel: - yield self._context.object( - symbol_type, layer, offset=self.vol.offset - relative_offset - ) + obj_offset = self.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + + yield self._context.object(symbol_type, layer_name, offset=obj_offset) + seen = {self.vol.offset} while link.vol.offset not in seen: - obj = self._context.object( - symbol_type, layer, offset=link.vol.offset - relative_offset - ) - yield obj + obj_offset = link.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + + yield self._context.object(symbol_type, layer_name, offset=obj_offset) seen.add(link.vol.offset) - try: - link = getattr(link, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(link, direction) + if not (link_ptr and link_ptr.is_readable()): break + link = link_ptr.dereference() def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index f12fd3f5b..214002f49 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -962,56 +962,55 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): ) -> Iterator[interfaces.objects.ObjectInterface]: """Returns an iterator of the entries in the list.""" - layer = layer or self.vol.layer_name + layer_name = layer or self.vol.layer_name + native_layer_name = layer_name or self.vol.native_layer_name + + trans_layer = self._context.layers[layer_name] + if not trans_layer.is_valid(self.vol.offset): + return None relative_offset = self._context.symbol_space.get_type( symbol_type ).relative_child_offset(member) - direction = "Blink" - if forward: - direction = "Flink" + direction = "Flink" if forward else "Blink" - trans_layer = self._context.layers[layer] - - try: - is_valid = trans_layer.is_valid(self.vol.offset) - if not is_valid: - return None - - link = getattr(self, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(self, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() if not sentinel: + obj_offset = self.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + yield self._context.object( symbol_type, - layer, - offset=self.vol.offset - relative_offset, - native_layer_name=layer or self.vol.native_layer_name, + layer_name, + offset=obj_offset, + native_layer_name=native_layer_name, ) seen = {self.vol.offset} while link.vol.offset not in seen: obj_offset = link.vol.offset - relative_offset - if not trans_layer.is_valid(obj_offset): return None - obj = self._context.object( + yield self._context.object( symbol_type, - layer, + layer_name, offset=obj_offset, - native_layer_name=layer or self.vol.native_layer_name, + native_layer_name=native_layer_name, ) - yield obj seen.add(link.vol.offset) - try: - link = getattr(link, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(link, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) From 0d9715136cc9cc96637996b2bb027a76b8b5e87a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:13:44 +1100 Subject: [PATCH 322/348] Linux: Ensure VMA enumration functions yield only valid objects consistently --- .../symbols/linux/extensions/__init__.py | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..61562270a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -811,23 +811,30 @@ class mm_struct(objects.StructType): def _get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or - _get_mmap_iter() automatically as required.""" + _get_mmap_iter() automatically as required. + + Yields: + vm_area_struct objects + """ if not self.has_member("mmap"): raise AttributeError( "_get_mmap_iter called on mm_struct where no mmap member exists." ) - if not self.mmap: + vma_pointer = self.mmap + if not (vma_pointer and vma_pointer.is_readable()): return None - yield self.mmap + vma_object = vma_pointer.dereference() + yield vma_object - seen = {self.mmap.vol.offset} - link = self.mmap.vm_next + seen = {vma_pointer} + vma_pointer = vma_pointer.vm_next - while link != 0 and link.vol.offset not in seen: - yield link - seen.add(link.vol.offset) - link = link.vm_next + while vma_pointer and vma_pointer.is_readable() and vma_pointer not in seen: + vma_object = vma_pointer.dereference() + yield vma_object + seen.add(vma_pointer) + vma_pointer = vma_pointer.vm_next # TODO: As of version 3.0.0 this method should be removed def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: @@ -842,7 +849,11 @@ class mm_struct(objects.StructType): def _get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mm_mt member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or - get_mmap_iter() automatically as required.""" + get_mmap_iter() automatically as required. + + Yields: + vm_area_struct objects + """ if not self.has_member("mm_mt"): raise AttributeError( @@ -850,20 +861,27 @@ class mm_struct(objects.StructType): ) symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): - # convert pointer to vm_area_struct and yield - vma = self._context.object( + # Convert pointer to vm_area_struct and yield + vma_object = self._context.object( symbol_table_name + constants.BANG + "vm_area_struct", layer_name=self.vol.native_layer_name, offset=vma_pointer, ) - yield vma + yield vma_object def get_vma_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" + """Returns an iterator for the VMAs in an mm_struct. + Automatically choosing the mmap or mm_mt as required. + + Yields: + vm_area_struct objects + """ if self.has_member("mmap"): + # kernels < 6.1 yield from self._get_mmap_iter() elif self.has_member("mm_mt"): + # kernels >= 6.1 d4af56c5c7c6781ca6ca8075e2cf5bc119ed33d1 yield from self._get_maple_tree_iter() else: raise AttributeError("Unable to find mmap or mm_mt in mm_struct") From 8bc04529350c3ce5a927099a72bc4e419a049db5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 11:21:23 +1100 Subject: [PATCH 323/348] linux: Improve compatibility with ancient kernels --- .../symbols/linux/extensions/__init__.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index a50b8ae09..4a1a263dd 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -316,16 +316,26 @@ class task_struct(generic.GenericIntelProcess): if self.pid < 0: return False - if not (self.signal and self.signal.is_readable()): + if self.has_member("signal") and not ( + self.signal and self.signal.is_readable() + ): return False - if not (self.nsproxy and self.nsproxy.is_readable()): + if self.has_member("nsproxy") and not ( + self.nsproxy and self.nsproxy.is_readable() + ): return False - if not (self.real_parent and self.real_parent.is_readable()): + if self.has_member("real_parent") and not ( + self.real_parent and self.real_parent.is_readable() + ): return False - if self.active_mm and not self.active_mm.is_readable(): + if ( + self.has_member("active_mm") + and self.active_mm + and not self.active_mm.is_readable() + ): return False if self.mm: From 7fc2af5b4ecf4b1ced5c71357d164981b05ed309 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 11:39:48 +1100 Subject: [PATCH 324/348] linux: Add an additional quick check before validating pointer readability --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 4a1a263dd..e2c9454f6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -313,7 +313,7 @@ class task_struct(generic.GenericIntelProcess): if not layer.is_valid(self.vol.offset, self.vol.size): return False - if self.pid < 0: + if self.pid < 0 or self.tgid < 0: return False if self.has_member("signal") and not ( From 1a84f96c70060bc09aab3c1b3348d980f8b9bc0e Mon Sep 17 00:00:00 2001 From: Kerry Goodwine Date: Thu, 9 Jan 2025 15:49:24 -0500 Subject: [PATCH 325/348] Actions: Add new workflow for generating windows EXEs with pyinstaller --- .github/workflows/build-pyinstaller.yml | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/build-pyinstaller.yml diff --git a/.github/workflows/build-pyinstaller.yml b/.github/workflows/build-pyinstaller.yml new file mode 100644 index 000000000..bcba95403 --- /dev/null +++ b/.github/workflows/build-pyinstaller.yml @@ -0,0 +1,50 @@ +name: build-pyinstaller +on: + push: + branches: + - stable + - develop + - 'release/**' + pull_request: + branches: + - stable + - 'release/**' + +jobs: + + exe: + runs-on: windows-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pyinstaller + + - name: Pyinstall executable + run: | + pyinstaller --clean -y vol.spec + pyinstaller --clean -y volshell.spec + + - name: Move files + run: | + mv dist/vol.exe vol.exe + mv dist/volshell.exe volshell.exe + + - name: Archive + uses: actions/upload-artifact@v4 + with: + name: volatility3-pyinstaller + path: | + vol.exe + volshell.exe + README.md + LICENSE.txt From 9f08af47b161579bf31f9d45c8b248c23861388a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 12:51:52 +1100 Subject: [PATCH 326/348] Linux: Add support for Intel 32bit with PAE --- volatility3/framework/automagic/linux.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f22cae012..542d26a8d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -71,6 +71,12 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): elif "init_level4_pgt" in table.symbols: layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" + elif ( + "pkmap_count" in table.symbols + and table.get_symbol("pkmap_count").type.count == 512 + ): + layer_class = intel.LinuxIntelPAE + dtb_symbol_name = "swapper_pg_dir" else: layer_class = intel.LinuxIntel dtb_symbol_name = "swapper_pg_dir" From 28c74f8c1b853df3680de14f6fdc22958516a9c2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 13:23:39 +1100 Subject: [PATCH 327/348] Linux: Add support for Intel 32bit with PAE in early kernels, including versions 2.3.27 and 2.3.28. --- volatility3/framework/automagic/linux.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 542d26a8d..cb4f3cc64 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -71,10 +71,9 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): elif "init_level4_pgt" in table.symbols: layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" - elif ( - "pkmap_count" in table.symbols - and table.get_symbol("pkmap_count").type.count == 512 - ): + elif "pkmap_count" in table.symbols and table.get_symbol( + "pkmap_count" + ).type.count in (512, 2048): layer_class = intel.LinuxIntelPAE dtb_symbol_name = "swapper_pg_dir" else: From b27f98fed258b597e043a5c80304d8489da27b32 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 14:37:35 +1100 Subject: [PATCH 328/348] linux: pslist: fix task credentials rendering --- volatility3/framework/plugins/linux/pslist.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 37cf000fc..77b57e000 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -179,6 +179,10 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_output = "VMA start matching task start_code not found" return file_output + @staticmethod + def _format_cred(cred): + return renderers.NotAvailableValue() if cred is None else cred + def _generator( self, pid_filter: Callable[[Any], bool], @@ -212,16 +216,21 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): task_fields = self.get_task_fields(task, decorate_comm) + task_uid = self._format_cred(task_fields.uid) + task_gid = self._format_cred(task_fields.gid) + task_euid = self._format_cred(task_fields.euid) + task_egid = self._format_cred(task_fields.egid) + yield 0, ( format_hints.Hex(task_fields.offset), task_fields.user_pid, task_fields.user_tid, task_fields.user_ppid, task_fields.name, - task_fields.uid or renderers.NotAvailableValue(), - task_fields.gid or renderers.NotAvailableValue(), - task_fields.euid or renderers.NotAvailableValue(), - task_fields.egid or renderers.NotAvailableValue(), + task_uid, + task_gid, + task_euid, + task_egid, task_fields.creation_time or renderers.NotAvailableValue(), file_output, ) From b447bfa81c36e91c3cf30bdc432e6eba48afbc53 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:24:38 +0100 Subject: [PATCH 329/348] remove module tainting proxies --- .../framework/plugins/linux/modxview.py | 16 ++++++++++-- .../symbols/linux/extensions/__init__.py | 25 ------------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 3655200e8..69c6ac8bb 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -140,9 +140,21 @@ class Modxview(interfaces.plugins.PluginInterface): seen_addresses.add(module.vol.offset) if self.config.get("plain_taints"): - taints = module.get_taints_as_plain_string() + taints = tainting.Tainting.get_taints_as_plain_string( + self.context, + kernel_name, + module.taints, + True, + ) else: - taints = ",".join(module.get_taints_parsed()) + taints = ",".join( + tainting.Tainting.get_taints_parsed( + self.context, + kernel_name, + module.taints, + True, + ) + ) yield ( 0, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index ac2f87df0..289d6c0a4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -19,7 +19,6 @@ from volatility3.framework.layers import linear, intel from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf -from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -279,30 +278,6 @@ class module(generic.GenericIntelProcess): return None - def get_taints_as_plain_string(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Convenient wrapper around framework's Tainting capabilities. - - Returns: - The raw taints string. - """ - return tainting.Tainting( - self._context, - linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, - ).get_taints_as_plain_string(self.taints, True) - - def get_taints_parsed(self) -> List[str]: - """Convert the module's taints string to a 1-1 descriptor mapping. - Convenient wrapper around framework's Tainting capabilities. - - Returns: - A comprehensive (user-friendly) taint descriptor list. - """ - return tainting.Tainting( - self._context, - linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, - ).get_taints_parsed(self.taints, True) - @property def section_symtab(self): if self.has_member("kallsyms"): From 94704c6674d7f5fb9d57698faa0d9ed943c6158c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:26:57 +0100 Subject: [PATCH 330/348] 2.16.0 -> 2.17.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 832b2a5ba..3d68ab810 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 18 # Number of changes that only add to the interface +VERSION_MINOR = 17 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From cd8690a8059836f7e216c211c4397924ae311c84 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:27:22 +0100 Subject: [PATCH 331/348] require framework version 2.17.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 69c6ac8bb..3c2c5f05e 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 18, 0) + _required_framework_version = (2, 17, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From adf81bc74a388d6ff5bffabe588bc5ca72147506 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 16 Jan 2025 19:59:31 +0000 Subject: [PATCH 332/348] Update copyright dates --- README.md | 2 +- doc/source/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cc33d3cc4..b74bdab0b 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ The latest generated copy of the documentation can be found at: Date: Sat, 18 Jan 2025 02:19:11 +0100 Subject: [PATCH 333/348] pre-process module triaging to improve readability --- .../framework/plugins/linux/modxview.py | 71 ++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 3c2c5f05e..125c1cc33 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -82,7 +82,8 @@ class Modxview(interfaces.plugins.PluginInterface): kernel_name: str, run_hidden_modules: bool = True, ) -> Dict[str, List[extensions.module]]: - """Run module scanning plugins and aggregate the results. + """Run module scanning plugins and aggregate the results. It is designed + to not operate any inter-plugin results triage. Args: run_hidden_modules: specify if the hidden_modules plugin should be run @@ -128,46 +129,50 @@ class Modxview(interfaces.plugins.PluginInterface): def _generator(self): kernel_name = self.config["kernel"] run_results = self.run_modules_scanners(self.context, kernel_name) - modules_offsets = {} - for key in ["lsmod", "check_modules", "hidden_modules"]: - modules_offsets[key] = set(module.vol.offset for module in run_results[key]) + aggregated_modules = {} + # We want to be explicit on the plugins results we are interested in + for plugin_name in ["lsmod", "check_modules", "hidden_modules"]: + # Iterate over each recovered module + for module in run_results[plugin_name]: + # Use offsets as unique keys, whether a module + # appears in many plugin runs or not + if aggregated_modules.get(module.vol.offset): + # Append the plugin to the list of originating plugins + aggregated_modules[module.vol.offset][1].append(plugin_name) + else: + aggregated_modules[module.vol.offset] = (module, [plugin_name]) - seen_addresses = set() - for modules_list in run_results.values(): - for module in modules_list: - if module.vol.offset in seen_addresses: - continue - seen_addresses.add(module.vol.offset) - - if self.config.get("plain_taints"): - taints = tainting.Tainting.get_taints_as_plain_string( + for module_offset, (module, originating_plugins) in aggregated_modules.items(): + # Tainting parsing capabilities applied to the module + if self.config.get("plain_taints"): + taints = tainting.Tainting.get_taints_as_plain_string( + self.context, + kernel_name, + module.taints, + True, + ) + else: + taints = ",".join( + tainting.Tainting.get_taints_parsed( self.context, kernel_name, module.taints, True, ) - else: - taints = ",".join( - tainting.Tainting.get_taints_parsed( - self.context, - kernel_name, - module.taints, - True, - ) - ) - - yield ( - 0, - ( - module.get_name() or NotAvailableValue(), - format_hints.Hex(module.vol.offset), - module.vol.offset in modules_offsets["lsmod"], - module.vol.offset in modules_offsets["check_modules"], - module.vol.offset in modules_offsets["hidden_modules"], - taints or NotAvailableValue(), - ), ) + yield ( + 0, + ( + module.get_name() or NotAvailableValue(), + format_hints.Hex(module_offset), + "lsmod" in originating_plugins, + "check_modules" in originating_plugins, + "hidden_modules" in originating_plugins, + taints or NotAvailableValue(), + ), + ) + def run(self): columns = [ ("Name", str), From 3b679cbafbb50a2c986a63efd223cf9088bbc330 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:26:43 +0100 Subject: [PATCH 334/348] explicit None check --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 125c1cc33..c74bf28e8 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -136,7 +136,7 @@ class Modxview(interfaces.plugins.PluginInterface): for module in run_results[plugin_name]: # Use offsets as unique keys, whether a module # appears in many plugin runs or not - if aggregated_modules.get(module.vol.offset): + if aggregated_modules.get(module.vol.offset, None) is not None: # Append the plugin to the list of originating plugins aggregated_modules[module.vol.offset][1].append(plugin_name) else: From bb6556dbc0145682866d56bb2608b5b841e381e8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:37:52 +0100 Subject: [PATCH 335/348] correct arguments for pre_4_10_rc1 --- volatility3/framework/symbols/linux/utilities/tainting.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 552f51b98..14b69d3d6 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -117,9 +117,7 @@ class Tainting(interfaces.configuration.VersionableInterface): return cls._module_flags_taint_post_4_10_rc1( context, kernel_module_name, taints, is_module ) - return cls._module_flags_taint_pre_4_10_rc1( - context, kernel_module_name, taints, is_module - ) + return cls._module_flags_taint_pre_4_10_rc1(taints, is_module) @classmethod def get_taints_parsed( From 0b82f731375583076abdfabd332ce067612d69f5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:45:30 +0100 Subject: [PATCH 336/348] functools caching and doc. --- .../framework/symbols/linux/utilities/tainting.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 14b69d3d6..c1136436e 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -1,3 +1,5 @@ +import functools + from volatility3 import framework from volatility3.framework import interfaces from volatility3.framework.constants import linux as linux_constants @@ -18,11 +20,18 @@ class Tainting(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) @classmethod + @functools.lru_cache def _get_kernel_taint_flags_list( cls, context: interfaces.context.ContextInterface, kernel_module_name: str, ) -> Optional[List[interfaces.objects.ObjectInterface]]: + """Determine whether the kernel embeds taint flags definition + in-memory or not. + + Returns: + A list of "taint_flag" kernel objects if taint_flags symbok exists + """ kernel = context.modules[kernel_module_name] if kernel.has_symbol("taint_flags"): return list(kernel.object_from_symbol("taint_flags")) From 8095924e8a926990f6002f16d2c7259c5c750980 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:46:13 +0100 Subject: [PATCH 337/348] typo --- volatility3/framework/symbols/linux/utilities/tainting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index c1136436e..2360401d5 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -30,7 +30,7 @@ class Tainting(interfaces.configuration.VersionableInterface): in-memory or not. Returns: - A list of "taint_flag" kernel objects if taint_flags symbok exists + A list of "taint_flag" kernel objects if taint_flags symbol exists """ kernel = context.modules[kernel_module_name] if kernel.has_symbol("taint_flags"): From 0849c163a1c517fa8595f9cc7610a737d1904fc2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:48:37 +0100 Subject: [PATCH 338/348] appropriate symbols type hinting --- volatility3/framework/contexts/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index f527544c0..17a91e827 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -337,7 +337,7 @@ class Module(interfaces.context.ModuleInterface): ) @property - def symbols(self): + def symbols(self) -> Iterable[str]: return self.context.symbol_space[self.symbol_table_name].symbols get_symbol = get_module_wrapper("get_symbol") From d46cb3328d07ae2216045ba3fc33679c2ab13fbc Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:48:46 +0100 Subject: [PATCH 339/348] appropriate symbols type hinting --- volatility3/framework/interfaces/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index a87e0f1e8..2b95a18ad 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -303,8 +303,8 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): """Determines whether an enumeration is present in the module's symbol table.""" @abstractmethod - def symbols(self) -> List: - """Lists the symbols contained in the symbol table for this module""" + def symbols(self) -> Iterable[str]: + """Returns an iterable of the symbols contained in the symbol table for this module""" @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: From fb93d2333b8d3854d348548decc76ac67f358699 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:50:14 +0100 Subject: [PATCH 340/348] improve comments --- volatility3/framework/interfaces/symbols.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index b8712e38d..c0bebe1e2 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -122,7 +122,7 @@ class BaseSymbolTableInterface: @property def symbols(self) -> Iterable[str]: - """Returns an iterator of the Symbol names.""" + """Returns an iterable of the available symbol names.""" raise NotImplementedError( "Abstract property symbols not implemented by subclass." ) @@ -131,7 +131,7 @@ class BaseSymbolTableInterface: @property def types(self) -> Iterable[str]: - """Returns an iterator of the Symbol type names.""" + """Returns an iterable of the available symbol type names.""" raise NotImplementedError( "Abstract property types not implemented by subclass." ) @@ -149,7 +149,7 @@ class BaseSymbolTableInterface: @property def enumerations(self) -> Iterable[Any]: - """Returns an iterator of the Enumeration names.""" + """Returns an iterable of the available enumerations names.""" raise NotImplementedError( "Abstract property enumerations not implemented by subclass." ) @@ -366,6 +366,7 @@ class NativeTableInterface(BaseSymbolTableInterface): @property def symbols(self) -> Iterable[str]: + """Returns an iterable of the available symbol names.""" return [] def get_enumeration(self, name: str) -> objects.Template: From 1e9551b0530be824ab8d9a40db57cbd813d48136 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:50:28 +0100 Subject: [PATCH 341/348] types base class and comments improvements --- volatility3/framework/interfaces/symbols.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index c0bebe1e2..752d288f7 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -375,7 +375,13 @@ class NativeTableInterface(BaseSymbolTableInterface): ) @property - def enumerations(self) -> Iterable[str]: + def enumerations(self) -> Iterable[Any]: + """Returns an iterable of the available enumerations.""" + return [] + + @property + def types(self) -> Iterable[str]: + """Returns an iterable of the available symbol type names.""" return [] From aa99410dd2c50ee556293db959c469739c882684 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:51:31 +0100 Subject: [PATCH 342/348] prefer KeysView iterable to lists --- volatility3/framework/symbols/intermed.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 6802af7d6..0a30148aa 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -411,18 +411,23 @@ class Version1Format(ISFormatTable): @property def symbols(self) -> Iterable[str]: - """Returns an iterator of the symbol names.""" - return list(self._json_object.get("symbols", {})) + """Returns an iterable (KeysView) of the available symbol names.""" + return self._json_object.get("symbols", {}).keys() @property - def enumerations(self) -> Iterable[str]: - """Returns an iterator of the available enumerations.""" - return list(self._json_object.get("enums", {})) + def enumerations(self) -> Iterable[Any]: + """Returns an iterable (KeysView) of the available enumerations.""" + return self._json_object.get("enums", {}).keys() @property - def types(self) -> Iterable[str]: - """Returns an iterator of the symbol type names.""" - return list(self._json_object.get("user_types", {})) + list(self.natives.types) + def types(self): + """Returns an iterable (KeysView) of the available symbol type names.""" + # self.natives.types (set) is generally very small compared to user_types, + # so the dict conversion overhead can be neglected + return { + **self._json_object.get("user_types", {}), + **dict.fromkeys(self.natives.types), + }.keys() def get_type_class(self, name: str) -> Type[interfaces.objects.ObjectInterface]: return self._overrides.get(name, objects.AggregateType) From 3a2933155b6f92a8585666f611cd2069424ce5a9 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:51:52 +0100 Subject: [PATCH 343/348] improve comments --- volatility3/framework/symbols/native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/native.py b/volatility3/framework/symbols/native.py index 7c3e1b312..61417532e 100644 --- a/volatility3/framework/symbols/native.py +++ b/volatility3/framework/symbols/native.py @@ -30,7 +30,7 @@ class NativeTable(interfaces.symbols.NativeTableInterface): @property def types(self) -> Iterable[str]: - """Returns an iterator of the symbol type names.""" + """Returns an iterable (set) of the available symbol type names.""" return self._types def get_type(self, type_name: str) -> interfaces.objects.Template: From ba09db6952d37590f62a647265c2bb5bb903ec3c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:54:04 +0100 Subject: [PATCH 344/348] improve comments --- volatility3/framework/interfaces/symbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index 752d288f7..2d142de9a 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -149,7 +149,7 @@ class BaseSymbolTableInterface: @property def enumerations(self) -> Iterable[Any]: - """Returns an iterable of the available enumerations names.""" + """Returns an iterable of the available enumerations.""" raise NotImplementedError( "Abstract property enumerations not implemented by subclass." ) From 4b3d93b0f0637c7d41acc545f397e3522a913978 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:00:03 +0100 Subject: [PATCH 345/348] 2.17.0 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 3d68ab810..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 17 # Number of changes that only add to the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From c5628c5d79496ae051942598bab08c19d3632a18 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:48:44 +0100 Subject: [PATCH 346/348] revert the mistakenly removed types type hinting --- volatility3/framework/symbols/intermed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 0a30148aa..9ece69d8b 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -420,7 +420,7 @@ class Version1Format(ISFormatTable): return self._json_object.get("enums", {}).keys() @property - def types(self): + def types(self) -> Iterable[str]: """Returns an iterable (KeysView) of the available symbol type names.""" # self.natives.types (set) is generally very small compared to user_types, # so the dict conversion overhead can be neglected From 0ee016e65554539318705a5b7c292864fcc2f436 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:50:28 +0100 Subject: [PATCH 347/348] 2.17.0 -> 2.17.1 bump --- volatility3/framework/constants/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 832b2a5ba..041439909 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 18 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_MINOR = 17 # Number of changes that only add to the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From c2ef3c2fe575f2c3ea49541b7e1207e7d93884f1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:59:51 +0100 Subject: [PATCH 348/348] add fixme about merge operator --- volatility3/framework/symbols/intermed.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 9ece69d8b..cb0b67969 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -422,8 +422,12 @@ class Version1Format(ISFormatTable): @property def types(self) -> Iterable[str]: """Returns an iterable (KeysView) of the available symbol type names.""" - # self.natives.types (set) is generally very small compared to user_types, - # so the dict conversion overhead can be neglected + # We use ** instead of + # `set(self._json_object.get("user_types", {}).keys()).union(self.natives.types)` + # because converting user_types dict to a set is costly. + # It is more efficient to convert the (very small) self.natives.types set to a dict. + # FIXME: On Python3.8 support drop, merge the two dicts using the merge operator: + # (self._json_object.get("user_types", {}) | dict.fromkeys(self.natives.types)).keys() return { **self._json_object.get("user_types", {}), **dict.fromkeys(self.natives.types),