From 0b33a79dc6465884105a60cb3d2054c61885a2eb Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Wed, 2 Nov 2022 21:10:51 +0000 Subject: [PATCH 001/104] Report IRP entries that point inside a hidden module. This is a common rootkit technique. --- volatility3/framework/plugins/windows/driverirp.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index 7f9bc6b08..4013bdb8f 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -48,7 +48,10 @@ class DriverIrp(interfaces.plugins.PluginInterface): for i, address in enumerate(driver.MajorFunction): module_symbols = collection.get_module_symbols_by_absolute_location(address) + module_found = False + for module_name, symbol_generator in module_symbols: + module_found = True symbols_found = False for symbol in symbol_generator: @@ -60,6 +63,11 @@ class DriverIrp(interfaces.plugins.PluginInterface): yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], format_hints.Hex(address), module_name, renderers.NotAvailableValue())) + if not module_found: + yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], + format_hints.Hex(address), renderers.NotAvailableValue(), renderers.NotAvailableValue())) + + def run(self): return renderers.TreeGrid([ From 7469872c8bfcd8d7a84637ede54180008e624b0a Mon Sep 17 00:00:00 2001 From: RuBublik Date: Wed, 17 May 2023 21:41:37 +0300 Subject: [PATCH 002/104] added 'PoolConstraint' of Thread objects to 'PoolScanner.default_constraints' as part of adding support for thread pool tag scanning --- .../framework/plugins/windows/poolscanner.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index e131c5f78..028241bb8 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -224,6 +224,20 @@ class PoolScanner(plugins.PluginInterface): size=(600, None), page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), + # threads on windows before windows8 + PoolConstraint(b'Thr\xe5', # -> “protected” allocation, MSB is set. + type_name = symbol_table + constants.BANG + "_ETHREAD", + object_type="Thread", + size = (600, None), # -> 0x0258 - size of strcut in win5.1 + page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + ), + # threads on windows starting with windows8 + PoolConstraint(b'Thre', + type_name = symbol_table + constants.BANG + "_ETHREAD", + object_type="Thread", + size = (600, None), # -> 0x0258 - size of strcut in win5.1 + page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + ), # files on windows before windows 8 PoolConstraint( b"Fil\xe5", From 550d1096ebe2c2d1f22a6c6446495051d54e84ec Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sat, 20 May 2023 14:20:15 +0300 Subject: [PATCH 003/104] temporary fix to ETHREAD class, add 'is_valid' method --- .../framework/symbols/windows/extensions/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ba00a4053..cfaa3aced 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -492,9 +492,13 @@ class KMUTANT(objects.StructType, pool.ExecutiveObject): return header.NameInfo.Name.String # type: ignore -class ETHREAD(objects.StructType): +class ETHREAD(objects.StructType, pool.ExecutiveObject): """A class for executive thread objects.""" + def is_valid(self) -> bool: + """Determine if the object is valid.""" + return True # temporary, need to implement validation later. + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 0fd475e10bb9644c28abef36c8f4c7dabec318d6 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sat, 20 May 2023 15:49:09 +0300 Subject: [PATCH 004/104] added permanent implementation for 'ETHREAD.is_valid' --- .../symbols/windows/extensions/__init__.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index cfaa3aced..e17128897 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -497,7 +497,30 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): def is_valid(self) -> bool: """Determine if the object is valid.""" - return True # temporary, need to implement validation later. + + try: + + # validation by thread creation time: + ctime = self.get_create_time() + if not isinstance(ctime, datetime.datetime): + return False + + # validation by parent process: + own_proc = self.owning_process() + # return own_proc.is_valid() + if own_proc.UniqueProcessId % 4 != 0: # NT pids are divisible by 4 + return False + + # passed all valitations + return True + except: + return False + + def get_create_time(self): + return conversion.wintime_to_datetime(self.CreateTime.QuadPart) + + def get_exit_time(self): + return conversion.wintime_to_datetime(self.ExitTime.QuadPart) def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 601870829e33c32688fbecccede7423f1e3b7839 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:12:46 +0300 Subject: [PATCH 005/104] twicked thread constraint for edge cases --- volatility3/framework/plugins/windows/poolscanner.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 028241bb8..ce1015789 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -222,17 +222,21 @@ class PoolScanner(plugins.PluginInterface): type_name=symbol_table + constants.BANG + "_EPROCESS", object_type="Process", size=(600, None), + skip_type_test = True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows before windows8 - PoolConstraint(b'Thr\xe5', # -> “protected” allocation, MSB is set. + PoolConstraint( + b'Thr\xe5', # -> “protected” allocation, MSB is set. type_name = symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", size = (600, None), # -> 0x0258 - size of strcut in win5.1 + skip_type_test = True, page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE ), # threads on windows starting with windows8 - PoolConstraint(b'Thre', + PoolConstraint( + b'Thre', type_name = symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", size = (600, None), # -> 0x0258 - size of strcut in win5.1 From 046c8e4d1e7f3bf9d6857de5601ebd4ab089af6d Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:18:02 +0300 Subject: [PATCH 006/104] fix of is_valid - removed reliace on owning _eprocess and added exclusion for system process (does not have creation time) --- .../symbols/windows/extensions/__init__.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index e17128897..fe4884dbc 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -499,18 +499,21 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): """Determine if the object is valid.""" try: - - # validation by thread creation time: - ctime = self.get_create_time() - if not isinstance(ctime, datetime.datetime): - return False - - # validation by parent process: - own_proc = self.owning_process() - # return own_proc.is_valid() - if own_proc.UniqueProcessId % 4 != 0: # NT pids are divisible by 4 + + # validation by TID: + if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 return False + # validation by PID of parent process: + if self.Cid.UniqueProcess % 4 != 0: + return False + + # validation by thread creation time: + if self.Cid.UniqueProcess != 4: # The System process (PID 4) has no create time + ctime = self.get_create_time() + if not isinstance(ctime, datetime.datetime): + return False + # passed all valitations return True except: From fab818b48d950a1210c77e97a4da09cf24f2bd03 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:24:01 +0300 Subject: [PATCH 007/104] added thrdscan plugin to utilize added support for ethread pool tag scanning --- .../framework/plugins/windows/thrdscan.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 volatility3/framework/plugins/windows/thrdscan.py diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py new file mode 100644 index 000000000..b0b80fe46 --- /dev/null +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -0,0 +1,107 @@ +## +## plugin for testing addition of threads scan support to poolscanner.py +## +import logging +import datetime +from typing import Iterable + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import poolscanner + +vollog = logging.getLogger(__name__) + + +class ThrdScan(interfaces.plugins.PluginInterface): + """Scans for windows threads.""" + + # cuz installed Framework interface version 2 + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), + ] + + @classmethod + def scan_threads( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Scans for threads using the poolscanner module and constraints. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + + Returns: + A list of _ETHREAD objects found by scanning memory for the "Thre" / "Thr\\xE5" pool signatures + """ + + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"Thr\xe5", b"Thre"] + ) + + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): + _constraint, mem_object, _header = result + yield mem_object + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + for ethread in self.scan_threads( + self.context, kernel.layer_name, kernel.symbol_table_name + ): + try: + thread_offset = ethread.vol.offset + owner_proc_pid = ethread.Cid.UniqueProcess + thread_tid = ethread.Cid.UniqueThread + thread_start_addr = ethread.StartAddress + thread_create_time = ethread.get_create_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_exit_time = ethread.get_exit_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + except (ValueError, exceptions.InvalidAddressException): + vollog.debug( + "Thread :{}, invalid address {} in layer {}".format( + thread_tid, thread_start_addr, kernel.layer_name + ) + ) + continue + + yield ( + 0, + ( + hex(format_hints.Hex(thread_offset)), + owner_proc_pid, + thread_tid, + hex(thread_start_addr), + str(thread_create_time) if isinstance(thread_create_time, datetime.datetime) else "", + str(thread_exit_time) if isinstance(thread_exit_time, datetime.datetime) else "" + ) + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", str), + ("PID", int), + ("TID", int), + ("Start Address", str), + ("Create Time", str), + ("Exit Time", str), + ], + self._generator(), + ) \ No newline at end of file From 5749b3ef5283fc36d064a766e5228dc2eef5b402 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 19:22:22 +0300 Subject: [PATCH 008/104] added test for thrdscan plugin --- test/test_volatility.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index aaad615bc..5e41899da 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -189,6 +189,16 @@ def test_windows_svcscan(image, volatility, python): assert rc == 0 +def test_windows_thrdscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.thrdscan.ThrdScan", image, volatility, python) + # find pid 4 (of system process) which starts with lowest tids + assert out.find(b"\t4\t8") != -1 + assert out.find(b"\t4\t12") != -1 + assert out.find(b"\t4\t16") != -1 + #assert out.find(b"this raieses AssertionError") != -1 + assert rc == 0 + + def test_windows_privileges(image, volatility, python): rc, out, err = runvol_plugin( "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"] From 950a76d1e0dfb460215c3b6e5dc1108a0421863f Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 11:17:23 +0300 Subject: [PATCH 009/104] fixed typo --- .../framework/symbols/windows/extensions/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index fe4884dbc..4ad74f61a 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -514,10 +514,11 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False - # passed all valitations - return True - except: + except exceptions.InvalidAddressException: return False + + # passed all validations + return True def get_create_time(self): return conversion.wintime_to_datetime(self.CreateTime.QuadPart) From bef5149ba64e640c39e83774e0fa969893d5ee18 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 11:39:38 +0300 Subject: [PATCH 010/104] changed TreeGrid yielded types to specific simpletypes instead of str --- .../framework/plugins/windows/thrdscan.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index b0b80fe46..8b445991f 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -84,24 +84,24 @@ class ThrdScan(interfaces.plugins.PluginInterface): yield ( 0, ( - hex(format_hints.Hex(thread_offset)), + format_hints.Hex(thread_offset), owner_proc_pid, thread_tid, - hex(thread_start_addr), - str(thread_create_time) if isinstance(thread_create_time, datetime.datetime) else "", - str(thread_exit_time) if isinstance(thread_exit_time, datetime.datetime) else "" + format_hints.Hex(thread_start_addr), + thread_create_time, + thread_exit_time, ) ) def run(self): return renderers.TreeGrid( [ - ("Offset", str), + ("Offset", format_hints.Hex), ("PID", int), ("TID", int), - ("Start Address", str), - ("Create Time", str), - ("Exit Time", str), + ("Start Address", format_hints.Hex), + ("Create Time", datetime.datetime), + ("Exit Time", datetime.datetime), ], self._generator(), ) \ No newline at end of file From c6c501ecb57e1757a510d019a3752af78dda96cc Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:31:25 +0300 Subject: [PATCH 011/104] implemented generate_timeline method in ThrdScan --- .../framework/plugins/windows/thrdscan.py | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 8b445991f..ca3f4fc69 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -9,15 +9,16 @@ from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import poolscanner +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class ThrdScan(interfaces.plugins.PluginInterface): +class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -93,15 +94,40 @@ class ThrdScan(interfaces.plugins.PluginInterface): ) ) + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + row_dict = {} + ( + row_dict["Offset"], + row_dict["PID"], + row_dict["TID"], + row_dict["StartAddress"], + row_dict["CreateTime"], + row_dict["ExitTime"], + ) = row_data + + # Skip threads with no creation time + # - mainly system process threads + if not isinstance(row_dict["CreateTime"], datetime.datetime): + continue + description = (f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})") + + # yield created time, and if there is exit time, yield it too. + yield (description, timeliner.TimeLinerType.CREATED, row_dict["CreateTime"]) + if isinstance(row_dict["ExitTime"], datetime.datetime): + yield (description, timeliner.TimeLinerType.MODIFIED, row_dict["ExitTime"]) + + def run(self): return renderers.TreeGrid( [ ("Offset", format_hints.Hex), ("PID", int), ("TID", int), - ("Start Address", format_hints.Hex), - ("Create Time", datetime.datetime), - ("Exit Time", datetime.datetime), + ("StartAddress", format_hints.Hex), + ("CreateTime", datetime.datetime), + ("ExitTime", datetime.datetime), ], self._generator(), ) \ No newline at end of file From 9bdd249aadb5b60ca5e45c91c396f1c591f9b45e Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:39:12 +0300 Subject: [PATCH 012/104] added _version to ThrdScan --- volatility3/framework/plugins/windows/thrdscan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index ca3f4fc69..cbbe64988 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -19,6 +19,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # cuz installed Framework interface version 2 _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls): From 5072728e897c1489e11f3422ea6974cf542d7e64 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:50:22 +0300 Subject: [PATCH 013/104] formated with black --- .../framework/plugins/windows/thrdscan.py | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index cbbe64988..813883c2d 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -18,7 +18,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """Scans for windows threads.""" # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod @@ -33,7 +33,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) ), ] - + @classmethod def scan_threads( cls, @@ -53,7 +53,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """ constraints = poolscanner.PoolScanner.builtin_constraints( - symbol_table, [b"Thr\xe5", b"Thre"] + symbol_table, [b"Thr\xe5", b"Thre"] ) for result in poolscanner.PoolScanner.generate_pool_scan( @@ -69,12 +69,16 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) self.context, kernel.layer_name, kernel.symbol_table_name ): try: - thread_offset = ethread.vol.offset - owner_proc_pid = ethread.Cid.UniqueProcess - thread_tid = ethread.Cid.UniqueThread - thread_start_addr = ethread.StartAddress - thread_create_time = ethread.get_create_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object - thread_exit_time = ethread.get_exit_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_offset = ethread.vol.offset + owner_proc_pid = ethread.Cid.UniqueProcess + thread_tid = ethread.Cid.UniqueThread + thread_start_addr = ethread.StartAddress + thread_create_time = ( + ethread.get_create_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_exit_time = ( + ethread.get_exit_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object except (ValueError, exceptions.InvalidAddressException): vollog.debug( "Thread :{}, invalid address {} in layer {}".format( @@ -86,13 +90,13 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) yield ( 0, ( - format_hints.Hex(thread_offset), - owner_proc_pid, - thread_tid, - format_hints.Hex(thread_start_addr), - thread_create_time, - thread_exit_time, - ) + format_hints.Hex(thread_offset), + owner_proc_pid, + thread_tid, + format_hints.Hex(thread_start_addr), + thread_create_time, + thread_exit_time, + ), ) def generate_timeline(self): @@ -112,13 +116,16 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # - mainly system process threads if not isinstance(row_dict["CreateTime"], datetime.datetime): continue - description = (f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})") - + description = f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})" + # yield created time, and if there is exit time, yield it too. yield (description, timeliner.TimeLinerType.CREATED, row_dict["CreateTime"]) if isinstance(row_dict["ExitTime"], datetime.datetime): - yield (description, timeliner.TimeLinerType.MODIFIED, row_dict["ExitTime"]) - + yield ( + description, + timeliner.TimeLinerType.MODIFIED, + row_dict["ExitTime"], + ) def run(self): return renderers.TreeGrid( @@ -131,4 +138,4 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ("ExitTime", datetime.datetime), ], self._generator(), - ) \ No newline at end of file + ) From 5d32ca542c8918224f070163f02ff454f0bffffa Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 28 May 2023 20:52:26 +0300 Subject: [PATCH 014/104] fix black formatting --- .../framework/plugins/windows/poolscanner.py | 20 +++++++++---------- .../symbols/windows/extensions/__init__.py | 17 ++++++++-------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index ce1015789..5539e1e84 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -222,25 +222,25 @@ class PoolScanner(plugins.PluginInterface): type_name=symbol_table + constants.BANG + "_EPROCESS", object_type="Process", size=(600, None), - skip_type_test = True, + skip_type_test=True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows before windows8 PoolConstraint( - b'Thr\xe5', # -> “protected” allocation, MSB is set. - type_name = symbol_table + constants.BANG + "_ETHREAD", + b"Thr\xe5", # -> “protected” allocation, MSB is set. + type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size = (600, None), # -> 0x0258 - size of strcut in win5.1 - skip_type_test = True, - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + size=(600, None), # -> 0x0258 - size of strcut in win5.1 + skip_type_test=True, + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows starting with windows8 PoolConstraint( - b'Thre', - type_name = symbol_table + constants.BANG + "_ETHREAD", + b"Thre", + type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size = (600, None), # -> 0x0258 - size of strcut in win5.1 - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + size=(600, None), # -> 0x0258 - size of strcut in win5.1 + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # files on windows before windows 8 PoolConstraint( diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 4ad74f61a..8790f41a7 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -499,17 +499,18 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): """Determine if the object is valid.""" try: - # validation by TID: - if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 + if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 return False - + # validation by PID of parent process: if self.Cid.UniqueProcess % 4 != 0: return False - + # validation by thread creation time: - if self.Cid.UniqueProcess != 4: # The System process (PID 4) has no create time + if ( + self.Cid.UniqueProcess != 4 + ): # The System process (PID 4) has no create time ctime = self.get_create_time() if not isinstance(ctime, datetime.datetime): return False @@ -518,14 +519,14 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return False # passed all validations - return True - + return True + def get_create_time(self): return conversion.wintime_to_datetime(self.CreateTime.QuadPart) def get_exit_time(self): return conversion.wintime_to_datetime(self.ExitTime.QuadPart) - + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 524ff59107ee857c6d4e86697fd6db7f92c05156 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Fri, 7 Jul 2023 20:22:58 +0300 Subject: [PATCH 015/104] bumped MINOR_VERSION to 2.5.2 after changes, and updated dependent thrdscan plugin's required version to this --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3a6b24ea8..09dded076 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,7 +44,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 4 # Number of changes that only add to the interface +VERSION_MINOR = 5 # Number of changes that only add to the interface VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 813883c2d..4afc29cdb 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + # version 2.5.2 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 5, 2) _version = (1, 0, 0) @classmethod From abf1c2e03d67f02ea5c31cc9f4b16029cbbe946d Mon Sep 17 00:00:00 2001 From: RuBublik Date: Fri, 7 Jul 2023 20:25:02 +0300 Subject: [PATCH 016/104] fixed typos --- volatility3/framework/plugins/windows/poolscanner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 5539e1e84..1f70cfb8c 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -230,7 +230,7 @@ class PoolScanner(plugins.PluginInterface): b"Thr\xe5", # -> “protected” allocation, MSB is set. type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size=(600, None), # -> 0x0258 - size of strcut in win5.1 + size=(600, None), # -> 0x0258 - size of struct in win5.1 skip_type_test=True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), @@ -239,7 +239,7 @@ class PoolScanner(plugins.PluginInterface): b"Thre", type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size=(600, None), # -> 0x0258 - size of strcut in win5.1 + size=(600, None), # -> 0x0258 - size of struct in win5.1 page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # files on windows before windows 8 From 338238c6396aa4c53a579a61a64eb4139de6cc76 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 16 Jul 2023 00:10:31 +0300 Subject: [PATCH 017/104] fixed build number - resets when MINOR version goes up --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09dded076..de1674885 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 4afc29cdb..6e19f9bd5 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # version 2.5.2 adds support for scanning for 'Ethread' structures by pool tags - _required_framework_version = (2, 5, 2) + # version 2.5.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 5, 0) _version = (1, 0, 0) @classmethod From 62466c7953cb0264bc3bb491a694a55732115d7a Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 3 Oct 2023 21:35:37 +0300 Subject: [PATCH 018/104] fixed merge conflicts with 'volatilityfoundation:develop' branch - bumped VERSION_PATCH --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index de1674885..c3ebaca27 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 57d995a81d3b6a9d1843439477c0be6933215df0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 8 Oct 2023 03:15:23 +0200 Subject: [PATCH 019/104] manually instantiate queue_entry for tasks symbol --- volatility3/framework/plugins/mac/pslist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 88045a277..c0e149fc2 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object_from_symbol(symbol_name="tasks") + queue_entry = kernel.object("queue_entry", kernel.get_symbol("tasks").address) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 6e5d41c38b3c494b9a43d5a7fca515aa84e1b6d4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 8 Oct 2023 03:37:17 +0200 Subject: [PATCH 020/104] manually instantiate queue_entry for tasks symbol --- volatility3/framework/plugins/mac/pslist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index c0e149fc2..e8c490dff 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object("queue_entry", kernel.get_symbol("tasks").address) + queue_entry = kernel.object(object_type="queue_entry", offset=kernel.get_symbol("tasks").address) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 3a656266711b21951ea8c48fe9d8ac4a4cf64775 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 23 Oct 2023 17:47:57 +0200 Subject: [PATCH 021/104] black formatting --- volatility3/framework/plugins/mac/pslist.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index e8c490dff..9835644b8 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,9 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object(object_type="queue_entry", offset=kernel.get_symbol("tasks").address) + queue_entry = kernel.object( + object_type="queue_entry", offset=kernel.get_symbol("tasks").address + ) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 25637a41e05e0bc5fccded01cf1913d45668ac25 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 8 Jan 2024 23:35:21 +0200 Subject: [PATCH 022/104] added account for XP timestamps - bit shifted --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 8790f41a7..9a286bc26 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -522,6 +522,9 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return True def get_create_time(self): + # For Windows XPs + if self.has_member("ThreadsProcess"): + return conversion.wintime_to_datetime(self.CreateTime.QuadPart >> 3) return conversion.wintime_to_datetime(self.CreateTime.QuadPart) def get_exit_time(self): From 550112b848913954e8b03d8c40bff9e2fd7902d7 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 8 Jan 2024 23:37:38 +0200 Subject: [PATCH 023/104] added another sanity check to ETHREAD.is_valid --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 9a286bc26..f4e2cc485 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -515,6 +515,9 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False + if not (1998 < ctime.year < 2030): + return False + except exceptions.InvalidAddressException: return False From 7c370121181af5da71b75d2b844def89341ddecd Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 9 Jan 2024 00:16:18 +0200 Subject: [PATCH 024/104] bumped version constants to mark change of interface (add of support for ETHREAD) --- volatility3/framework/constants/__init__.py | 4 ++-- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index c3ebaca27..9aaafb933 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 6 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 6e19f9bd5..80906b3b9 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # version 2.5.0 adds support for scanning for 'Ethread' structures by pool tags - _required_framework_version = (2, 5, 0) + # version 2.6.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 6, 0) _version = (1, 0, 0) @classmethod From b0d84e55ab1846004318e87bd7b5f427e7d70551 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 9 Jan 2024 21:32:29 +0200 Subject: [PATCH 025/104] fix indentation (typo) --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index f4e2cc485..a0af29d18 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -516,7 +516,7 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return False if not (1998 < ctime.year < 2030): - return False + return False except exceptions.InvalidAddressException: return False From cc35fecf91b1d2704c87031427c623e066ac968a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 19:40:37 +1100 Subject: [PATCH 026/104] Linux: Add library_list plugin and other ELF related code enhacements. - Add library_list plugin - Add ELF dynamic table enum types in elf.json - Update missing program header enum types in elf.json - Add PAGE constants - Add ELF ident and class enums - Replace ELF hardcoded type numbers for enum description matching - Fix unmanaged ValueError exception issue in Elf64Layer::_load_segments() --- .../framework/constants/linux/__init__.py | 26 +++ volatility3/framework/layers/elf.py | 11 +- volatility3/framework/plugins/linux/elfs.py | 34 ++-- .../framework/plugins/linux/library_list.py | 169 ++++++++++++++++++ volatility3/framework/symbols/linux/elf.json | 138 +++++++++++++- .../symbols/linux/extensions/__init__.py | 3 +- .../framework/symbols/linux/extensions/elf.py | 122 +++++++++++-- 7 files changed, 464 insertions(+), 39 deletions(-) create mode 100644 volatility3/framework/plugins/linux/library_list.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 6e8883f19..5e82e580e 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -5,11 +5,15 @@ Linux-specific values that aren't found in debug symbols """ +from enum import IntEnum KERNEL_NAME = "__kernel__" # arch/x86/include/asm/page_types.h PAGE_SHIFT = 12 +PAGE_SIZE = 1 << PAGE_SHIFT +PAGE_MASK = ~(PAGE_SIZE - 1) + """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" # include/linux/sched.h @@ -281,3 +285,25 @@ CAPABILITIES = ( ) ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 + + +class ELF_IDENT(IntEnum): + """ELF header e_ident indexes""" + + EI_MAG0 = 0 + EI_MAG1 = 1 + EI_MAG2 = 2 + EI_MAG3 = 3 + EI_CLASS = 4 + EI_DATA = 5 + EI_VERSION = 6 + EI_OSABI = 7 + EI_PAD = 8 + + +class ELF_CLASS(IntEnum): + """ELF header class types""" + + ELFCLASSNONE = 0 + ELFCLASS32 = 1 + ELFCLASS64 = 2 diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index b2fd6d4d1..6bd5c2d63 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -6,9 +6,11 @@ import struct from typing import Optional from volatility3.framework import exceptions, interfaces, constants +from volatility3.framework.constants.linux import ELF_CLASS from volatility3.framework.layers import segmented from volatility3.framework.symbols import intermed + vollog = logging.getLogger(__name__) @@ -21,7 +23,7 @@ class Elf64Layer(segmented.SegmentedLayer): _header_struct = struct.Struct(" 0 ): diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index e688ecb42..7171a6616 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -14,8 +14,14 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux.extensions import elf +from volatility3.framework.constants.linux import ( + PAGE_SIZE, + PAGE_MASK, + ELF_MAX_EXTRACTION_SIZE, +) from volatility3.plugins.linux import pslist + vollog = logging.getLogger(__name__) @@ -23,7 +29,7 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -87,7 +93,10 @@ class Elfs(plugins.PluginInterface): sections = {} # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? for phdr in elf_object.get_program_headers(): - if phdr.p_type != 1: # PT_LOAD = 1 + try: + if phdr.p_type.description != "PT_LOAD": + continue + except ValueError: continue start = phdr.p_vaddr @@ -95,18 +104,18 @@ class Elfs(plugins.PluginInterface): end = start + size # Use complete memory pages for dumping - # If start isn't a multiple of 4096, stick to the highest multiple < start - # If end isn't a multiple of 4096, stick to the lowest multiple > end - if start % 4096: - start = start & ~0xFFF + # If start isn't a multiple of a page, stick to the highest multiple < start + # If end isn't a multiple of a page, stick to the lowest multiple > end + if start % PAGE_SIZE: + start = start & PAGE_MASK - if end % 4096: - end = (end & ~0xFFF) + 4096 + if end % PAGE_SIZE: + end = (end & PAGE_MASK) + PAGE_SIZE real_size = end - start # Check if ELF has a legitimate size - if real_size < 0 or real_size > constants.linux.ELF_MAX_EXTRACTION_SIZE: + if real_size < 0 or real_size > ELF_MAX_EXTRACTION_SIZE: raise ValueError(f"The claimed size of the ELF is invalid: {real_size}") sections[start] = real_size @@ -140,12 +149,7 @@ class Elfs(plugins.PluginInterface): for vma in task.mm.get_vma_iter(): hdr = proc_layer.read(vma.vm_start, 4, pad=True) - if not ( - hdr[0] == 0x7F - and hdr[1] == 0x45 - and hdr[2] == 0x4C - and hdr[3] == 0x46 - ): + if hdr != b"\x7fELF": continue path = vma.get_name(self.context, task) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py new file mode 100644 index 000000000..ed5545347 --- /dev/null +++ b/volatility3/framework/plugins/linux/library_list.py @@ -0,0 +1,169 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import Iterable, Tuple + +from volatility3.framework import interfaces, renderers, constants, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.objects import utility +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.linux.extensions import elf +from volatility3.plugins.linux import pslist + + +vollog = logging.getLogger(__name__) + + +class LibraryList(interfaces.plugins.PluginInterface): + """Enumerate libraries loaded into processes""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + ), + requirements.ListRequirement( + name="pids", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + ] + + def get_libdl_libraries( + self, proc_layer_name: str, vma_start: int + ) -> interfaces.objects.ObjectInterface: + """Get the ELF link map objects for the given VMA address + + Args: + proc_layer_name (str): Name of the process layer + vma_start (int): VMA start address + + Yields: + ELF link map objects for the given VMA address + """ + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "linux", + "elf", + class_types=elf.class_types, + ) + elf_object = self.context.object( + elf_table_name + constants.BANG + "Elf", + offset=vma_start, + layer_name=proc_layer_name, + ) + + if not elf_object or not elf_object.is_valid(): + return None + + kernel = self.context.modules[self.config["kernel"]] + + try: + for link_map in elf_object.get_link_maps(kernel.symbol_table_name): + if link_map.l_addr and link_map.l_name: + yield link_map + except exceptions.InvalidAddressException: + # Protection against memory smear in this VMA + pass + + def get_libdl_maps( + self, task: interfaces.objects.ObjectInterface, proc_layer_name: str + ) -> interfaces.objects.ObjectInterface: + """Get the ELF link maps objects for a task + + Args: + task (task_struct): A reference task + proc_layer_name (str): Name of the process layer + + Yields: + ELF link map objects + """ + + link_map_seen = set() + for vma in task.mm.get_vma_iter(): + for link_map in self.get_libdl_libraries(proc_layer_name, vma.vm_start): + if link_map.l_addr in link_map_seen: + continue + + yield link_map + link_map_seen.add(link_map.l_addr) + + def get_task_libraries( + self, task: interfaces.objects.ObjectInterface + ) -> Tuple[int, str]: + """Get the task libraries from the ELF headers found within the memory maps + + Args: + task (task_struct): The reference task + + Yields: + Tuples with a ELF link map address and name + """ + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + return + + for elf_link_map in self.get_libdl_maps(task, proc_layer_name): + name = elf_link_map.get_name() + if not name: + continue + yield elf_link_map.l_addr, name + + def get_tasks_libraries( + self, + tasks: Iterable[interfaces.objects.ObjectInterface], + ) -> Iterable[Tuple[str, int, int, str]]: + """Get the task libraries from the ELF headers found within the memory maps for + all the tasks. + + Args: + tasks: An iterable of tasks + + Yields: + Tuples with a task name, task tgid, an ELF link map address and name + """ + for task in tasks: + task_name = utility.array_to_string(task.comm) + for linkmap_addr, linkmap_name in self.get_task_libraries(task): + yield task_name, task.tgid, linkmap_addr, linkmap_name + + def _format_fields(self, fields): + task_name, task_pid, addr, name = fields + return task_name, task_pid, format_hints.Hex(addr), name + + def _generator( + self, tasks: Iterable[interfaces.objects.ObjectInterface] + ) -> Iterable[Tuple[int, Tuple]]: + for fields in self.get_tasks_libraries(tasks): + yield 0, self._format_fields(fields) + + def run(self): + pids = self.config.get("pids") + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=pid_filter + ) + + headers = [ + ("Name", str), + ("Pid", int), + ("LoadAddress", format_hints.Hex), + ("Path", str), + ] + + return renderers.TreeGrid(headers, self._generator(tasks)) diff --git a/volatility3/framework/symbols/linux/elf.json b/volatility3/framework/symbols/linux/elf.json index 76cd8a2ec..e0a95bbba 100644 --- a/volatility3/framework/symbols/linux/elf.json +++ b/volatility3/framework/symbols/linux/elf.json @@ -270,8 +270,8 @@ "d_tag": { "offset": 0, "type": { - "kind": "base", - "name": "long long" + "kind": "enum", + "name": "DtypeEnum64" } }, "d_ptr": { @@ -699,8 +699,8 @@ "d_tag": { "offset": 0, "type": { - "kind": "base", - "name": "long" + "kind": "enum", + "name": "DtypeEnum32" } }, "d_ptr": { @@ -905,11 +905,139 @@ "PT_PHDR": 6, "PT_TLS": 7, "PT_LOOS": 1610612736, + "PT_GNU_EH_FRAME": 1685382480, + "PT_GNU_STACK": 1685382481, + "PT_GNU_RELRO": 1685382482, + "PT_GNU_PROPERTY": 1685382483, "PT_HIOS": 1879048191, "PT_LOWPROC": 1879048192, "PT_HIPROC": 2147483647 }, "size": 4 + }, + "DtypeEnum32": { + "base": "long", + "constants": { + "DT_NULL": 0, + "DT_NEEDED": 1, + "DT_PLTRELSZ": 2, + "DT_PLTGOT": 3, + "DT_HASH": 4, + "DT_STRTAB": 5, + "DT_SYMTAB": 6, + "DT_RELA": 7, + "DT_RELASZ": 8, + "DT_RELAENT": 9, + "DT_STRSZ": 10, + "DT_SYMENT": 11, + "DT_INIT": 12, + "DT_FINI": 13, + "DT_SONAME": 14, + "DT_RPATH": 15, + "DT_SYMBOLIC": 16, + "DT_REL": 17, + "DT_RELSZ": 18, + "DT_RELENT": 19, + "DT_PLTREL": 20, + "DT_DEBUG": 21, + "DT_TEXTREL": 22, + "DT_JMPREL": 23, + "DT_BIND_NOW": 24, + "DT_INIT_ARRAY": 25, + "DT_FINI_ARRAY": 26, + "DT_INIT_ARRAYSZ": 27, + "DT_FINI_ARRAYSZ": 28, + "DT_RUNPATH": 29, + "DT_FLAGS": 30, + "DT_ENCODING": 32, + "DT_PREINIT_ARRAYSZ": 33, + "DT_SYMTAB_SHNDX": 34, + "DT_RELRSZ": 35, + "DT_RELR": 36, + "DT_RELRENT": 37, + "DT_NUM": 38, + "OLD_DT_LOOS": 1610612736, + "DT_LOOS": 1610612749, + "DT_HIOS": 1879044096, + "DT_VALRNGLO": 1879047424, + "DT_VALRNGHI": 1879047679, + "DT_ADDRRNGLO": 1879047680, + "DT_GNU_HASH": 1879047925, + "DT_ADDRRNGHI": 1879047935, + "DT_VERSYM": 1879048176, + "DT_RELACOUNT": 1879048185, + "DT_RELCOUNT": 1879048186, + "DT_FLAGS_1": 1879048187, + "DT_VERDEF": 1879048188, + "DT_VERDEFNUM": 1879048189, + "DT_VERNEED": 1879048190, + "DT_VERNEEDNUM": 1879048191, + "DT_LOPROC": 1879048192, + "DT_HIPROC": 2147483647 + }, + "size": 4 + }, + "DtypeEnum64": { + "base": "long long", + "constants": { + "DT_NULL": 0, + "DT_NEEDED": 1, + "DT_PLTRELSZ": 2, + "DT_PLTGOT": 3, + "DT_HASH": 4, + "DT_STRTAB": 5, + "DT_SYMTAB": 6, + "DT_RELA": 7, + "DT_RELASZ": 8, + "DT_RELAENT": 9, + "DT_STRSZ": 10, + "DT_SYMENT": 11, + "DT_INIT": 12, + "DT_FINI": 13, + "DT_SONAME": 14, + "DT_RPATH": 15, + "DT_SYMBOLIC": 16, + "DT_REL": 17, + "DT_RELSZ": 18, + "DT_RELENT": 19, + "DT_PLTREL": 20, + "DT_DEBUG": 21, + "DT_TEXTREL": 22, + "DT_JMPREL": 23, + "DT_BIND_NOW": 24, + "DT_INIT_ARRAY": 25, + "DT_FINI_ARRAY": 26, + "DT_INIT_ARRAYSZ": 27, + "DT_FINI_ARRAYSZ": 28, + "DT_RUNPATH": 29, + "DT_FLAGS": 30, + "DT_ENCODING": 32, + "DT_PREINIT_ARRAYSZ": 33, + "DT_SYMTAB_SHNDX": 34, + "DT_RELRSZ": 35, + "DT_RELR": 36, + "DT_RELRENT": 37, + "DT_NUM": 38, + "OLD_DT_LOOS": 1610612736, + "DT_LOOS": 1610612749, + "DT_HIOS": 1879044096, + "DT_VALRNGLO": 1879047424, + "DT_VALRNGHI": 1879047679, + "DT_ADDRRNGLO": 1879047680, + "DT_GNU_HASH": 1879047925, + "DT_ADDRRNGHI": 1879047935, + "DT_VERSYM": 1879048176, + "DT_RELACOUNT": 1879048185, + "DT_RELCOUNT": 1879048186, + "DT_FLAGS_1": 1879048187, + "DT_VERDEF": 1879048188, + "DT_VERDEFNUM": 1879048189, + "DT_VERNEED": 1879048190, + "DT_VERNEEDNUM": 1879048191, + "DT_LOPROC": 1879048192, + "DT_HIPROC": 2147483647 + }, + "size": 8 } }, "base_types": { @@ -958,7 +1086,7 @@ }, "metadata": { "producer": { - "version": "0.0.1", + "version": "0.0.2", "name": "ikelos-by-hand", "datetime": "2019-10-21T22:52:00" }, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d73d0cfb9..0a3db9fcd 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,14 +7,13 @@ import logging import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List -from volatility3.framework import constants +from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework.constants.linux import CAPABILITIES -from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index fe85b194f..4b2b29b54 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -6,6 +6,10 @@ from typing import Dict, Tuple import logging from volatility3.framework import constants +from volatility3.framework.constants.linux import ( + ELF_IDENT, + ELF_CLASS, +) from volatility3.framework import objects, interfaces, exceptions vollog = logging.getLogger(__name__) @@ -59,13 +63,15 @@ class elf(objects.StructType): ei_class = self._context.object( symbol_table_name + constants.BANG + "unsigned char", layer_name=layer_name, - offset=object_info.offset + 0x4, + offset=object_info.offset + ELF_IDENT.EI_CLASS, ) - if ei_class == 1: + if ei_class == ELF_CLASS.ELFCLASS32: self._type_prefix = "Elf32_" - elif ei_class == 2: + self._ei_class_size = 32 + elif ei_class == ELF_CLASS.ELFCLASS64: self._type_prefix = "Elf64_" + self._ei_class_size = 64 else: raise ValueError(f"Unsupported ei_class value {ei_class}") @@ -140,36 +146,103 @@ class elf(objects.StructType): ) return section_headers + def get_link_maps(self, kernel_symbol_table_name): + """Get the ELF link map objects for the given VMA address + + Args: + kernel_symbol_table_name (str): Kernel symbol table name + + Yields: + The ELF link map objects + """ + got_entry_size = self._ei_class_size // 8 + + elf_symbol_table = self.get_symbol_table_name() + + link_maps_seen = set() + for phdr in self.get_program_headers(): + try: + if phdr.p_type.description != "PT_DYNAMIC": + continue + except ValueError: + continue + + for dsec in phdr.dynamic_sections(): + try: + if dsec.d_tag.description != "DT_PLTGOT": + continue + except ValueError: + continue + + got_start = dsec.d_ptr + + # link_map is stored at the second GOT entry + link_map_addr = got_start + got_entry_size + + # It needs the kernel symbol table to create a pointer + link_map_ptr = self._context.object( + kernel_symbol_table_name + constants.BANG + "pointer", + offset=link_map_addr, + layer_name=self.vol.layer_name, + ) + if not link_map_ptr: + continue + + linkmap_symname = ( + elf_symbol_table + constants.BANG + self._type_prefix + "LinkMap" + ) + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map_ptr, + layer_name=self.vol.layer_name, + ) + + while link_map and link_map.vol.offset != 0: + if link_map.vol.offset in link_maps_seen: + break + link_maps_seen.add(link_map.vol.offset) + + yield link_map + + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map.l_next, + layer_name=self.vol.layer_name, + ) + def _find_symbols(self): dt_strtab = None dt_symtab = None dt_strent = None for phdr in self.get_program_headers(): + # Find PT_DYNAMIC segment try: - # Find PT_DYNAMIC segment - if str(phdr.p_type.description) != "PT_DYNAMIC": + if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: - # If the p_type value is outside the ones declared in the enumeration, an - # exception is raised - return None + continue # This section contains pointers to the strtab, symtab, and strent sections for dsec in phdr.dynamic_sections(): - if dsec.d_tag == 5: + try: + dtag = dsec.d_tag.description + except ValueError: + continue + + if dtag == "DT_STRTAB": dt_strtab = dsec.d_ptr - elif dsec.d_tag == 6: + elif dtag == "DT_SYMTAB": dt_symtab = dsec.d_ptr - elif dsec.d_tag == 11: + elif dtag == "DT_SYMENT": # Size of the symtab symbol entry dt_strent = dsec.d_ptr break - if dt_strtab is None or dt_symtab is None or dt_strent is None: + if not (dt_strtab and dt_symtab and dt_strent): return None self._cached_symtab = dt_symtab @@ -274,15 +347,18 @@ class elf_phdr(objects.StructType): def get_vaddr(self): offset = self.__getattr__("p_vaddr") - if self._parent_e_type == 3: # ET_DYN - offset = self._parent_offset + offset + try: + if self._parent_e_type.description == "ET_DYN": + offset = self._parent_offset + offset + except ValueError: + pass return offset def dynamic_sections(self): # sanity check try: - if str(self.p_type.description) != "PT_DYNAMIC": + if self.p_type.description != "PT_DYNAMIC": return None except ValueError: # If the value is outside the ones declared in the enumeration, an @@ -314,10 +390,26 @@ class elf_phdr(objects.StructType): break +class elf_linkmap(objects.StructType): + def get_name(self): + try: + buf = self._context.layers.read(self.vol.layer_name, self.l_name, 256) + except exceptions.PagedInvalidAddressException: + # Protection against memory smear + return None + + idx = buf.find(b"\x00") + if idx != -1: + buf = buf[:idx] + return buf.decode() + + class_types = { "Elf": elf, "Elf64_Phdr": elf_phdr, "Elf32_Phdr": elf_phdr, "Elf32_Sym": elf_sym, "Elf64_Sym": elf_sym, + "Elf32_LinkMap": elf_linkmap, + "Elf64_LinkMap": elf_linkmap, } From 6e7b5b59416a0f0830660d6415144ec8ee0039ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 19:41:30 +1100 Subject: [PATCH 027/104] Linux: Add linux_library_list test case --- test/test_volatility.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index aaad615bc..7b151fd6c 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -6,6 +6,7 @@ # import os +import re import subprocess import sys import shutil @@ -331,6 +332,32 @@ def test_linux_tty_check(image, volatility, python): assert rc == 0 +def test_linux_library_list(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.library_list.LibraryList", image, volatility, python + ) + + assert re.search( + rb"NetworkManager\s2363\s0x7f52cdda0000\s/lib/x86_64-linux-gnu/libnss_files.so.2", + out, + ) + assert re.search( + rb"gnome-settings-\s3807\s0x7f7e660b5000\s/lib/x86_64-linux-gnu/libbz2.so.1.0", + out, + ) + assert re.search( + rb"gdu-notificatio\s3878\s0x7f25ce33e000\s/usr/lib/x86_64-linux-gnu/libXau.so.6", + out, + ) + assert re.search( + rb"bash\s8600\s0x7fe78a85f000\s/lib/x86_64-linux-gnu/libnss_files.so.2", + out, + ) + + assert out.count(b"\n") >= 2677 + assert rc == 0 + + # MAC From cf81ceda8262b1bdfc528277cd405c24d572e320 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 20:20:25 +1100 Subject: [PATCH 028/104] Fix CodeQL suggestion --- volatility3/framework/symbols/linux/extensions/elf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 4b2b29b54..828370fe8 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -351,6 +351,8 @@ class elf_phdr(objects.StructType): if self._parent_e_type.description == "ET_DYN": offset = self._parent_offset + offset except ValueError: + # Unknown ELF object file type. Anyway, if the ELF object file type is not a + # shared object (ET_DYN), the virtual address is 'p_vaddr'. pass return offset From 0899ba8d7fedece67b520c8a2a31c45868177ce1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 28 Feb 2024 23:09:46 +0000 Subject: [PATCH 029/104] Documentation: Fix up syntax errors involving * character --- volatility3/framework/interfaces/plugins.py | 2 +- volatility3/framework/plugins/linux/kmsg.py | 4 +++- volatility3/framework/plugins/linux/pslist.py | 6 +++--- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- volatility3/framework/plugins/windows/mftscan.py | 4 ++-- .../framework/symbols/linux/extensions/__init__.py | 8 ++++---- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 29395aadf..697e4cdc3 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -60,7 +60,7 @@ class FileHandlerInterface(io.RawIOBase): @staticmethod def sanitize_filename(filename: str) -> str: """Sanititizes the filename to ensure only a specific whitelist of characters is allowed through""" - allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]\{\}!$%^:#~?<>,|" + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]{}!$%^:#~?<>,|" result = "" for char in filename: if char in allowed: diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index dd707a7ff..c5e0fc302 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -198,7 +198,9 @@ class ABCKmsg(ABC): class Kmsg_pre_3_5(ABCKmsg): """The kernel ring buffer (log_buf) is a char array that sequentially stores log lines, each separated by newline (LF) characters. i.e: - <6>[ 9565.250411] line1!\n<6>[ 9565.250412] line2\n... + + <6>[ 9565.250411] line1!\\n<6>[ 9565.250412] line2\\n... + """ @classmethod diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 771040dc0..046ee43d8 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -83,11 +83,11 @@ class PsList(interfaces.plugins.PluginInterface): cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False ) -> Tuple[int, int, int, str]: """Extract the fields needed for the final output + Args: task: A task object from where to get the fields. - decorate_comm: If True, it decorates the comm string of - - User threads: in curly brackets, - - Kernel threads: in square brackets + decorate_comm: If True, it decorates the comm string of user threads in curly brackets, + and of Kernel threads in square brackets. Defaults to False. Returns: A tuple with the fields to show in the plugin output. diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index e9c98a227..78217fbec 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -83,7 +83,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock: Kernel generic `sock` object Returns a tuple with: - sock: The respective kernel's \*_sock object for that socket family + sock: The respective kernel's \\*_sock object for that socket family sock_stat: A tuple with the source and destination (address and port) along with its state string socket_filter: A dictionary with information about the socket filter """ @@ -501,7 +501,7 @@ class Sockstat(plugins.PluginInterface): family: Socket family string (AF_UNIX, AF_INET, etc) sock_type: Socket type string (STREAM, DGRAM, etc) protocol: Protocol string (UDP, TCP, etc) - sock_fields: A tuple with the \*_sock object, the sock stats and the extended info dictionary + sock_fields: A tuple with the \\*_sock object, the sock stats and the extended info dictionary """ vmlinux = context.modules[symbol_table] diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 91a2e9152..85c072037 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -38,7 +38,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\*|BAAD/"} + {"yara_rules": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File @@ -197,7 +197,7 @@ class ADS(interfaces.plugins.PluginInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\*|BAAD/"} + {"yara_rules": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 751c80cb5..588ce3874 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1137,17 +1137,17 @@ class vfsmount(objects.StructType): """Helper to make sure it is comparing two pointers to 'vfsmount'. Depending on the kernel version, the calling object (self) could be - a 'vfsmount \*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust + a 'vfsmount \\*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust in the framework "auto" dereferencing ability to assure that when we reach this point 'self' will be a 'vfsmount' already and self.vol.offset - a 'vfsmount \*' and not a 'vfsmount \*\*'. The argument must be a 'vfsmount \*'. + a 'vfsmount \\*' and not a 'vfsmount \\*\\*'. The argument must be a 'vfsmount \\*'. Typically, it's called from do_get_path(). Args: - vfsmount_ptr (vfsmount \*): A pointer to a 'vfsmount' + vfsmount_ptr (vfsmount *): A pointer to a 'vfsmount' Raises: - exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \*' + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \\*' Returns: bool: 'True' if the given argument points to the the same 'vfsmount' From 338106dfe8e0667a07b0ab0ba4d52fbf5f4d51e7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 14:46:35 +1100 Subject: [PATCH 030/104] Move the memory page parameters to the Intel layer --- volatility3/framework/constants/linux/__init__.py | 5 ----- volatility3/framework/layers/intel.py | 12 ++++++++++++ volatility3/framework/plugins/linux/elfs.py | 14 +++++--------- .../framework/symbols/linux/extensions/__init__.py | 2 +- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 5e82e580e..3eabc2341 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -9,11 +9,6 @@ from enum import IntEnum KERNEL_NAME = "__kernel__" -# arch/x86/include/asm/page_types.h -PAGE_SHIFT = 12 -PAGE_SIZE = 1 << PAGE_SHIFT -PAGE_MASK = ~(PAGE_SIZE - 1) - """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" # include/linux/sched.h diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index ae477854d..75e561b33 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -67,6 +67,12 @@ class Intel(linear.LinearlyMappedLayer): math.ceil(math.log2(struct.calcsize(self._entry_format))) ) + @classproperty + @functools.lru_cache() + def page_shift(cls) -> int: + """Page shift for the intel memory layers.""" + return cls._page_size_in_bits + @classproperty @functools.lru_cache() def page_size(cls) -> int: @@ -76,6 +82,12 @@ class Intel(linear.LinearlyMappedLayer): """ return 1 << cls._page_size_in_bits + @classproperty + @functools.lru_cache() + def page_mask(cls) -> int: + """Page mask for the intel memory layers.""" + return ~(cls.page_size - 1) + @classproperty @functools.lru_cache() def bits_per_register(cls) -> int: diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 7171a6616..43cd6bb8b 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -14,11 +14,7 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux.extensions import elf -from volatility3.framework.constants.linux import ( - PAGE_SIZE, - PAGE_MASK, - ELF_MAX_EXTRACTION_SIZE, -) +from volatility3.framework.constants.linux import ELF_MAX_EXTRACTION_SIZE from volatility3.plugins.linux import pslist @@ -106,11 +102,11 @@ class Elfs(plugins.PluginInterface): # Use complete memory pages for dumping # If start isn't a multiple of a page, stick to the highest multiple < start # If end isn't a multiple of a page, stick to the lowest multiple > end - if start % PAGE_SIZE: - start = start & PAGE_MASK + if start % proc_layer.page_size: + start = start & proc_layer.page_mask - if end % PAGE_SIZE: - end = (end & PAGE_MASK) + PAGE_SIZE + if end % proc_layer.page_size: + end = (end & proc_layer.page_mask) + proc_layer.page_size real_size = end - start diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0a3db9fcd..1faafc267 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -640,7 +640,7 @@ class vm_area_struct(objects.StructType): elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True elif proclayer and "x" in flags_str: - for i in range(self.vm_start, self.vm_end, 1 << constants.linux.PAGE_SHIFT): + for i in range(self.vm_start, self.vm_end, proclayer.page_size): try: if proclayer.is_dirty(i): vollog.warning( From d5a0543a2d2d950c6743455f3f0ffa0b5afc2097 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 15:34:20 +1100 Subject: [PATCH 031/104] Use the ELF class constant instead of hardcoding a value. Fixed some f-strings --- volatility3/framework/layers/xen.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index 927b30430..e7aa0ccec 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -5,6 +5,7 @@ from typing import Optional from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.layers import elf from volatility3.framework.symbols import intermed +from volatility3.framework.constants.linux import ELF_CLASS vollog = logging.getLogger(__name__) @@ -14,7 +15,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): _header_struct = struct.Struct(" Date: Thu, 29 Feb 2024 15:34:48 +1100 Subject: [PATCH 032/104] Update author and modification time --- volatility3/framework/symbols/linux/elf.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/elf.json b/volatility3/framework/symbols/linux/elf.json index e0a95bbba..79a96e07a 100644 --- a/volatility3/framework/symbols/linux/elf.json +++ b/volatility3/framework/symbols/linux/elf.json @@ -1087,8 +1087,8 @@ "metadata": { "producer": { "version": "0.0.2", - "name": "ikelos-by-hand", - "datetime": "2019-10-21T22:52:00" + "name": "gcmoreira-by-hand", + "datetime": "2024-02-19T14:37:00" }, "format": "6.1.0" } From 9b0915dc85470df78bc739148093bb72d659297c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 15:36:46 +1100 Subject: [PATCH 033/104] Add logging for unknown ELF types --- volatility3/framework/layers/elf.py | 4 ++ volatility3/framework/plugins/linux/elfs.py | 4 ++ .../framework/symbols/linux/extensions/elf.py | 66 ++++++++++++++++--- 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index 6bd5c2d63..81f3c3634 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -55,6 +55,10 @@ class Elf64Layer(segmented.SegmentedLayer): try: ptype = phdr.p_type.description except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue if ( diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 43cd6bb8b..6820576dc 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -93,6 +93,10 @@ class Elfs(plugins.PluginInterface): if phdr.p_type.description != "PT_LOAD": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue start = phdr.p_vaddr diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 828370fe8..2cf5c3d4e 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -165,6 +165,10 @@ class elf(objects.StructType): if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue for dsec in phdr.dynamic_sections(): @@ -172,6 +176,10 @@ class elf(objects.StructType): if dsec.d_tag.description != "DT_PLTGOT": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF dynamic section type: {dsec.d_tag}", + ) continue got_start = dsec.d_ptr @@ -186,16 +194,27 @@ class elf(objects.StructType): layer_name=self.vol.layer_name, ) if not link_map_ptr: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid ELF link map pointer at 0x{link_map_addr:x}", + ) continue linkmap_symname = ( elf_symbol_table + constants.BANG + self._type_prefix + "LinkMap" ) - link_map = self._context.object( - object_type=linkmap_symname, - offset=link_map_ptr, - layer_name=self.vol.layer_name, - ) + try: + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map_ptr, + layer_name=self.vol.layer_name, + ) + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid ELF link map address at 0x{link_map_ptr:x}", + ) + continue while link_map and link_map.vol.offset != 0: if link_map.vol.offset in link_maps_seen: @@ -204,11 +223,18 @@ class elf(objects.StructType): yield link_map - link_map = self._context.object( - object_type=linkmap_symname, - offset=link_map.l_next, - layer_name=self.vol.layer_name, - ) + try: + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map.l_next, + layer_name=self.vol.layer_name, + ) + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"ELF link map linked list is corrupt at 0x{self.vol.offset:x}", + ) + break def _find_symbols(self): dt_strtab = None @@ -221,6 +247,10 @@ class elf(objects.StructType): if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue # This section contains pointers to the strtab, symtab, and strent sections @@ -228,6 +258,10 @@ class elf(objects.StructType): try: dtag = dsec.d_tag.description except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF dynamic section type: {dsec.d_tag}", + ) continue if dtag == "DT_STRTAB": @@ -353,6 +387,10 @@ class elf_phdr(objects.StructType): except ValueError: # Unknown ELF object file type. Anyway, if the ELF object file type is not a # shared object (ET_DYN), the virtual address is 'p_vaddr'. + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF object type: {self._parent_e_type}", + ) pass return offset @@ -365,6 +403,10 @@ class elf_phdr(objects.StructType): except ValueError: # If the value is outside the ones declared in the enumeration, an # exception is raised + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {self.p_type}", + ) return None # the buffer of array starts at elf_base + our virtual address ( offset ) @@ -398,6 +440,10 @@ class elf_linkmap(objects.StructType): buf = self._context.layers.read(self.vol.layer_name, self.l_name, 256) except exceptions.PagedInvalidAddressException: # Protection against memory smear + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid l_name address for ELF link map at 0x{self.vol.offset:x}", + ) return None idx = buf.find(b"\x00") From 3c9af096e15402a403b41b29a79ab3c74cc617f1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 16:17:47 +1100 Subject: [PATCH 034/104] Add missing PAGE_SHIFT replacement --- volatility3/framework/symbols/linux/extensions/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1faafc267..04cfa099f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -611,7 +611,8 @@ class vm_area_struct(objects.StructType): def get_page_offset(self) -> int: if self.vm_file == 0: return 0 - return self.vm_pgoff << constants.linux.PAGE_SHIFT + parent_layer = self._context.layers[self.vol.layer_name] + return self.vm_pgoff << parent_layer.page_shift def get_name(self, context, task): if self.vm_file != 0: From b681438ddcfdc33defa34d15cf48bd9ce4c9ca58 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 16:21:34 +1100 Subject: [PATCH 035/104] Remove unnecessary 'pass' statement. --- volatility3/framework/symbols/linux/extensions/elf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 2cf5c3d4e..eadcbbae0 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -391,7 +391,6 @@ class elf_phdr(objects.StructType): constants.LOGLEVEL_VVVV, f"Skipping unknown ELF object type: {self._parent_e_type}", ) - pass return offset From a8e10828273fb8cfa74743137ba70fd730e42619 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 29 Feb 2024 14:36:46 +0100 Subject: [PATCH 036/104] restore pslist, add queue_head_t type class --- volatility3/framework/plugins/mac/pslist.py | 8 ++------ volatility3/framework/symbols/mac/__init__.py | 4 +++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 9835644b8..9b570f3f9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -49,9 +49,7 @@ class PsList(interfaces.plugins.PluginInterface): ] @classmethod - def get_list_tasks( - cls, method: str - ) -> Callable[ + def get_list_tasks(cls, method: str) -> Callable[ [interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface], ]: @@ -188,9 +186,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object( - object_type="queue_entry", offset=kernel.get_symbol("tasks").address - ) + queue_entry = kernel.object_from_symbol(symbol_name="tasks") seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index 56ac96633..be4fe8cd1 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -21,12 +21,14 @@ class MacKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("vm_map_object", extensions.vm_map_object) self.set_type_class("socket", extensions.socket) self.set_type_class("inpcb", extensions.inpcb) - self.set_type_class("queue_entry", extensions.queue_entry) self.set_type_class("ifnet", extensions.ifnet) self.set_type_class("sockaddr_dl", extensions.sockaddr_dl) self.set_type_class("sockaddr", extensions.sockaddr) self.set_type_class("sysctl_oid", extensions.sysctl_oid) self.set_type_class("kauth_scope", extensions.kauth_scope) + # https://developer.apple.com/documentation/kernel/queue_head_t + self.set_type_class("queue_entry", extensions.queue_entry) + self.set_type_class("queue_head_t", extensions.queue_entry) class MacUtilities(interfaces.configuration.VersionableInterface): From d56ccbfd0b749d8f2ad69764658bf8f3856b8d21 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 14:18:30 +0100 Subject: [PATCH 037/104] allow specifying integers as 0x in ListRequirement --- volatility3/cli/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 3de7d8f8f..a67762a31 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,7 +827,11 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - additional["type"] = requirement.element_type + # Allow a list of integers, specified with convenient 0x hexadecimal format + if requirement.element_type == int: + additional["type"] = lambda x: int(x, 0) + else: + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 3040bc5fc21f31e27f75ac8c3ec359e43454d098 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 14:20:13 +0100 Subject: [PATCH 038/104] add --dump mechanism, similarly to linux.proc --- .../framework/plugins/mac/proc_maps.py | 173 +++++++++++++++++- 1 file changed, 169 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 781b3ed66..204d414e4 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -2,17 +2,23 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.mac import pslist +from typing import Callable, Generator, Type, Optional +import logging + +vollog = logging.getLogger(__name__) class Maps(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" _required_framework_version = (2, 0, 0) + _version = (1, 1, 0) + MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod def get_requirements(cls): @@ -31,14 +37,155 @@ class Maps(interfaces.plugins.PluginInterface): element_type=int, optional=True, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed memory segments", + default=False, + optional=True, + ), + requirements.ListRequirement( + name="address", + description="Process virtual memory addresses to include " + "(all other VMA sections are excluded). This can be any " + "virtual address within the VMA section. Virtual addresses " + "must be separated by a space.", + element_type=int, + optional=True, + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size for dumped VMA sections " + "(all the bigger sections will be ignored)", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), ] + @classmethod + def list_vmas( + cls, + task: interfaces.objects.ObjectInterface, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: True, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Lists the Virtual Memory Areas of a specific process. + + Args: + task: task object from which to list the vma + filter_func: Function to take a vma and return False if it should be filtered out + + Returns: + Yields vmas based on the task and filtered based on the filter function + """ + for vma in task.get_map_iter(): + if filter_func(vma): + yield vma + else: + vollog.debug( + f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.p_pid} due to filter_func" + ) + + @classmethod + def vma_dump( + cls, + context: interfaces.context.ContextInterface, + task: interfaces.objects.ObjectInterface, + vm_start: int, + vm_end: int, + open_method: Type[interfaces.plugins.FileHandlerInterface], + maxsize: int = MAXSIZE_DEFAULT, + ) -> Optional[interfaces.plugins.FileHandlerInterface]: + """Extracts the complete data for VMA as a FileInterface. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + task: an task_struct instance + vm_start: The start virtual address from the vma to dump + vm_end: The end virtual address from the vma to dump + open_method: class to provide context manager for opening the file + maxsize: Max size of VMA section (default MAXSIZE_DEFAULT) + + Returns: + An open FileInterface object containing the complete data for the task or None in the case of failure + """ + pid = task.p_pid + + try: + proc_layer_name = task.add_process_layer() + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + pid, excp.invalid_address, excp.layer_name + ) + ) + return None + vm_size = vm_end - vm_start + + # check if vm_size is negative, this should never happen. + if vm_size < 0: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is negative." + ) + return None + # check if vm_size is larger than the maxsize limit, and therefore is not saved out. + if maxsize <= vm_size: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is larger than maxsize limit of {maxsize}" + ) + return None + proc_layer = context.layers[proc_layer_name] + file_name = f"pid.{pid}.vma.{vm_start:#x}-{vm_end:#x}.dmp" + try: + file_handle = open_method(file_name) + chunk_size = 1024 * 1024 * 10 + offset = vm_start + while offset < vm_start + vm_size: + to_read = min(chunk_size, vm_start + vm_size - offset) + data = proc_layer.read(offset, to_read, pad=True) + file_handle.write(data) + offset += to_read + except Exception as excp: + vollog.debug(f"Unable to dump virtual memory {file_name}: {excp}") + return None + return file_handle + def _generator(self, tasks): + address_list = self.config.get("address", None) + if not address_list: + # do not filter as no address_list was supplied + vma_filter_func = lambda _: True + else: + # filter for any vm_start that matches the supplied address config + def vma_filter_function(task: interfaces.objects.ObjectInterface) -> bool: + addrs_in_vma = [ + addr + for addr in address_list + if task.links.start <= addr <= task.links.end + ] + + # if any of the user supplied addresses would fall within this vma return true + if addrs_in_vma: + return True + else: + return False + + vma_filter_func = vma_filter_function + for task in tasks: process_name = utility.array_to_string(task.p_comm) process_pid = task.p_pid - for vma in task.get_map_iter(): + for vma in self.list_vmas(task, filter_func=vma_filter_func): + try: + vm_start = vma.links.start + vm_end = vma.links.end + except AttributeError: + vollog.debug( + f"Unable to find the vm_start and vm_end for vma at {vma.vol.offset:#x} for pid {process_pid}" + ) + continue + path = vma.get_path( self.context, self.context.modules[self.config["kernel"]].symbol_table_name, @@ -46,15 +193,32 @@ class Maps(interfaces.plugins.PluginInterface): if path == "": path = vma.get_special_path() + file_output = "Disabled" + if self.config["dump"]: + file_output = "Error outputting file" + file_handle = self.vma_dump( + self.context, + task, + vm_start, + vm_end, + self.open, + self.config["maxsize"], + ) + + if file_handle: + file_handle.close() + file_output = file_handle.preferred_filename + yield ( 0, ( process_pid, process_name, - format_hints.Hex(vma.links.start), - format_hints.Hex(vma.links.end), + format_hints.Hex(vm_start), + format_hints.Hex(vm_end), vma.get_perms(), path, + file_output, ), ) @@ -72,6 +236,7 @@ class Maps(interfaces.plugins.PluginInterface): ("End", format_hints.Hex), ("Protection", str), ("Map Name", str), + ("File output", str), ], self._generator( list_tasks(self.context, self.config["kernel"], filter_func=filter_func) From 965ddcb674d30528792a404b819635c60c207f8e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 19:30:46 +0100 Subject: [PATCH 039/104] make queue_head_t optional, as it might not exist in all ISF --- volatility3/framework/symbols/mac/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index be4fe8cd1..83aebb13b 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -28,7 +28,7 @@ class MacKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("kauth_scope", extensions.kauth_scope) # https://developer.apple.com/documentation/kernel/queue_head_t self.set_type_class("queue_entry", extensions.queue_entry) - self.set_type_class("queue_head_t", extensions.queue_entry) + self.optional_set_type_class("queue_head_t", extensions.queue_entry) class MacUtilities(interfaces.configuration.VersionableInterface): From 2e835099b58b54af2f988355ae04cbfeff9cd13e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 20:19:31 +0100 Subject: [PATCH 040/104] fix wrong list_head comparison + better naming --- .../symbols/mac/extensions/__init__.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index c89b527e6..0a6bfbb90 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -490,22 +490,24 @@ class queue_entry(objects.StructType): for attr in ["next", "prev"]: with contextlib.suppress(exceptions.InvalidAddressException): - n = getattr(self, attr).dereference().cast(type_name) - - while n is not None and n.vol.offset != list_head: - if n.vol.offset in seen: + queue_element = getattr(self, attr).dereference().cast(type_name) + while ( + queue_element is not None + and queue_element.vol.offset != list_head.vol.offset + ): + if queue_element.vol.offset in seen: break - yield n + yield queue_element - seen.add(n.vol.offset) + seen.add(queue_element.vol.offset) yielded = yielded + 1 if yielded == max_size: - return + return None - n = ( - getattr(n.member(attr=member_name), attr) + queue_element = ( + getattr(queue_element.member(attr=member_name), attr) .dereference() .cast(type_name) ) From caf108b604be6925a2e5e1a4afc627da1fa943a5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 17:17:48 +0100 Subject: [PATCH 041/104] macOS dmesg plugin support --- volatility3/framework/plugins/mac/dmesg.py | 79 ++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 volatility3/framework/plugins/mac/dmesg.py diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py new file mode 100644 index 000000000..a006ff854 --- /dev/null +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -0,0 +1,79 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility + +vollog = logging.getLogger(__name__) + + +class Dmesg(interfaces.plugins.PluginInterface): + """Prints the kernel log buffer.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + ] + + @classmethod + def get_kernel_log_buffer( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str + ): + """ + Online documentation : + - https://github.com/apple-open-source/macos/blob/master/xnu/bsd/sys/msgbuf.h + - https://github.com/apple-open-source/macos/blob/ea4cd5a06831aca49e33df829d2976d6de5316ec/xnu/bsd/kern/subr_log.c#L751 + Volatility 2 plugin : + - https://github.com/volatilityfoundation/volatility/blob/master/volatility/plugins/mac/dmesg.py + """ + + kernel = context.modules[kernel_module_name] + if not kernel.has_symbol("msgbufp"): + vollog.error( + 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' + ) + return [] + + msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") + msgbufp = msgbufp_ptr.dereference() + msg_size = msgbufp.msg_size # max buffer size + msg_bufx = msgbufp.msg_bufx # write pointer + msg_bufc = msgbufp.msg_bufc + # msg_bufc is circular, meaning that if its size exceeds msg_size, + # msg_bufx will point to the beginning of the buffer and start overwriting. + msg_bufc_data: str = utility.pointer_to_string(msg_bufc, msg_size) + # Avoid OOB reads + msg_bufx = msg_bufx if msg_bufx <= msg_size else 0 + # We directly take into account the case where the write buffer did a loop, + # as older messages will start at msg_bufx offset (not overwritten yet). + dmesg = msg_bufc_data[msg_bufx:] + dmesg += msg_bufc_data[:msg_bufx] + + # Yield each line + for dmesg_line in dmesg.splitlines(): + yield (dmesg_line.strip(),) + + def _generator(self): + for value in self.get_kernel_log_buffer( + context=self.context, kernel_module_name=self.config["kernel"] + ): + yield (0, value) + + def run(self): + return renderers.TreeGrid( + [ + ("line", str), + ], + self._generator(), + ) From 8d79e3211ba2e8ea751c657b5da9f99a8b3a90c2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 17:30:51 +0100 Subject: [PATCH 042/104] prefer TypeError to vollog.error --- volatility3/framework/plugins/mac/dmesg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index a006ff854..b837b9db6 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -40,10 +40,9 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): - vollog.error( + raise TypeError( 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) - return [] msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") msgbufp = msgbufp_ptr.dereference() From 2f0bb6b297ffe601c563d646f09c80b8b0a4864f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 18:39:48 +0100 Subject: [PATCH 043/104] do not strip each line --- volatility3/framework/plugins/mac/dmesg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index b837b9db6..d4f2c869d 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -61,7 +61,7 @@ class Dmesg(interfaces.plugins.PluginInterface): # Yield each line for dmesg_line in dmesg.splitlines(): - yield (dmesg_line.strip(),) + yield (dmesg_line,) def _generator(self): for value in self.get_kernel_log_buffer( From cb6f8c507268dd399f24e762766b3ba42498799a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 4 Mar 2024 20:47:49 +1100 Subject: [PATCH 044/104] Rename methods to be private --- .../framework/plugins/linux/library_list.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py index ed5545347..062ed078e 100644 --- a/volatility3/framework/plugins/linux/library_list.py +++ b/volatility3/framework/plugins/linux/library_list.py @@ -43,7 +43,7 @@ class LibraryList(interfaces.plugins.PluginInterface): ), ] - def get_libdl_libraries( + def _get_libdl_libraries( self, proc_layer_name: str, vma_start: int ) -> interfaces.objects.ObjectInterface: """Get the ELF link map objects for the given VMA address @@ -81,7 +81,7 @@ class LibraryList(interfaces.plugins.PluginInterface): # Protection against memory smear in this VMA pass - def get_libdl_maps( + def _get_libdl_maps( self, task: interfaces.objects.ObjectInterface, proc_layer_name: str ) -> interfaces.objects.ObjectInterface: """Get the ELF link maps objects for a task @@ -96,14 +96,14 @@ class LibraryList(interfaces.plugins.PluginInterface): link_map_seen = set() for vma in task.mm.get_vma_iter(): - for link_map in self.get_libdl_libraries(proc_layer_name, vma.vm_start): + for link_map in self._get_libdl_libraries(proc_layer_name, vma.vm_start): if link_map.l_addr in link_map_seen: continue yield link_map link_map_seen.add(link_map.l_addr) - def get_task_libraries( + def _get_task_libraries( self, task: interfaces.objects.ObjectInterface ) -> Tuple[int, str]: """Get the task libraries from the ELF headers found within the memory maps @@ -118,13 +118,13 @@ class LibraryList(interfaces.plugins.PluginInterface): if not proc_layer_name: return - for elf_link_map in self.get_libdl_maps(task, proc_layer_name): + for elf_link_map in self._get_libdl_maps(task, proc_layer_name): name = elf_link_map.get_name() if not name: continue yield elf_link_map.l_addr, name - def get_tasks_libraries( + def _get_tasks_libraries( self, tasks: Iterable[interfaces.objects.ObjectInterface], ) -> Iterable[Tuple[str, int, int, str]]: @@ -139,7 +139,7 @@ class LibraryList(interfaces.plugins.PluginInterface): """ for task in tasks: task_name = utility.array_to_string(task.comm) - for linkmap_addr, linkmap_name in self.get_task_libraries(task): + for linkmap_addr, linkmap_name in self._get_task_libraries(task): yield task_name, task.tgid, linkmap_addr, linkmap_name def _format_fields(self, fields): @@ -149,7 +149,7 @@ class LibraryList(interfaces.plugins.PluginInterface): def _generator( self, tasks: Iterable[interfaces.objects.ObjectInterface] ) -> Iterable[Tuple[int, Tuple]]: - for fields in self.get_tasks_libraries(tasks): + for fields in self._get_tasks_libraries(tasks): yield 0, self._format_fields(fields) def run(self): From 373bf9dfaa466fdaba9635a78a6449679af502bf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 4 Mar 2024 18:33:03 +0000 Subject: [PATCH 045/104] Windows: Fix driverirp black issue --- volatility3/framework/plugins/windows/driverirp.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index f7eca0359..433c61ca2 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -115,9 +115,17 @@ class DriverIrp(interfaces.plugins.PluginInterface): ) if not module_found: - yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], - format_hints.Hex(address), renderers.NotAvailableValue(), renderers.NotAvailableValue())) - + yield ( + 0, + ( + format_hints.Hex(driver.vol.offset), + driver_name, + MAJOR_FUNCTIONS[i], + format_hints.Hex(address), + renderers.NotAvailableValue(), + renderers.NotAvailableValue(), + ), + ) def run(self): return renderers.TreeGrid( From 403804431ae0ae5118b13cb0f894edfb9cc03d73 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:11:17 +0100 Subject: [PATCH 046/104] prefer SymbolError to TypeError --- volatility3/framework/plugins/mac/dmesg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index d4f2c869d..12c241641 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -3,7 +3,7 @@ # import logging -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -40,7 +40,7 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): - raise TypeError( + raise exceptions.SymbolError( 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) From 1bf18dbd8a9f68381dfbdb46c4d76ab92cbe0a68 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:11:50 +0100 Subject: [PATCH 047/104] implicit pointer dereference --- volatility3/framework/plugins/mac/dmesg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index 12c241641..daa6617ad 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -44,8 +44,7 @@ class Dmesg(interfaces.plugins.PluginInterface): 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) - msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") - msgbufp = msgbufp_ptr.dereference() + msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") msg_size = msgbufp.msg_size # max buffer size msg_bufx = msgbufp.msg_bufx # write pointer msg_bufc = msgbufp.msg_bufc From 98705e110df4691dc974210f0818621db2f27bbf Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:12:12 +0100 Subject: [PATCH 048/104] more specific msg_bufx comment --- volatility3/framework/plugins/mac/dmesg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index daa6617ad..7f817e605 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -46,7 +46,7 @@ class Dmesg(interfaces.plugins.PluginInterface): msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") msg_size = msgbufp.msg_size # max buffer size - msg_bufx = msgbufp.msg_bufx # write pointer + msg_bufx = msgbufp.msg_bufx # write index of the msg_bufc circular buffer msg_bufc = msgbufp.msg_bufc # msg_bufc is circular, meaning that if its size exceeds msg_size, # msg_bufx will point to the beginning of the buffer and start overwriting. From 69a6c7d5b7cbfb99e8ec70d05dac95a6dea71713 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:16:55 +0100 Subject: [PATCH 049/104] revert int as hex format commit --- volatility3/cli/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index a67762a31..3de7d8f8f 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,11 +827,7 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - # Allow a list of integers, specified with convenient 0x hexadecimal format - if requirement.element_type == int: - additional["type"] = lambda x: int(x, 0) - else: - additional["type"] = requirement.element_type + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 9914f339dfc1cdd511fc3fad676b92b06520243e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:19:15 +0100 Subject: [PATCH 050/104] simplify addrs_in_vma check --- volatility3/framework/plugins/mac/proc_maps.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 204d414e4..fe5179dfa 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -165,10 +165,7 @@ class Maps(interfaces.plugins.PluginInterface): ] # if any of the user supplied addresses would fall within this vma return true - if addrs_in_vma: - return True - else: - return False + return bool(addrs_in_vma) vma_filter_func = vma_filter_function From f4b12c6406e2f1accc147069403e363e7c6a01dc Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:26:18 +0100 Subject: [PATCH 051/104] allow ints in the 0x format in ListRequirement --- volatility3/cli/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 3de7d8f8f..35e977eba 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,7 +827,11 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - additional["type"] = requirement.element_type + # Allow a list of integers, specified with the convenient 0x hexadecimal format + if requirement.element_type == int: + additional["type"] = lambda x: int(x, 0) + else: + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 1f9a983f492ff7983030b8fa27b509106594f735 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:38:01 +0100 Subject: [PATCH 052/104] correct use of SymbolError --- volatility3/framework/plugins/mac/dmesg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index 7f817e605..f9f06a666 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -41,7 +41,9 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): raise exceptions.SymbolError( - 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' + "msgbufp", + kernel.symbol_table_name, + 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.', ) msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") From 9edf33b7212d46682b48dbb40b744f198f8741a8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 11 Mar 2024 21:20:29 +0000 Subject: [PATCH 053/104] Layers: Improve logging on crashdump layer --- volatility3/framework/layers/crash.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 8efd4f7c7..8fd0abbcc 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -261,11 +261,15 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): progress_callback: constants.ProgressCallback = None, ) -> Optional[interfaces.layers.DataLayerInterface]: for layer in [WindowsCrashDump32Layer, WindowsCrashDump64Layer]: - with contextlib.suppress(WindowsCrashDumpFormatException): + try: layer.check_header(context.layers[layer_name]) new_name = context.layers.free_layer_name(layer.__name__) context.config[ interfaces.configuration.path_join(new_name, "base_layer") ] = layer_name return layer(context, new_name, new_name) + except (WindowsCrashDump32Layer, WindowsCrashDump64Layer) as excp: + vollog.log( + constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" + ) return None From 084ea38f84b36e8db2594fc31f1c2d61b3b7c6ce Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 12 Mar 2024 00:13:02 +0000 Subject: [PATCH 054/104] Layers: Fix up typo in recent crashdump patch --- volatility3/framework/layers/crash.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 8fd0abbcc..5598fc2e4 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -268,8 +268,8 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): interfaces.configuration.path_join(new_name, "base_layer") ] = layer_name return layer(context, new_name, new_name) - except (WindowsCrashDump32Layer, WindowsCrashDump64Layer) as excp: + except WindowsCrashDumpFormatException as excp: vollog.log( constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" - ) + )\ return None From 8dbc64f4a8678455adbac80ac716dfa62b3aecb2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 12 Mar 2024 00:14:30 +0000 Subject: [PATCH 055/104] Layers: Fix up typo in recent crashdump patch - take 2 --- volatility3/framework/layers/crash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 5598fc2e4..042b18ddc 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -271,5 +271,5 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): except WindowsCrashDumpFormatException as excp: vollog.log( constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" - )\ + ) return None From f6495d3d986e01fb96789a88aa89cc2c8e40cb71 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 16 Mar 2024 08:42:53 +0000 Subject: [PATCH 056/104] Documentation: Improve logging level docstrings --- volatility3/framework/constants/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 921f602fd..3c5e0eb2f 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -59,14 +59,18 @@ PACKAGE_VERSION = ( AUTOMAGIC_CONFIG_PATH = "automagic" """The root section within the context configuration for automagic values""" +LOGLEVEL_INFO = 20 +"""Logging level for information data, showed when use the requests any logging: -v""" +LOGLEVEL_DEBUG = 10 +"""Logging level for debugging data, showed when the user requests more logging detail: -vv""" LOGLEVEL_V = 9 -"""Logging level for a single -v""" +"""Logging level for the lowest "extra" level of logging: -vvv""" LOGLEVEL_VV = 8 -"""Logging level for -vv""" +"""Logging level for two levels of detail: -vvvv""" LOGLEVEL_VVV = 7 -"""Logging level for -vvv""" +"""Logging level for three levels of detail: -vvvvv""" LOGLEVEL_VVVV = 6 -"""Logging level for -vvvv""" +"""Logging level for four levels of detail: -vvvvvv""" CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" From 436d355ef301d9cd73883b5129f8b9f744c2ecb7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 18:44:09 +0100 Subject: [PATCH 057/104] incrementally order extra log levels --- volatility3/cli/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 35e977eba..ec9918a65 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -264,6 +264,17 @@ class CommandLine: file_logger.setFormatter(file_formatter) rootlog.addHandler(file_logger) vollog.info("Logging started") + + for level, level_value in enumerate( + [ + constants.LOGLEVEL_V, + constants.LOGLEVEL_VV, + constants.LOGLEVEL_VVV, + constants.LOGLEVEL_VVVV, + ] + ): + logging.addLevelName(level_value, f"DETAIL {level+1}") + if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None From 33d653e7a66b8b96350a3568c6d156eadb0cf878 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 18:44:19 +0100 Subject: [PATCH 058/104] use logging explicit constants --- volatility3/cli/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index ec9918a65..116cde114 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -278,9 +278,9 @@ class CommandLine: if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None - console.setLevel(30 - (partial_args.verbosity * 10)) + console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: - console.setLevel(10 - (partial_args.verbosity - 2)) + console.setLevel(logging.DEBUG - (partial_args.verbosity - 2)) for level, msg in delayed_logs: vollog.log(level, msg) From 745491c846314fb37231c892e2cfab9eed4275f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:02 +0100 Subject: [PATCH 059/104] put extra log level ordering in a function --- volatility3/cli/__init__.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 116cde114..457a49311 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -265,16 +265,7 @@ class CommandLine: rootlog.addHandler(file_logger) vollog.info("Logging started") - for level, level_value in enumerate( - [ - constants.LOGLEVEL_V, - constants.LOGLEVEL_VV, - constants.LOGLEVEL_VVV, - constants.LOGLEVEL_VVVV, - ] - ): - logging.addLevelName(level_value, f"DETAIL {level+1}") - + self.order_extra_verbose_levels() if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None @@ -706,6 +697,17 @@ class CommandLine: ) context.config[extended_path] = value + def order_extra_verbose_levels(self): + for level, level_value in enumerate( + [ + constants.LOGLEVEL_V, + constants.LOGLEVEL_VV, + constants.LOGLEVEL_VVV, + constants.LOGLEVEL_VVVV, + ] + ): + logging.addLevelName(level_value, f"DETAIL {level+1}") + def file_handler_class_factory(self, direct=True): output_dir = self.output_dir From 7ff5f57e5d4485ad2e5f2ad4dc78017880442c05 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:28 +0100 Subject: [PATCH 060/104] use logging explicit constants --- volatility3/cli/volshell/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 9e74acfec..998c8245b 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -198,9 +198,9 @@ class VolShell(cli.CommandLine): vollog.info("Logging started") if partial_args.verbosity < 3: - console.setLevel(30 - (partial_args.verbosity * 10)) + console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: - console.setLevel(10 - (partial_args.verbosity - 2)) + console.setLevel(logging.DEBUG - (partial_args.verbosity - 2)) for level, msg in delayed_logs: vollog.log(level, msg) From de4a3359982b67b74a354001d380502ff41804d1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:45 +0100 Subject: [PATCH 061/104] call extra verbose level ordering --- volatility3/cli/volshell/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 998c8245b..035ed9b2e 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -197,6 +197,7 @@ class VolShell(cli.CommandLine): vollog.addHandler(file_logger) vollog.info("Logging started") + self.order_extra_verbose_levels() if partial_args.verbosity < 3: console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: From 47891c3477fbd7bd008e8b3b8f119936289104de Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Apr 2024 21:17:42 +0100 Subject: [PATCH 062/104] Core: Develop nested requirement files --- requirements-dev.txt | 21 ++++----------------- requirements.txt | 4 ++-- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c9b615cd8..ae3482290 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,22 +1,9 @@ -# The following packages are required for core functionality. -pefile>=2023.2.7 - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 - -# This is required for several plugins that perform malware analysis and disassemble code. -# It can also improve accuracy of Windows 8 and later memory samples. -capstone>=3.0.5 - -# This is required by plugins that decrypt passwords, password hashes, etc. -pycryptodome +-r requirements.txt # This can improve error messages regarding improperly configured ISF files, # but is only recommended for development jsonschema>=2.3.0 -# This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0 +# Used to build executable file +pyinstaller>=6.5.0 +pyinstaller-hooks-contrib>=2024.3 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4d09ff82a..c63dc0b36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -# The following packages are required for core functionality. -pefile>=2023.2.7 +# Include the minimal requirements +-r requirements-minimal.txt # The following packages are optional. # If certain packages are not necessary, place a comment (#) at the start of the line. From c1f239b8d2171e83e331ad51bf7ff6e48ed04e52 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 7 Apr 2024 19:34:17 +0100 Subject: [PATCH 063/104] Linux: Improve debugging of pslist dump feature --- volatility3/framework/plugins/linux/elfs.py | 1 + volatility3/framework/plugins/linux/pslist.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 6820576dc..22e39d127 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -84,6 +84,7 @@ class Elfs(plugins.PluginInterface): ) if not elf_object.is_valid(): + vollog.debug("ELF object to be dumped is not valid") return None sections = {} diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 046ee43d8..1888bd7b8 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -142,6 +142,8 @@ class PsList(interfaces.plugins.PluginInterface): file_output = str(file_handle.preferred_filename) file_handle.close() break + else: + file_output = "VMA start matching task start_code not found" return file_output def _generator( From 974e6a4107cda567ce24a3d6384c28939ae64dd1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 21 Apr 2024 18:28:13 +0100 Subject: [PATCH 064/104] Core: When clearing the cache, actually clear cached files --- volatility3/framework/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 1565b2267..422c083d0 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -225,6 +225,12 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: def clear_cache(complete=False): try: + if complete: + glob_pattern = "*.cache" + for cache_filename in glob.glob( + os.path.join(constants.CACHE_PATH, glob_pattern) + ): + os.unlink(cache_filename) os.unlink(os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)) except FileNotFoundError: vollog.log(constants.LOGLEVEL_VVVV, "Attempting to clear a non-existant cache") From 8f0d1b37cc2f88646b5858509e1bc0d5bf4941e0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Apr 2024 20:51:25 +0100 Subject: [PATCH 065/104] Core: Improve error handling for proxy authentication issue --- volatility3/framework/layers/resources.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index a64fa7d7a..2dba7caa8 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -151,6 +151,12 @@ class ResourceAccessor(object): raise excp else: raise excp + except ValueError as excp: + # Reraise errors such as proxy auth errors as offline exception errors + # Example Proxy auth error - ValueError: AbstractDigestAuthHandler does not support the following scheme: 'Negotiate' + vollog.info(f"Cannot access {url} due to {excp} - Setting OFFLINE") + constants.OFFLINE = True + raise exceptions.OfflineException(url) except exceptions.OfflineException: vollog.info(f"Not accessing {url} in offline mode") raise From 35c583721892b84ae4f9b151ddccbb2e4ef60e9f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 30 Apr 2024 12:34:46 +0100 Subject: [PATCH 066/104] Core: Default to completely clearing the cache when requested --- volatility3/framework/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 422c083d0..feb97810b 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -223,7 +223,7 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: return plugin_list -def clear_cache(complete=False): +def clear_cache(complete=True): try: if complete: glob_pattern = "*.cache" From 314725f0c53811b6652f90c4b8050982b8329ab7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 1 May 2024 04:30:36 +0900 Subject: [PATCH 067/104] Fix: typo for CITATION.cff --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index c36c3b7d5..ac45dfc18 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -14,7 +14,7 @@ authors: identifiers: - type: url value: 'https://github.com/volatilityfoundation/volatility3' - description: Volatility 3 source code respository + description: Volatility 3 source code repository repository-code: 'https://github.com/volatilityfoundation/volatility3' url: 'https://github.com/volatilityfoundation/volatility3' abstract: >- From c7f29936dcfb81bb9ae3fe33e261db2bb7c3be85 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:15:45 +1000 Subject: [PATCH 068/104] CommandLine: Fix issue when --output filename doesn't contain an extension. If the argument is i.e. "--output aaa" ... it returned ".aaa" (hidden filename in linux) then "-1.aaa", "-2.aaa", etc. --- volatility3/cli/__init__.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 457a49311..c0eed20ee 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -716,19 +716,17 @@ class CommandLine: """Gets the final filename""" if output_dir is None: raise TypeError("Output directory is not a string") + os.makedirs(output_dir, exist_ok=True) - pref_name_array = self.preferred_filename.split(".") - filename, extension = ( - os.path.join(output_dir, ".".join(pref_name_array[:-1])), - pref_name_array[-1], - ) - output_filename = f"{filename}.{extension}" + output_filename = os.path.join(output_dir, self.preferred_filename) + filename, extension = os.path.splitext(output_filename) counter = 1 while os.path.exists(output_filename): - output_filename = f"{filename}-{counter}.{extension}" + output_filename = f"{filename}-{counter}{extension}" counter += 1 + return output_filename class CLIMemFileHandler(io.BytesIO, CLIFileHandler): From 32a5f131fd22d5351eee264fd6e58420a1458e9c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:17:51 +1000 Subject: [PATCH 069/104] LayerWriter plugin: Fix log wrong (non-existent) variable --- volatility3/framework/plugins/layerwriter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 1bee5f20d..a8664b3f1 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -115,9 +115,7 @@ class LayerWriter(plugins.PluginInterface): ) file_handle.close() except IOError as excp: - yield 0, ( - f"Layer cannot be written to {self.config['output_name']}: {excp}", - ) + yield 0, (f"Layer cannot be written to {output_name}: {excp}",) yield 0, (f"Layer has been written to {output_name}",) From 1246b20b6521e11110038fe091af912e2b5f4439 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:25:45 +1000 Subject: [PATCH 070/104] LayerWriter plugin: Code improvements --- volatility3/framework/plugins/layerwriter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index a8664b3f1..021c44248 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -95,7 +95,7 @@ class LayerWriter(plugins.PluginInterface): if not self.config["layers"]: self.config["layers"] = [] for name in self.context.layers: - if not self.context.layers[name].metadata.get("mapped", False): + if "mapped" not in self.context.layers[name].metadata: self.config["layers"] = [name] for name in self.config["layers"]: @@ -103,7 +103,8 @@ class LayerWriter(plugins.PluginInterface): if name not in self.context.layers: yield 0, (f"Layer Name {name} does not exist",) else: - output_name = self.config.get("output", ".".join([name, "raw"])) + default_output_name = f"{name}.raw" + output_name = self.config.get("output", default_output_name) try: file_handle = self.write_layer( self.context, From 6d22347ce6dd0152746564a2d77022ca1b4d9045 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:27:15 +1000 Subject: [PATCH 071/104] LayerWriter plugin: Fix --output argument. It's referenced in the code but never mentioned as a requirement --- volatility3/framework/plugins/layerwriter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 021c44248..d60ead682 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -38,6 +38,10 @@ class LayerWriter(plugins.PluginInterface): default=False, optional=True, ), + requirements.StringRequirement( + name="output", + description="Output filename", + ), requirements.ListRequirement( name="layers", element_type=str, From e5a5b895771b655d21c36689c33a534034c31e36 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:06:02 +1000 Subject: [PATCH 072/104] Intel layer: Fix. This if statement will never be executed unless "minimum_address > maximum_address" which doesn't make sense to me. --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 75e561b33..8589cdbb6 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -180,7 +180,7 @@ class Intel(linear.LinearlyMappedLayer): position = self._initial_position entry = self._initial_entry - if self.minimum_address > offset > self.maximum_address: + if not (self.minimum_address <= offset <= self.maximum_address): raise exceptions.PagedInvalidAddressException( self.name, offset, From f116c08a7ec60f62e3ef931de7639e402f421d65 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:07:23 +1000 Subject: [PATCH 073/104] NonLinearlySegmentedLayer: Fix maximum addresses in segmented layers --- volatility3/framework/layers/segmented.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/segmented.py b/volatility3/framework/layers/segmented.py index 0d29d8bff..e8c067072 100644 --- a/volatility3/framework/layers/segmented.py +++ b/volatility3/framework/layers/segmented.py @@ -152,7 +152,7 @@ class NonLinearlySegmentedLayer( raise ValueError("SegmentedLayer must contain some segments") if self._maxaddr is None: mapped, _, length, _ = self._segments[-1] - self._maxaddr = mapped + length + self._maxaddr = mapped + length - 1 return self._maxaddr @property From a6c77c488436c7b05040b5bf475be79cb59457f3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:09:45 +1000 Subject: [PATCH 074/104] LayerWriter: Fix - Last chunk size is wrongly calculated --- volatility3/framework/plugins/layerwriter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index d60ead682..178d86ba9 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -18,7 +18,7 @@ class LayerWriter(plugins.PluginInterface): default_block_size = 0x500000 _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -81,7 +81,7 @@ class LayerWriter(plugins.PluginInterface): file_handle = open_method(preferred_name) for i in range(0, layer.maximum_address, chunk_size): - current_chunk_size = min(chunk_size, layer.maximum_address - i) + current_chunk_size = min(chunk_size, layer.maximum_address + 1 - i) data = layer.read(i, current_chunk_size, pad=True) file_handle.write(data) if progress_callback: From d7aae3a9828ba03bd5531aa385724a954e48501a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:11:52 +1000 Subject: [PATCH 075/104] Store the real/final output filename so that we can notify it correctly to the user --- volatility3/cli/__init__.py | 4 ++-- volatility3/framework/plugins/layerwriter.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index c0eed20ee..9873791e3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -790,8 +790,8 @@ class CommandLine: return None self._file.close() - output_filename = self._get_final_filename() - os.rename(self._name, output_filename) + self._output_filename = self._get_final_filename() + os.rename(self._name, self._output_filename) if direct: return CLIDirectFileHandler diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 178d86ba9..20d10d636 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -119,6 +119,7 @@ class LayerWriter(plugins.PluginInterface): progress_callback=self._progress_callback, ) file_handle.close() + output_name = file_handle._output_filename except IOError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) From 85052b6238414617cc87f6812894f14917e03393 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 7 May 2024 08:12:31 +1000 Subject: [PATCH 076/104] LayerWriter: Fix missing optional flag for the --output argument --- volatility3/framework/plugins/layerwriter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 20d10d636..6a06cc607 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -41,6 +41,7 @@ class LayerWriter(plugins.PluginInterface): requirements.StringRequirement( name="output", description="Output filename", + optional=True, ), requirements.ListRequirement( name="layers", From 3f3f1a9c0daae397a84a515e84b29ba9ef0a0300 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 8 May 2024 11:23:01 -0500 Subject: [PATCH 077/104] Prevent duplicate processing of the same file object --- volatility3/framework/plugins/windows/dumpfiles.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 48539c752..33d2d0d41 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -244,6 +244,8 @@ class DumpFiles(interfaces.plugins.PluginInterface): symbol_table=kernel.symbol_table_name, ) + dumped_files = set() + for proc in procs: try: object_table = proc.ObjectTable @@ -267,6 +269,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_re.search(name): continue + if file_obj.vol.offset in dumped_files: + continue + dumped_files.add(file_obj.vol.offset) + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): @@ -303,6 +309,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_re.search(name): continue + if file_obj.vol.offset in dumped_files: + continue + dumped_files.add(file_obj.vol.offset) + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): From 3933061551eb0f31b99229ac8346718f773f574c Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 9 May 2024 09:17:32 -0500 Subject: [PATCH 078/104] Add a default 0 value to macb columns to avoid mactime not reporting timeline entries --- volatility3/framework/plugins/timeliner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index d1c1c0f70..26a100f53 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -183,16 +183,16 @@ class Timeliner(interfaces.plugins.PluginInterface): plugin_name, self._sanitize_body_format(item), self._text_format( - times.get(TimeLinerType.ACCESSED, "") + times.get(TimeLinerType.ACCESSED, "0") ), self._text_format( - times.get(TimeLinerType.MODIFIED, "") + times.get(TimeLinerType.MODIFIED, "0") ), self._text_format( - times.get(TimeLinerType.CHANGED, "") + times.get(TimeLinerType.CHANGED, "0") ), self._text_format( - times.get(TimeLinerType.CREATED, "") + times.get(TimeLinerType.CREATED, "0") ), ) ) From 4c937a922d92175135e64be2192b94e8a232c4df Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 9 May 2024 14:47:56 -0500 Subject: [PATCH 079/104] Correctly check for a failed read --- volatility3/framework/plugins/windows/handles.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index d43d26ef1..93ca37c6b 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -161,8 +161,9 @@ class Handles(interfaces.plugins.PluginInterface): except exceptions.SymbolError: return None - data = self.context.layers.read(virtual_layer_name, kvo + func_addr, 0x200) - if data is None: + try: + data = self.context.layers.read(virtual_layer_name, kvo + func_addr, 0x200) + except exceptions.InvalidAddressException: return None md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) From 2b165a56be4d5c804ec6104e165cdf0f862b7204 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 9 May 2024 14:53:24 -0500 Subject: [PATCH 080/104] Fix pre-existing formatting issue from black checks --- volatility3/framework/plugins/windows/handles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 93ca37c6b..15f5f69df 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -162,7 +162,9 @@ class Handles(interfaces.plugins.PluginInterface): return None try: - data = self.context.layers.read(virtual_layer_name, kvo + func_addr, 0x200) + data = self.context.layers.read( + virtual_layer_name, kvo + func_addr, 0x200 + ) except exceptions.InvalidAddressException: return None From 5d5fa96e368f24e11f0860bdd43882da6677de25 Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 16 May 2024 09:19:59 +0100 Subject: [PATCH 081/104] Windows: add extra debugging messages to handles plugin, ref #1146 --- .../framework/plugins/windows/handles.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 15f5f69df..a19e7a397 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -25,7 +25,7 @@ class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -145,6 +145,9 @@ class Handles(interfaces.plugins.PluginInterface): if self._sar_value is None: if not has_capstone: + vollog.debug( + "capstone module is missing, unable to create disassembly of ObpCaptureHandleInformationEx" + ) return None kernel = self.context.modules[self.config["kernel"]] @@ -159,28 +162,46 @@ class Handles(interfaces.plugins.PluginInterface): try: func_addr = ntkrnlmp.get_symbol("ObpCaptureHandleInformationEx").address except exceptions.SymbolError: + vollog.debug("Unable to locate ObpCaptureHandleInformationEx symbol") return None try: + func_addr_to_read = kvo + func_addr + num_bytes_to_read = 0x200 + vollog.debug( + f"ObpCaptureHandleInformationEx symbol located at {hex(func_addr_to_read)}" + ) data = self.context.layers.read( - virtual_layer_name, kvo + func_addr, 0x200 + virtual_layer_name, func_addr_to_read, num_bytes_to_read ) except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}" + ) return None md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) + instruction_count = 0 for address, size, mnemonic, op_str in md.disasm_lite( data, kvo + func_addr ): # print("{} {} {} {}".format(address, size, mnemonic, op_str)) - + instruction_count += 1 if mnemonic.startswith("sar"): # if we don't want to parse op strings, we can disasm the # single sar instruction again, but we use disasm_lite for speed self._sar_value = int(op_str.split(",")[1].strip(), 16) + vollog.debug( + f"SAR located at {hex(address)} with value of {hex(self._sar_value)}" + ) break + if self._sar_value is None: + vollog.debug( + f"Failed to to locate SAR value having parsed {instruction_count} instructions" + ) + return self._sar_value @classmethod From c2b2321622ad1c170e10cf847b566796842a8020 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:09:14 -0500 Subject: [PATCH 082/104] Add a new getcellroutine plugin that reports hooked GetCellRoutine handlers of memory mapped Windows registry hives --- .../windows/registry/getcellroutine.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 volatility3/framework/plugins/windows/registry/getcellroutine.py diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py new file mode 100644 index 000000000..08523d2f1 --- /dev/null +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -0,0 +1,97 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import ssdt +from volatility3.plugins.windows.registry import hivelist + +vollog = logging.getLogger(__name__) + +class GetCellRoutine(interfaces.plugins.PluginInterface): + """ Reports registry hives with a hooked GetCellRoutine handler """ + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), + ] + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + collection = ssdt.SSDT.build_module_collection( + self.context, kernel.layer_name, kernel.symbol_table_name + ) + + # walk each hive and validate that the GetCellRoutine handler + # is inside of the kernel (ntoskrnl) + for hive_object in hivelist.HiveList.list_hives( + context=self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name + ): + hive = hive_object.hive + + try: + cellroutine = hive.GetCellRoutine + except exceptions.InvalidAddressException: + continue + + module_symbols = list( + collection.get_module_symbols_by_absolute_location(cellroutine) + ) + + if module_symbols: + for module_name, _ in module_symbols: + # GetCellRoutine handlers should only be in the kernel + if module_name not in constants.windows.KERNEL_MODULE_NAMES: + yield ( + 0, + ( + format_hints.Hex(hive.vol.offset), + hive_object.get_name() or "", + module_name, + format_hints.Hex(cellroutine) + ) + ) + # Doesn't map to any module... + else: + yield ( + 0, + ( + format_hints.Hex(hive.vol.offset), + hive_object.get_name() or "", + renderers.NotAvailableValue(), + format_hints.Hex(cellroutine) + ) + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Hive Offset", renderers.format_hints.Hex), + ("Hive Name", str), + ("GetCellRoutine Module", str), + ("GetCellRoutine Handler", renderers.format_hints.Hex) + ], + self._generator(), + ) From 47eb42204e19c482ff332ce9bb36fcbe54fa49d3 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:21:03 -0500 Subject: [PATCH 083/104] Fixes for black --- .../plugins/windows/registry/getcellroutine.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index 08523d2f1..2cbb87565 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -14,7 +14,7 @@ from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) class GetCellRoutine(interfaces.plugins.PluginInterface): - """ Reports registry hives with a hooked GetCellRoutine handler """ + """Reports registry hives with a hooked GetCellRoutine handler""" _required_framework_version = (2, 0, 0) @@ -32,7 +32,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) ), - ] + ] def _generator(self): kernel = self.context.modules[self.config["kernel"]] @@ -47,7 +47,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): context=self.context, base_config_path=self.config_path, layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name + symbol_table=kernel.symbol_table_name, ): hive = hive_object.hive @@ -70,7 +70,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): format_hints.Hex(hive.vol.offset), hive_object.get_name() or "", module_name, - format_hints.Hex(cellroutine) + format_hints.Hex(cellroutine), ) ) # Doesn't map to any module... @@ -81,7 +81,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): format_hints.Hex(hive.vol.offset), hive_object.get_name() or "", renderers.NotAvailableValue(), - format_hints.Hex(cellroutine) + format_hints.Hex(cellroutine), ) ) From 4ed7d40ecb61f1d4e671f3323a1b075ad2501bfc Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:23:21 -0500 Subject: [PATCH 084/104] Fixes for black --- .../framework/plugins/windows/registry/getcellroutine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index 2cbb87565..98b3c2517 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -71,7 +71,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): hive_object.get_name() or "", module_name, format_hints.Hex(cellroutine), - ) + ), ) # Doesn't map to any module... else: @@ -82,7 +82,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): hive_object.get_name() or "", renderers.NotAvailableValue(), format_hints.Hex(cellroutine), - ) + ), ) def run(self): @@ -91,7 +91,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): ("Hive Offset", renderers.format_hints.Hex), ("Hive Name", str), ("GetCellRoutine Module", str), - ("GetCellRoutine Handler", renderers.format_hints.Hex) + ("GetCellRoutine Handler", renderers.format_hints.Hex), ], self._generator(), ) From bc8666b64a4b722918879eda5efcadeab633bfd2 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:24:53 -0500 Subject: [PATCH 085/104] Fixes for black --- volatility3/framework/plugins/windows/registry/getcellroutine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index 98b3c2517..ed54a135d 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -13,6 +13,7 @@ from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) + class GetCellRoutine(interfaces.plugins.PluginInterface): """Reports registry hives with a hooked GetCellRoutine handler""" From 901b0fd6baeaf9779a7111172b579f13688e8ec8 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 15:11:06 -0500 Subject: [PATCH 086/104] Fix year in header --- .../framework/plugins/windows/registry/getcellroutine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index ed54a135d..200a45a82 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # From 3eab0671b48818aeb4af1873214bbe506fd7b7de Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 20 May 2024 20:47:12 +0100 Subject: [PATCH 087/104] Generic: Add vmscan plugin --- volatility3/framework/plugins/vmscan.py | 217 ++++++++++++++++++ .../generic/vmcs/haswell-architecture.json | 131 +++++++++++ .../generic/vmcs/skylake-architecture.json | 131 +++++++++++ 3 files changed, 479 insertions(+) create mode 100644 volatility3/framework/plugins/vmscan.py create mode 100644 volatility3/symbols/generic/vmcs/haswell-architecture.json create mode 100644 volatility3/symbols/generic/vmcs/skylake-architecture.json diff --git a/volatility3/framework/plugins/vmscan.py b/volatility3/framework/plugins/vmscan.py new file mode 100644 index 000000000..9d82ba214 --- /dev/null +++ b/volatility3/framework/plugins/vmscan.py @@ -0,0 +1,217 @@ +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import enum +import logging +import os +import struct +from typing import Dict, List + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import configuration, plugins +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed + +vollog = logging.getLogger(__name__) + + +class VMCSTest(enum.IntFlag): + VMCS_ABORT_INVALID = enum.auto() + VMCS_LINK_PTR_IS_NOT_FS = enum.auto() + VMCS_HOST_CR4_NO_VTX = enum.auto() + VMCS_CR3_IS_ZERO = enum.auto() + VMCS_GUEST_CR4_RESERVED = enum.auto() + + +class PageStartScanner(interfaces.layers.ScannerInterface): + def __init__(self, signatures: List[bytes], page_size: int = 0x1000): + super().__init__() + if not len(signatures): + raise ValueError("No signatures passed to constructor") + self._siglen = len(signatures[0]) + for item in signatures: + if len(item) != self._siglen: + raise ValueError( + "Signatures of different lengths passed to PageStartScanner" + ) + self._signatures = signatures + self._page_size = page_size + + def __call__(self, data: bytes, data_offset: int): + """Scans only the start of every page, to see whether a signature is present or not""" + for page_start in range( + data_offset % self._page_size, len(data), self._page_size + ): + if data[page_start : page_start + self._siglen] in self._signatures: + yield ( + page_start + data_offset, + data[page_start : page_start + self._siglen], + ) + + +class Vmscan(plugins.PluginInterface): + """Scans for Intel VT-d structues and generates VM volatility configs for them""" + + _required_framework_version = (2, 2, 0) + _version = (1, 0, 0) + + STRICTLY_REQUIRED_TESTS = { + VMCSTest.VMCS_ABORT_INVALID, + VMCSTest.VMCS_LINK_PTR_IS_NOT_FS, + VMCSTest.VMCS_HOST_CR4_NO_VTX, + } + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.TranslationLayerRequirement( + name="primary", description="Physical base memory layer" + ), + requirements.IntRequirement( + name="log-threshold", + description="Number of criteria failed to log to debug output", + default=2, + optional=True, + ), + ] + + # Scan for VMCS structures based on the known VMCS structures + # found in symbols/vmcs directory + + def _gather_vmcs_structures( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[bytes, str]: + """Enumerate all JSON files containing VMCS information and return the structures + Signatures can be generated using data extracted using the vmcs_layout tool at + https://github.com/google/rekall/tree/master/tools/linux/vmcs_layout + + Args: + context: The volatility context to work against + config_path: The location to store symbol table configurations under + + Returns: + A dictionary of pattern bytes to the string representation of the architecture + """ + filenames = intermed.IntermediateSymbolTable.file_symbol_url( + os.path.join("generic", "vmcs") + ) + table_names = [] + for filename in filenames: + base_name = os.path.basename(filename).split(".")[0] + table_name = intermed.IntermediateSymbolTable.create( + context, + configuration.path_join(config_path, "vmcs"), + os.path.join("generic", "vmcs"), + filename=base_name, + ) + table_names.append(table_name) + + result = {} + for table_name in table_names: + symbol_table = context.symbol_space[table_name] + revision_id = struct.pack( + " List[str]: + """Runs tests to verify whether a block of data is a VMCS page + Some tests based on the Hypervisor Memory Forensics paper by + Mariano Graziano, Andrea Lanzi and Davide Balzarotti + + Args: + context: The volatility context to be used for this call + vmcs: The instantiated VMCS object to verify + + Returns: + The list of failed criteria that the VMCS did not meet + """ + + # The VMCS should have been constructed on the physical layer (even a nested VMCS) + physical_layer_name = vmcs.vol.layer_name + + failed_tests: VMCSTest = VMCSTest(0) + # The abort field must be valid (generally 0, although other abort codes may exist) + if context.layers[physical_layer_name].read(vmcs.vol.offset + 4, 4) not in [ + b"\x00\x00\x00\x00" + ]: + failed_tests |= VMCSTest.VMCS_ABORT_INVALID + # The vmcs link pointer is supposed to always be set + if vmcs.vmcs_link_ptr != 0xFFFFFFFFFFFFFFFF: + failed_tests |= VMCSTest.VMCS_LINK_PTR_IS_NOT_FS + # To have a VMCS the host needs the VTx bit set in CR4, this can false positive often when all bits are set + if (vmcs.host_cr4 & 1 << 13) == 0: + failed_tests |= VMCSTest.VMCS_HOST_CR4_NO_VTX + # The guest CR3 is *exceptionally* unlikely to be 0 and the guest cr4 is likely to have some bits unset + if (vmcs.guest_cr3 == 0) or (vmcs.host_cr3 == 0): + failed_tests |= VMCSTest.VMCS_CR3_IS_ZERO + # CR4 registers have certain bits reserved that should not be set + if vmcs.guest_cr4 & 0xFFFFFFFFFF889000: + failed_tests |= VMCSTest.VMCS_GUEST_CR4_RESERVED + + if failed_tests and failed_tests.name: + failed_list = failed_tests.name.split("|") + return failed_list + + return [] + + def _generator(self): + # Gather VMCS structures + structures = self._gather_vmcs_structures(self.context, self.config_path) + # Scan memory for them + layer = self.context.layers[self.config["primary"]] + + # Try to move down to the highest physical layer + if layer.config.get("memory_layer"): + layer = self.context.layers[layer.config["memory_layer"]] + + # Run the scan + for offset, match in layer.scan( + self.context, + PageStartScanner(list(structures.keys())), + self._progress_callback, + ): + try: + vmcs = self.context.object( + structures[match] + constants.BANG + "_VMCS", + layer.name, + offset=offset, + ) + failed_list = self._verify_vmcs_page(self.context, vmcs) + if not failed_list: + yield ( + 0, + ( + structures[match], + format_hints.Hex(vmcs.vol.offset), + format_hints.Hex(vmcs.ept), + format_hints.Hex(vmcs.guest_cr3), + ), + ) + if len(failed_list) <= self.config["log-threshold"]: + vollog.debug( + f"Potential {structures[match]} VMCS found at {vmcs.vol.offset:x} with failed criteria: {failed_list}" + ) + except (exceptions.InvalidAddressException, AttributeError): + # Not what we're looking for + continue + + def run(self): + return renderers.TreeGrid( + [ + ("Architecture", str), + ("VMCS Physical offset", format_hints.Hex), + ("EPT", format_hints.Hex), + ("Guest CR3", format_hints.Hex), + ], + self._generator(), + ) diff --git a/volatility3/symbols/generic/vmcs/haswell-architecture.json b/volatility3/symbols/generic/vmcs/haswell-architecture.json new file mode 100644 index 000000000..33b2e55d8 --- /dev/null +++ b/volatility3/symbols/generic/vmcs/haswell-architecture.json @@ -0,0 +1,131 @@ +{ + "base_types": { + "pointer": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned char": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + } + }, + "enums": {}, + "metadata": { + "format": "6.1.0", + "producer": { + "datetime": "2021-07-31T17:37:28.313255", + "name": "vmextract-by-hand", + "version": "0.0.1" + } + }, + "symbols": { + "revision_id": { + "address": 0, + "constant_data": "MTg=" + } + }, + "user_types": { + "_VMCS": { + "fields": { + "ept": { + "offset": 320, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "executive_vmcs_ptr": { + "offset": 208, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr3": { + "offset": 528, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr4": { + "offset": 536, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_pdpte": { + "offset": 544, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "unsigned long long" + } + } + }, + "guest_physical_addr": { + "offset": 328, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr3": { + "offset": 816, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr4": { + "offset": 824, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vmcs_link_ptr": { + "offset": 248, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vpid": { + "offset": 206, + "type": { + "kind": "struct", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 4096 + } + } +} \ No newline at end of file diff --git a/volatility3/symbols/generic/vmcs/skylake-architecture.json b/volatility3/symbols/generic/vmcs/skylake-architecture.json new file mode 100644 index 000000000..78b110d94 --- /dev/null +++ b/volatility3/symbols/generic/vmcs/skylake-architecture.json @@ -0,0 +1,131 @@ +{ + "base_types": { + "pointer": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned char": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + } + }, + "enums": {}, + "metadata": { + "format": "6.1.0", + "producer": { + "datetime": "2021-07-16T16:21:01.062423", + "name": "vmextract-by-hand", + "version": "0.0.1" + } + }, + "symbols": { + "revision_id": { + "address": 0, + "constant_data": "NA==" + } + }, + "user_types": { + "_VMCS": { + "fields": { + "ept": { + "offset": 320, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "executive_vmcs_ptr": { + "offset": 208, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr3": { + "offset": 528, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr4": { + "offset": 536, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_pdpte": { + "offset": 544, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "unsigned long long" + } + } + }, + "guest_physical_addr": { + "offset": 328, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr3": { + "offset": 816, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr4": { + "offset": 824, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vmcs_link_ptr": { + "offset": 248, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vpid": { + "offset": 206, + "type": { + "kind": "struct", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 4096 + } + } +} From c5a45f91fc4968a46389781a4e21c34c771578a2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 20 May 2024 21:09:00 +0100 Subject: [PATCH 088/104] Linux: Replace uses of specific types with the more generic pointer --- volatility3/framework/automagic/linux.py | 2 +- volatility3/framework/plugins/linux/kmsg.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2eebcc2dc..7d7c563b6 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -159,7 +159,7 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): # This we get for free aslr_shift = ( - init_task.files.cast("long unsigned int") + init_task.files.cast("pointer") - module.get_symbol("init_files").address ) kaslr_shift = init_task_address - cls.virtual_to_physical_address( diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index c5e0fc302..e26d69543 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -66,7 +66,7 @@ class ABCKmsg(ABC): self._config = config self.vmlinux = context.modules[self._config["kernel"]] self.layer_name = self.vmlinux.layer_name # type: ignore - self.long_unsigned_int_size = self.vmlinux.get_type("long unsigned int").size + self.long_unsigned_int_size = self.vmlinux.get_type("pointer").size @classmethod def run_all( From c6d207d170c50ad460de4b9970e74405bd0e1d0d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 21 May 2024 09:06:01 +0100 Subject: [PATCH 089/104] Generic: Fix code scanning issue with vmscan --- volatility3/framework/plugins/vmscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/vmscan.py b/volatility3/framework/plugins/vmscan.py index 9d82ba214..64377d7d8 100644 --- a/volatility3/framework/plugins/vmscan.py +++ b/volatility3/framework/plugins/vmscan.py @@ -120,7 +120,7 @@ class Vmscan(plugins.PluginInterface): @classmethod def _verify_vmcs_page( - self, + cls, context: interfaces.context.ContextInterface, vmcs: interfaces.objects.ObjectInterface, ) -> List[str]: From 08493e9c3affda195c7c83a27c9e80bb020fe4c2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 21 May 2024 23:13:31 +0100 Subject: [PATCH 090/104] Automagic: Do some slight gymnastics to get the true value of the pointer --- volatility3/framework/automagic/linux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 7d7c563b6..52a73f45a 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -159,7 +159,10 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): # This we get for free aslr_shift = ( - init_task.files.cast("pointer") + int.from_bytes( + init_task.files.cast("bytes", length=init_task.files.vol.size), + byteorder=init_task.files.vol.data_format.byteorder, + ) - module.get_symbol("init_files").address ) kaslr_shift = init_task_address - cls.virtual_to_physical_address( From 57b3a99a1e42bfdb102f5038469999a815880330 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 May 2024 20:40:36 +0100 Subject: [PATCH 091/104] Infra: Upgrade deprecated githubs actions --- .github/workflows/build-pypi.yml | 6 +++--- .github/workflows/install.yml | 2 +- .github/workflows/test.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index f21898971..5a90364cd 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -20,9 +20,9 @@ jobs: matrix: python-version: ["3.7"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -37,7 +37,7 @@ jobs: python setup.py bdist_wheel - name: Archive dist - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: volatility3-pypi path: | diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index cc2a7fd3e..917685187 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b1a9dd31b..7ce6b13bb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,9 +8,9 @@ jobs: matrix: python-version: ["3.7"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From 35122df27d6d65c661c5fc43508e61609f262714 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 May 2024 20:42:25 +0100 Subject: [PATCH 092/104] Infra: Upgrade deprecated githubs actions 2 --- .github/workflows/black.yml | 2 +- .github/workflows/install.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 5f4523072..5adab3259 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -6,7 +6,7 @@ jobs: lint: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: options: "--check --diff --verbose" diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 917685187..e13161085 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -10,10 +10,10 @@ jobs: host: [ ubuntu-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 771ed10b44573a7f8baa32822f3bc524195fe0c9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 May 2024 20:44:04 +0100 Subject: [PATCH 093/104] Core: Bump to 2.7.1 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3c5e0eb2f..6963867f7 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 7 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From a72062f2889fdfe5d03e3974e4bc314a35e85700 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jun 2024 17:20:47 +1000 Subject: [PATCH 094/104] Revert "LayerWriter plugin: Fix --output argument. It's referenced in the code but never mentioned as a requirement" This reverts commit 6d22347ce6dd0152746564a2d77022ca1b4d9045. --- volatility3/framework/plugins/layerwriter.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 6a06cc607..319e4fd7a 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -38,11 +38,6 @@ class LayerWriter(plugins.PluginInterface): default=False, optional=True, ), - requirements.StringRequirement( - name="output", - description="Output filename", - optional=True, - ), requirements.ListRequirement( name="layers", element_type=str, From ab84070df30ddfaccc2e304e26fb7815a5555444 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jun 2024 17:25:05 +1000 Subject: [PATCH 095/104] Revert "Store the real/final output filename so that we can notify it correctly to the user" This reverts commit d7aae3a9828ba03bd5531aa385724a954e48501a. --- volatility3/cli/__init__.py | 4 ++-- volatility3/framework/plugins/layerwriter.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 9873791e3..c0eed20ee 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -790,8 +790,8 @@ class CommandLine: return None self._file.close() - self._output_filename = self._get_final_filename() - os.rename(self._name, self._output_filename) + output_filename = self._get_final_filename() + os.rename(self._name, output_filename) if direct: return CLIDirectFileHandler diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 319e4fd7a..61c3118b5 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -115,7 +115,6 @@ class LayerWriter(plugins.PluginInterface): progress_callback=self._progress_callback, ) file_handle.close() - output_name = file_handle._output_filename except IOError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) From 77fb0b7b26c2c52140380a2ce9f1a896a84c258a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jun 2024 18:37:38 +1000 Subject: [PATCH 096/104] Update the output filename (preferred_filename) so that we can notify it correctly to the user --- volatility3/cli/__init__.py | 10 +++++++++- volatility3/framework/plugins/layerwriter.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index c0eed20ee..6b17edac0 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -789,8 +789,16 @@ class CommandLine: if self._file.closed: return None - self._file.close() output_filename = self._get_final_filename() + + # Update the filename, which may have changed if a file with + # the same name already existed. This needs to be done before + # closing the file, otherwise FileHandlerInterface will raise + # an exception. Also, the preferred_filename setter only allows + # a specific set of characters, where '/' is not in that list + self.preferred_filename = os.path.basename(output_filename) + + self._file.close() os.rename(self._name, output_filename) if direct: diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 61c3118b5..24149a390 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -115,6 +115,10 @@ class LayerWriter(plugins.PluginInterface): progress_callback=self._progress_callback, ) file_handle.close() + + # Update the filename, which may have changed if a file + # with the same name already existed. + output_name = file_handle.preferred_filename except IOError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) From e4aac0c97634487db735f762bdc2f449fa8cd5ae Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 10 Jun 2024 11:30:15 +0100 Subject: [PATCH 097/104] Windows: ldrmodules update exception handling to InvalidAddressException --- .../framework/plugins/windows/ldrmodules.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 4c8456fa9..ffd84ea4a 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -1,3 +1,9 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -5,12 +11,14 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import pslist, vadinfo +vollog = logging.getLogger(__name__) + class LdrModules(interfaces.plugins.PluginInterface): """Lists the loaded modules in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -71,7 +79,11 @@ class LdrModules(interfaces.plugins.PluginInterface): # Filter out VADs that do not start with a MZ header if dos_header.e_magic != 0x5A4D: continue - except exceptions.PagedInvalidAddressException: + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping vad at {hex(dos_header.vol.offset)} due to InvalidAddressException", + ) continue mapped_files[vad.get_start()] = vad.get_file_name() From 0ee3573be239966a888df644d2aca3a5dd18b02d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jun 2024 23:14:45 +0100 Subject: [PATCH 098/104] Revert "Intel layer: Fix. This if statement will never be executed unless "minimum_address > maximum_address" which doesn't make sense to me." This reverts commit e5a5b895771b655d21c36689c33a534034c31e36. --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 8589cdbb6..75e561b33 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -180,7 +180,7 @@ class Intel(linear.LinearlyMappedLayer): position = self._initial_position entry = self._initial_entry - if not (self.minimum_address <= offset <= self.maximum_address): + if self.minimum_address > offset > self.maximum_address: raise exceptions.PagedInvalidAddressException( self.name, offset, From 04ca0214eb264268bd368401f8d5eaa940f2524e Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 13 Jun 2024 15:24:32 -0500 Subject: [PATCH 099/104] Prevent backtrace in ADS scanning when contents cannot be recovered. Fix missing parantheses as well as format_hints call on bad value --- volatility3/framework/plugins/windows/mftscan.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 85c072037..1b1a1b45e 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -264,9 +264,10 @@ class ADS(interfaces.plugins.PluginInterface): disasm = interfaces.renderers.Disassembly( content, 0, architecture.lower() ) + content = format_hints.HexBytes(content) else: - content = renderers.NotAvailableValue - disasm = interfaces.renderers.BaseAbsentValue + content = renderers.NotAvailableValue() + disasm = interfaces.renderers.BaseAbsentValue() yield 0, ( format_hints.Hex(attr_data.vol.offset), @@ -275,7 +276,7 @@ class ADS(interfaces.plugins.PluginInterface): attr.Attr_Header.AttrType.lookup(), file_name, ads_name, - format_hints.HexBytes(content), + content, disasm, ) else: From 59f6a050688c30e2b05fec2b0efe54f57e2f46f3 Mon Sep 17 00:00:00 2001 From: atcuno Date: Fri, 14 Jun 2024 16:45:54 -0500 Subject: [PATCH 100/104] The notes variable is not reset for each VAD, allowing bleed through of a previously set note value to VADs enumerated afterwards --- volatility3/framework/plugins/windows/malfind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 8c0cb68ac..079274d69 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -155,12 +155,12 @@ class Malfind(interfaces.plugins.PluginInterface): for proc in procs: # by default, "Notes" column will be set to N/A - notes = renderers.NotApplicableValue() process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( self.context, kernel.layer_name, kernel.symbol_table_name, proc ): + notes = renderers.NotApplicableValue() # Check for unique headers and update "Notes" column if criteria is met if data[0:2] in refined_criteria: notes = refined_criteria[data[0:2]] From a68a4826329dbd44e95d4e173ea467be24dbc99b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 20 Jun 2024 04:00:20 +0200 Subject: [PATCH 101/104] fix #509 and simplify logic in _load_segments --- volatility3/framework/layers/crash.py | 105 +++++++++++++++----------- 1 file changed, 62 insertions(+), 43 deletions(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 042b18ddc..2d0f37008 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -96,12 +96,13 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): def get_summary_header(self) -> interfaces.objects.ObjectInterface: return self.context.object( self._crash_common_table_name + constants.BANG + "_SUMMARY_DUMP", - offset=0x1000 * self.headerpages, + offset=self._page_size * self.headerpages, layer_name=self._base_layer, ) def _load_segments(self) -> None: - """Loads up the segments from the meta_layer.""" + """Loads up the segments from the meta_layer. + A segment is a set of contiguous memory pages.""" segments = [] @@ -119,70 +120,88 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): for run in header.PhysicalMemoryBlockBuffer.Run: segments.append( ( - run.BasePage * 0x1000, - offset * 0x1000, - run.PageCount * 0x1000, - run.PageCount * 0x1000, + run.BasePage * self._page_size, + offset * self._page_size, + run.PageCount * self._page_size, + run.PageCount * self._page_size, ) ) offset += run.PageCount elif self.dump_type == 0x05: summary_header = self.get_summary_header() - first_bit = None # First bit in a run - first_offset = 0 # File offset of first bit - last_bit_seen = 0 # Most recent bit processed - offset = summary_header.HeaderSize # Size of file headers - buffer_char = summary_header.get_buffer_char() - buffer_long = summary_header.get_buffer_long() - - for outer_index in range(0, ((summary_header.BitmapSize + 31) // 32)): - if buffer_long[outer_index] == 0: - if first_bit is not None: - last_bit = ((outer_index - 1) * 32) + 31 - segment_length = (last_bit - first_bit + 1) * 0x1000 + seg_first_bit = None # First bit in a run + seg_first_offset = 0 # File offset of first bit + offset = ( + summary_header.HeaderSize + ) # Offset to the start of actual memory dump + ulong_bitmap_array = summary_header.get_buffer_long() + # outer_index points to a 32 bits array inside a list of arrays, + # each bit indicating a page mapping state + for outer_index in range(0, ulong_bitmap_array.vol.count): + ulong_bitmap = ulong_bitmap_array[outer_index] + # All pages in this 32 bits array are mapped (speedup iteration process) + if ulong_bitmap == 0xFFFFFFFF: + # New segment + if seg_first_bit is None: + seg_first_offset = offset + seg_first_bit = outer_index * 32 + offset += 32 * self._page_size + # No pages in this 32 bits array are mapped (speedup iteration process) + elif ulong_bitmap == 0: + # End of segment + if seg_first_bit is not None: + last_bit = (outer_index - 1) * 32 + 31 + segment_length = ( + last_bit - seg_first_bit + 1 + ) * self._page_size segments.append( ( - first_bit * 0x1000, - first_offset, + seg_first_bit * self._page_size, + seg_first_offset, segment_length, segment_length, ) ) - first_bit = None - elif buffer_long[outer_index] == 0xFFFFFFFF: - if first_bit is None: - first_offset = offset - first_bit = outer_index * 32 - offset = offset + (32 * 0x1000) + seg_first_bit = None + # Some pages in this 32 bits array are mapped and some aren't else: - for inner_index in range(0, 32): - bit_addr = outer_index * 32 + inner_index - if (buffer_char[bit_addr >> 3] >> (bit_addr & 0x7)) & 1: - if first_bit is None: - first_offset = offset - first_bit = bit_addr - offset = offset + 0x1000 + for inner_bit_position in range(0, 32): + current_bit = outer_index * 32 + inner_bit_position + page_mapped = ulong_bitmap & (1 << inner_bit_position) + if page_mapped: + # New segment + if seg_first_bit is None: + seg_first_offset = offset + seg_first_bit = current_bit + offset += self._page_size else: - if first_bit is not None: + # End of segment + if seg_first_bit is not None: segment_length = ( - (bit_addr - 1) - first_bit + 1 - ) * 0x1000 + current_bit - 1 - seg_first_bit + 1 + ) * self._page_size segments.append( ( - first_bit * 0x1000, - first_offset, + seg_first_bit * self._page_size, + seg_first_offset, segment_length, segment_length, ) ) - first_bit = None - last_bit_seen = (outer_index * 32) + 31 + seg_first_bit = None + else: + last_bit_seen = outer_index * 32 + 31 - if first_bit is not None: - segment_length = (last_bit_seen - first_bit + 1) * 0x1000 + if seg_first_bit is not None: + segment_length = (last_bit_seen - seg_first_bit + 1) * self._page_size segments.append( - (first_bit * 0x1000, first_offset, segment_length, segment_length) + ( + seg_first_bit * self._page_size, + seg_first_offset, + segment_length, + segment_length, + ) ) else: vollog.log( From c688744f66ae6af49123a530dd45abc29ed9abba Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 20 Jun 2024 04:16:32 +0200 Subject: [PATCH 102/104] revert useless for-else --- volatility3/framework/layers/crash.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 2d0f37008..3cfc0a25b 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -190,7 +190,6 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): ) ) seg_first_bit = None - else: last_bit_seen = outer_index * 32 + 31 if seg_first_bit is not None: From 898c0844c464f87ab7bbd1cae6756a69b2a8048e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 21 Jun 2024 16:15:17 -0500 Subject: [PATCH 103/104] Windows: fixes scanner bug for versions < win10 This commit fixes a bug where the `layer_name` gets discarded when constructing objects. Previously, it was assumed that we would not want to construct an object for a module with a layer_name different from that of the module. However, because we switch to scanning the memory layer on samples where the version is < 10, but still construct kernel executive objects based on the result of the memory layer scan, we actually do sometimes need to specify a different layer. --- volatility3/framework/contexts/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 73868a58f..4c169728c 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -256,12 +256,13 @@ class Module(interfaces.context.ModuleInterface): if not absolute: offset += self._offset - # Ensure we don't use a layer_name other than the module's, why would anyone do that? - if "layer_name" in kwargs: - del kwargs["layer_name"] + # We have to allow using an alternative layer name due to pool scanners switching + # to the memory layer for scanning samples prior to Windows 10. + layer_name = kwargs.pop("layer_name", self._layer_name) + return self._context.object( object_type=object_type, - layer_name=self._layer_name, + layer_name=layer_name, offset=offset, native_layer_name=native_layer_name or self._native_layer_name, **kwargs, From 7ece5fb1bf2f27d2731d09ed8329fb20552c713c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 23 Jun 2024 00:16:23 +0100 Subject: [PATCH 104/104] Windows: Improve Virtmap error messages slightly --- volatility3/framework/plugins/windows/virtmap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/virtmap.py b/volatility3/framework/plugins/windows/virtmap.py index 5190bec8d..3f3f270e2 100644 --- a/volatility3/framework/plugins/windows/virtmap.py +++ b/volatility3/framework/plugins/windows/virtmap.py @@ -78,7 +78,7 @@ class VirtMap(interfaces.plugins.PluginInterface): ) else: raise exceptions.SymbolError( - None, module.name, "Required structures not found" + "SystemVaRegions", module.name, "Required structures not found" ) elif module.has_symbol("MiSystemVaType"): system_range_start = module.object( @@ -99,7 +99,7 @@ class VirtMap(interfaces.plugins.PluginInterface): ) else: raise exceptions.SymbolError( - None, module.name, "Required structures not found" + "MiVisibleState", module.name, "Required structures not found" ) return result