From 0b33a79dc6465884105a60cb3d2054c61885a2eb Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Wed, 2 Nov 2022 21:10:51 +0000 Subject: [PATCH 01/76] Report IRP entries that point inside a hidden module. This is a common rootkit technique. --- volatility3/framework/plugins/windows/driverirp.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index 7f9bc6b08..4013bdb8f 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -48,7 +48,10 @@ class DriverIrp(interfaces.plugins.PluginInterface): for i, address in enumerate(driver.MajorFunction): module_symbols = collection.get_module_symbols_by_absolute_location(address) + module_found = False + for module_name, symbol_generator in module_symbols: + module_found = True symbols_found = False for symbol in symbol_generator: @@ -60,6 +63,11 @@ class DriverIrp(interfaces.plugins.PluginInterface): yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], format_hints.Hex(address), module_name, renderers.NotAvailableValue())) + if not module_found: + yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], + format_hints.Hex(address), renderers.NotAvailableValue(), renderers.NotAvailableValue())) + + def run(self): return renderers.TreeGrid([ From 7469872c8bfcd8d7a84637ede54180008e624b0a Mon Sep 17 00:00:00 2001 From: RuBublik Date: Wed, 17 May 2023 21:41:37 +0300 Subject: [PATCH 02/76] added 'PoolConstraint' of Thread objects to 'PoolScanner.default_constraints' as part of adding support for thread pool tag scanning --- .../framework/plugins/windows/poolscanner.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index e131c5f78..028241bb8 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -224,6 +224,20 @@ class PoolScanner(plugins.PluginInterface): size=(600, None), page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), + # threads on windows before windows8 + PoolConstraint(b'Thr\xe5', # -> “protected” allocation, MSB is set. + type_name = symbol_table + constants.BANG + "_ETHREAD", + object_type="Thread", + size = (600, None), # -> 0x0258 - size of strcut in win5.1 + page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + ), + # threads on windows starting with windows8 + PoolConstraint(b'Thre', + type_name = symbol_table + constants.BANG + "_ETHREAD", + object_type="Thread", + size = (600, None), # -> 0x0258 - size of strcut in win5.1 + page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + ), # files on windows before windows 8 PoolConstraint( b"Fil\xe5", From 550d1096ebe2c2d1f22a6c6446495051d54e84ec Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sat, 20 May 2023 14:20:15 +0300 Subject: [PATCH 03/76] temporary fix to ETHREAD class, add 'is_valid' method --- .../framework/symbols/windows/extensions/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ba00a4053..cfaa3aced 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -492,9 +492,13 @@ class KMUTANT(objects.StructType, pool.ExecutiveObject): return header.NameInfo.Name.String # type: ignore -class ETHREAD(objects.StructType): +class ETHREAD(objects.StructType, pool.ExecutiveObject): """A class for executive thread objects.""" + def is_valid(self) -> bool: + """Determine if the object is valid.""" + return True # temporary, need to implement validation later. + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 0fd475e10bb9644c28abef36c8f4c7dabec318d6 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sat, 20 May 2023 15:49:09 +0300 Subject: [PATCH 04/76] added permanent implementation for 'ETHREAD.is_valid' --- .../symbols/windows/extensions/__init__.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index cfaa3aced..e17128897 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -497,7 +497,30 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): def is_valid(self) -> bool: """Determine if the object is valid.""" - return True # temporary, need to implement validation later. + + try: + + # validation by thread creation time: + ctime = self.get_create_time() + if not isinstance(ctime, datetime.datetime): + return False + + # validation by parent process: + own_proc = self.owning_process() + # return own_proc.is_valid() + if own_proc.UniqueProcessId % 4 != 0: # NT pids are divisible by 4 + return False + + # passed all valitations + return True + except: + return False + + def get_create_time(self): + return conversion.wintime_to_datetime(self.CreateTime.QuadPart) + + def get_exit_time(self): + return conversion.wintime_to_datetime(self.ExitTime.QuadPart) def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 601870829e33c32688fbecccede7423f1e3b7839 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:12:46 +0300 Subject: [PATCH 05/76] twicked thread constraint for edge cases --- volatility3/framework/plugins/windows/poolscanner.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 028241bb8..ce1015789 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -222,17 +222,21 @@ class PoolScanner(plugins.PluginInterface): type_name=symbol_table + constants.BANG + "_EPROCESS", object_type="Process", size=(600, None), + skip_type_test = True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows before windows8 - PoolConstraint(b'Thr\xe5', # -> “protected” allocation, MSB is set. + PoolConstraint( + b'Thr\xe5', # -> “protected” allocation, MSB is set. type_name = symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", size = (600, None), # -> 0x0258 - size of strcut in win5.1 + skip_type_test = True, page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE ), # threads on windows starting with windows8 - PoolConstraint(b'Thre', + PoolConstraint( + b'Thre', type_name = symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", size = (600, None), # -> 0x0258 - size of strcut in win5.1 From 046c8e4d1e7f3bf9d6857de5601ebd4ab089af6d Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:18:02 +0300 Subject: [PATCH 06/76] fix of is_valid - removed reliace on owning _eprocess and added exclusion for system process (does not have creation time) --- .../symbols/windows/extensions/__init__.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index e17128897..fe4884dbc 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -499,18 +499,21 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): """Determine if the object is valid.""" try: - - # validation by thread creation time: - ctime = self.get_create_time() - if not isinstance(ctime, datetime.datetime): - return False - - # validation by parent process: - own_proc = self.owning_process() - # return own_proc.is_valid() - if own_proc.UniqueProcessId % 4 != 0: # NT pids are divisible by 4 + + # validation by TID: + if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 return False + # validation by PID of parent process: + if self.Cid.UniqueProcess % 4 != 0: + return False + + # validation by thread creation time: + if self.Cid.UniqueProcess != 4: # The System process (PID 4) has no create time + ctime = self.get_create_time() + if not isinstance(ctime, datetime.datetime): + return False + # passed all valitations return True except: From fab818b48d950a1210c77e97a4da09cf24f2bd03 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:24:01 +0300 Subject: [PATCH 07/76] added thrdscan plugin to utilize added support for ethread pool tag scanning --- .../framework/plugins/windows/thrdscan.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 volatility3/framework/plugins/windows/thrdscan.py diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py new file mode 100644 index 000000000..b0b80fe46 --- /dev/null +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -0,0 +1,107 @@ +## +## plugin for testing addition of threads scan support to poolscanner.py +## +import logging +import datetime +from typing import Iterable + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import poolscanner + +vollog = logging.getLogger(__name__) + + +class ThrdScan(interfaces.plugins.PluginInterface): + """Scans for windows threads.""" + + # cuz installed Framework interface version 2 + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), + ] + + @classmethod + def scan_threads( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Scans for threads using the poolscanner module and constraints. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + + Returns: + A list of _ETHREAD objects found by scanning memory for the "Thre" / "Thr\\xE5" pool signatures + """ + + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"Thr\xe5", b"Thre"] + ) + + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): + _constraint, mem_object, _header = result + yield mem_object + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + for ethread in self.scan_threads( + self.context, kernel.layer_name, kernel.symbol_table_name + ): + try: + thread_offset = ethread.vol.offset + owner_proc_pid = ethread.Cid.UniqueProcess + thread_tid = ethread.Cid.UniqueThread + thread_start_addr = ethread.StartAddress + thread_create_time = ethread.get_create_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_exit_time = ethread.get_exit_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + except (ValueError, exceptions.InvalidAddressException): + vollog.debug( + "Thread :{}, invalid address {} in layer {}".format( + thread_tid, thread_start_addr, kernel.layer_name + ) + ) + continue + + yield ( + 0, + ( + hex(format_hints.Hex(thread_offset)), + owner_proc_pid, + thread_tid, + hex(thread_start_addr), + str(thread_create_time) if isinstance(thread_create_time, datetime.datetime) else "", + str(thread_exit_time) if isinstance(thread_exit_time, datetime.datetime) else "" + ) + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", str), + ("PID", int), + ("TID", int), + ("Start Address", str), + ("Create Time", str), + ("Exit Time", str), + ], + self._generator(), + ) \ No newline at end of file From 5749b3ef5283fc36d064a766e5228dc2eef5b402 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 19:22:22 +0300 Subject: [PATCH 08/76] added test for thrdscan plugin --- test/test_volatility.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index aaad615bc..5e41899da 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -189,6 +189,16 @@ def test_windows_svcscan(image, volatility, python): assert rc == 0 +def test_windows_thrdscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.thrdscan.ThrdScan", image, volatility, python) + # find pid 4 (of system process) which starts with lowest tids + assert out.find(b"\t4\t8") != -1 + assert out.find(b"\t4\t12") != -1 + assert out.find(b"\t4\t16") != -1 + #assert out.find(b"this raieses AssertionError") != -1 + assert rc == 0 + + def test_windows_privileges(image, volatility, python): rc, out, err = runvol_plugin( "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"] From 950a76d1e0dfb460215c3b6e5dc1108a0421863f Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 11:17:23 +0300 Subject: [PATCH 09/76] fixed typo --- .../framework/symbols/windows/extensions/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index fe4884dbc..4ad74f61a 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -514,10 +514,11 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False - # passed all valitations - return True - except: + except exceptions.InvalidAddressException: return False + + # passed all validations + return True def get_create_time(self): return conversion.wintime_to_datetime(self.CreateTime.QuadPart) From bef5149ba64e640c39e83774e0fa969893d5ee18 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 11:39:38 +0300 Subject: [PATCH 10/76] changed TreeGrid yielded types to specific simpletypes instead of str --- .../framework/plugins/windows/thrdscan.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index b0b80fe46..8b445991f 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -84,24 +84,24 @@ class ThrdScan(interfaces.plugins.PluginInterface): yield ( 0, ( - hex(format_hints.Hex(thread_offset)), + format_hints.Hex(thread_offset), owner_proc_pid, thread_tid, - hex(thread_start_addr), - str(thread_create_time) if isinstance(thread_create_time, datetime.datetime) else "", - str(thread_exit_time) if isinstance(thread_exit_time, datetime.datetime) else "" + format_hints.Hex(thread_start_addr), + thread_create_time, + thread_exit_time, ) ) def run(self): return renderers.TreeGrid( [ - ("Offset", str), + ("Offset", format_hints.Hex), ("PID", int), ("TID", int), - ("Start Address", str), - ("Create Time", str), - ("Exit Time", str), + ("Start Address", format_hints.Hex), + ("Create Time", datetime.datetime), + ("Exit Time", datetime.datetime), ], self._generator(), ) \ No newline at end of file From c6c501ecb57e1757a510d019a3752af78dda96cc Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:31:25 +0300 Subject: [PATCH 11/76] implemented generate_timeline method in ThrdScan --- .../framework/plugins/windows/thrdscan.py | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 8b445991f..ca3f4fc69 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -9,15 +9,16 @@ from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import poolscanner +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class ThrdScan(interfaces.plugins.PluginInterface): +class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -93,15 +94,40 @@ class ThrdScan(interfaces.plugins.PluginInterface): ) ) + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + row_dict = {} + ( + row_dict["Offset"], + row_dict["PID"], + row_dict["TID"], + row_dict["StartAddress"], + row_dict["CreateTime"], + row_dict["ExitTime"], + ) = row_data + + # Skip threads with no creation time + # - mainly system process threads + if not isinstance(row_dict["CreateTime"], datetime.datetime): + continue + description = (f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})") + + # yield created time, and if there is exit time, yield it too. + yield (description, timeliner.TimeLinerType.CREATED, row_dict["CreateTime"]) + if isinstance(row_dict["ExitTime"], datetime.datetime): + yield (description, timeliner.TimeLinerType.MODIFIED, row_dict["ExitTime"]) + + def run(self): return renderers.TreeGrid( [ ("Offset", format_hints.Hex), ("PID", int), ("TID", int), - ("Start Address", format_hints.Hex), - ("Create Time", datetime.datetime), - ("Exit Time", datetime.datetime), + ("StartAddress", format_hints.Hex), + ("CreateTime", datetime.datetime), + ("ExitTime", datetime.datetime), ], self._generator(), ) \ No newline at end of file From 9bdd249aadb5b60ca5e45c91c396f1c591f9b45e Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:39:12 +0300 Subject: [PATCH 12/76] added _version to ThrdScan --- volatility3/framework/plugins/windows/thrdscan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index ca3f4fc69..cbbe64988 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -19,6 +19,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # cuz installed Framework interface version 2 _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls): From 5072728e897c1489e11f3422ea6974cf542d7e64 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:50:22 +0300 Subject: [PATCH 13/76] formated with black --- .../framework/plugins/windows/thrdscan.py | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index cbbe64988..813883c2d 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -18,7 +18,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """Scans for windows threads.""" # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod @@ -33,7 +33,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) ), ] - + @classmethod def scan_threads( cls, @@ -53,7 +53,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """ constraints = poolscanner.PoolScanner.builtin_constraints( - symbol_table, [b"Thr\xe5", b"Thre"] + symbol_table, [b"Thr\xe5", b"Thre"] ) for result in poolscanner.PoolScanner.generate_pool_scan( @@ -69,12 +69,16 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) self.context, kernel.layer_name, kernel.symbol_table_name ): try: - thread_offset = ethread.vol.offset - owner_proc_pid = ethread.Cid.UniqueProcess - thread_tid = ethread.Cid.UniqueThread - thread_start_addr = ethread.StartAddress - thread_create_time = ethread.get_create_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object - thread_exit_time = ethread.get_exit_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_offset = ethread.vol.offset + owner_proc_pid = ethread.Cid.UniqueProcess + thread_tid = ethread.Cid.UniqueThread + thread_start_addr = ethread.StartAddress + thread_create_time = ( + ethread.get_create_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_exit_time = ( + ethread.get_exit_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object except (ValueError, exceptions.InvalidAddressException): vollog.debug( "Thread :{}, invalid address {} in layer {}".format( @@ -86,13 +90,13 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) yield ( 0, ( - format_hints.Hex(thread_offset), - owner_proc_pid, - thread_tid, - format_hints.Hex(thread_start_addr), - thread_create_time, - thread_exit_time, - ) + format_hints.Hex(thread_offset), + owner_proc_pid, + thread_tid, + format_hints.Hex(thread_start_addr), + thread_create_time, + thread_exit_time, + ), ) def generate_timeline(self): @@ -112,13 +116,16 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # - mainly system process threads if not isinstance(row_dict["CreateTime"], datetime.datetime): continue - description = (f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})") - + description = f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})" + # yield created time, and if there is exit time, yield it too. yield (description, timeliner.TimeLinerType.CREATED, row_dict["CreateTime"]) if isinstance(row_dict["ExitTime"], datetime.datetime): - yield (description, timeliner.TimeLinerType.MODIFIED, row_dict["ExitTime"]) - + yield ( + description, + timeliner.TimeLinerType.MODIFIED, + row_dict["ExitTime"], + ) def run(self): return renderers.TreeGrid( @@ -131,4 +138,4 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ("ExitTime", datetime.datetime), ], self._generator(), - ) \ No newline at end of file + ) From 5d32ca542c8918224f070163f02ff454f0bffffa Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 28 May 2023 20:52:26 +0300 Subject: [PATCH 14/76] fix black formatting --- .../framework/plugins/windows/poolscanner.py | 20 +++++++++---------- .../symbols/windows/extensions/__init__.py | 17 ++++++++-------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index ce1015789..5539e1e84 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -222,25 +222,25 @@ class PoolScanner(plugins.PluginInterface): type_name=symbol_table + constants.BANG + "_EPROCESS", object_type="Process", size=(600, None), - skip_type_test = True, + skip_type_test=True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows before windows8 PoolConstraint( - b'Thr\xe5', # -> “protected” allocation, MSB is set. - type_name = symbol_table + constants.BANG + "_ETHREAD", + b"Thr\xe5", # -> “protected” allocation, MSB is set. + type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size = (600, None), # -> 0x0258 - size of strcut in win5.1 - skip_type_test = True, - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + size=(600, None), # -> 0x0258 - size of strcut in win5.1 + skip_type_test=True, + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows starting with windows8 PoolConstraint( - b'Thre', - type_name = symbol_table + constants.BANG + "_ETHREAD", + b"Thre", + type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size = (600, None), # -> 0x0258 - size of strcut in win5.1 - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + size=(600, None), # -> 0x0258 - size of strcut in win5.1 + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # files on windows before windows 8 PoolConstraint( diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 4ad74f61a..8790f41a7 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -499,17 +499,18 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): """Determine if the object is valid.""" try: - # validation by TID: - if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 + if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 return False - + # validation by PID of parent process: if self.Cid.UniqueProcess % 4 != 0: return False - + # validation by thread creation time: - if self.Cid.UniqueProcess != 4: # The System process (PID 4) has no create time + if ( + self.Cid.UniqueProcess != 4 + ): # The System process (PID 4) has no create time ctime = self.get_create_time() if not isinstance(ctime, datetime.datetime): return False @@ -518,14 +519,14 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return False # passed all validations - return True - + return True + def get_create_time(self): return conversion.wintime_to_datetime(self.CreateTime.QuadPart) def get_exit_time(self): return conversion.wintime_to_datetime(self.ExitTime.QuadPart) - + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 524ff59107ee857c6d4e86697fd6db7f92c05156 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Fri, 7 Jul 2023 20:22:58 +0300 Subject: [PATCH 15/76] bumped MINOR_VERSION to 2.5.2 after changes, and updated dependent thrdscan plugin's required version to this --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3a6b24ea8..09dded076 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,7 +44,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 4 # Number of changes that only add to the interface +VERSION_MINOR = 5 # Number of changes that only add to the interface VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 813883c2d..4afc29cdb 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + # version 2.5.2 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 5, 2) _version = (1, 0, 0) @classmethod From abf1c2e03d67f02ea5c31cc9f4b16029cbbe946d Mon Sep 17 00:00:00 2001 From: RuBublik Date: Fri, 7 Jul 2023 20:25:02 +0300 Subject: [PATCH 16/76] fixed typos --- volatility3/framework/plugins/windows/poolscanner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 5539e1e84..1f70cfb8c 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -230,7 +230,7 @@ class PoolScanner(plugins.PluginInterface): b"Thr\xe5", # -> “protected” allocation, MSB is set. type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size=(600, None), # -> 0x0258 - size of strcut in win5.1 + size=(600, None), # -> 0x0258 - size of struct in win5.1 skip_type_test=True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), @@ -239,7 +239,7 @@ class PoolScanner(plugins.PluginInterface): b"Thre", type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size=(600, None), # -> 0x0258 - size of strcut in win5.1 + size=(600, None), # -> 0x0258 - size of struct in win5.1 page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # files on windows before windows 8 From 338238c6396aa4c53a579a61a64eb4139de6cc76 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 16 Jul 2023 00:10:31 +0300 Subject: [PATCH 17/76] fixed build number - resets when MINOR version goes up --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09dded076..de1674885 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 4afc29cdb..6e19f9bd5 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # version 2.5.2 adds support for scanning for 'Ethread' structures by pool tags - _required_framework_version = (2, 5, 2) + # version 2.5.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 5, 0) _version = (1, 0, 0) @classmethod From 62466c7953cb0264bc3bb491a694a55732115d7a Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 3 Oct 2023 21:35:37 +0300 Subject: [PATCH 18/76] fixed merge conflicts with 'volatilityfoundation:develop' branch - bumped VERSION_PATCH --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index de1674885..c3ebaca27 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 57d995a81d3b6a9d1843439477c0be6933215df0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 8 Oct 2023 03:15:23 +0200 Subject: [PATCH 19/76] manually instantiate queue_entry for tasks symbol --- volatility3/framework/plugins/mac/pslist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 88045a277..c0e149fc2 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object_from_symbol(symbol_name="tasks") + queue_entry = kernel.object("queue_entry", kernel.get_symbol("tasks").address) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 6e5d41c38b3c494b9a43d5a7fca515aa84e1b6d4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 8 Oct 2023 03:37:17 +0200 Subject: [PATCH 20/76] manually instantiate queue_entry for tasks symbol --- volatility3/framework/plugins/mac/pslist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index c0e149fc2..e8c490dff 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object("queue_entry", kernel.get_symbol("tasks").address) + queue_entry = kernel.object(object_type="queue_entry", offset=kernel.get_symbol("tasks").address) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 3a656266711b21951ea8c48fe9d8ac4a4cf64775 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 23 Oct 2023 17:47:57 +0200 Subject: [PATCH 21/76] black formatting --- volatility3/framework/plugins/mac/pslist.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index e8c490dff..9835644b8 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,9 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object(object_type="queue_entry", offset=kernel.get_symbol("tasks").address) + queue_entry = kernel.object( + object_type="queue_entry", offset=kernel.get_symbol("tasks").address + ) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 25637a41e05e0bc5fccded01cf1913d45668ac25 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 8 Jan 2024 23:35:21 +0200 Subject: [PATCH 22/76] added account for XP timestamps - bit shifted --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 8790f41a7..9a286bc26 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -522,6 +522,9 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return True def get_create_time(self): + # For Windows XPs + if self.has_member("ThreadsProcess"): + return conversion.wintime_to_datetime(self.CreateTime.QuadPart >> 3) return conversion.wintime_to_datetime(self.CreateTime.QuadPart) def get_exit_time(self): From 550112b848913954e8b03d8c40bff9e2fd7902d7 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 8 Jan 2024 23:37:38 +0200 Subject: [PATCH 23/76] added another sanity check to ETHREAD.is_valid --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 9a286bc26..f4e2cc485 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -515,6 +515,9 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False + if not (1998 < ctime.year < 2030): + return False + except exceptions.InvalidAddressException: return False From 7c370121181af5da71b75d2b844def89341ddecd Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 9 Jan 2024 00:16:18 +0200 Subject: [PATCH 24/76] bumped version constants to mark change of interface (add of support for ETHREAD) --- volatility3/framework/constants/__init__.py | 4 ++-- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index c3ebaca27..9aaafb933 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 6 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 6e19f9bd5..80906b3b9 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # version 2.5.0 adds support for scanning for 'Ethread' structures by pool tags - _required_framework_version = (2, 5, 0) + # version 2.6.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 6, 0) _version = (1, 0, 0) @classmethod From b0d84e55ab1846004318e87bd7b5f427e7d70551 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 9 Jan 2024 21:32:29 +0200 Subject: [PATCH 25/76] fix indentation (typo) --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index f4e2cc485..a0af29d18 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -516,7 +516,7 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return False if not (1998 < ctime.year < 2030): - return False + return False except exceptions.InvalidAddressException: return False From d1c7f14a75dc36c32523caa907aa49375c772160 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 2 Feb 2024 13:28:04 +0000 Subject: [PATCH 26/76] Linux: attempt to fix issue 1089 --- .../symbols/linux/extensions/__init__.py | 75 ++++++++++++++----- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d8a2867cc..f67e3962b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -26,42 +26,79 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): + def _get_mod_mem_type(self): + """Attempt to get the mod_mem_type enum once to allow repeated access from other functions""" + if not self.has_member("mod_mem_type"): + try: + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( + self._context, self + ) + # mod_mem_type and module_memory were added in kernel 6.4 which replaces + # module_layout for storing the information around core_layout etc. + # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information + self.mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices + except exceptions.SymbolError: + vollog.debug( + f"Unable to find mod_mem_type enum. This is expected on kernels <6.4 but may cause issues with later kernels" + ) + self.mod_mem_type = None + def get_module_base(self): - if self.has_member("core_layout"): - return self.core_layout.base + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base else: - return self.module_core + if self.has_member("core_layout"): + return self.core_layout.base + else: + return self.module_core def get_init_size(self): - if self.has_member("init_layout"): - return self.init_layout.size - elif self.has_member("init_size"): - return self.init_size + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + else: + if self.has_member("init_layout"): + return self.init_layout.size + elif self.has_member("init_size"): + return self.init_size raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module" ) def get_core_size(self): - if self.has_member("core_layout"): - return self.core_layout.size - elif self.has_member("core_size"): - return self.core_size + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_TEXT"]].size + else: + if self.has_member("core_layout"): + return self.core_layout.size + elif self.has_member("core_size"): + return self.core_size raise AttributeError( "module -> get_core_size: Unable to determine core size of module" ) def get_module_core(self): - if self.has_member("core_layout"): - return self.core_layout.base - elif self.has_member("module_core"): - return self.module_core + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base + else: + if self.has_member("core_layout"): + return self.core_layout.base + elif self.has_member("module_core"): + return self.module_core raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - if self.has_member("init_layout"): - return self.init_layout.base - elif self.has_member("module_init"): - return self.module_init + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base + else: + if self.has_member("init_layout"): + return self.init_layout.base + elif self.has_member("module_init"): + return self.module_init raise AttributeError("module -> get_module_core: Unable to get module init") def get_name(self): From ecf20d99b4ddc3d28c577ee383902877c3090d45 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 2 Feb 2024 13:28:57 +0000 Subject: [PATCH 27/76] Linux: fix typo in error message in module.get_module_init extension --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f67e3962b..9c15acef5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -99,7 +99,7 @@ class module(generic.GenericIntelProcess): return self.init_layout.base elif self.has_member("module_init"): return self.module_init - raise AttributeError("module -> get_module_core: Unable to get module init") + raise AttributeError("module -> get_module_init: Unable to get module init") def get_name(self): """Get the name of the module as a string""" From 261d7ffbf2e6837980989292b3966ddd2042d277 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 2 Feb 2024 14:03:55 +0000 Subject: [PATCH 28/76] Linux: include all mod mem types in size calculations --- .../framework/symbols/linux/extensions/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9c15acef5..e3ff4bdd5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -56,7 +56,11 @@ class module(generic.GenericIntelProcess): def get_init_size(self): self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + return ( + self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size + + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size + ) else: if self.has_member("init_layout"): return self.init_layout.size @@ -69,7 +73,12 @@ class module(generic.GenericIntelProcess): def get_core_size(self): self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_TEXT"]].size + return ( + self.mem[self.mod_mem_type["MOD_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_DATA"]].size + + self.mem[self.mod_mem_type["MOD_RODATA"]].size + + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size + ) else: if self.has_member("core_layout"): return self.core_layout.size From 63a325c5157845d607d119971928657fb02ecdc2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 23:44:12 +0000 Subject: [PATCH 29/76] CLI: Add support for filtering lines from output --- volatility3/cli/__init__.py | 12 +++++++++++- volatility3/cli/text_renderer.py | 9 +++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 91bda7c66..2c6dde00a 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -22,6 +22,7 @@ import traceback from typing import Any, Dict, Type, Union from urllib import parse, request +from volatility3.cli import text_filter import volatility3.plugins import volatility3.symbols from volatility3 import framework @@ -230,6 +231,12 @@ class CommandLine: default=False, action="store_true", ) + parser.add_argument( + "--filters", + help="List of filters to apply to the output (in the form of [+-]columname,pattern[!])", + default=[], + action="append", + ) # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. @@ -444,7 +451,10 @@ class CommandLine: try: # Construct and run the plugin if constructed: - renderers[args.renderer]().render(constructed.run()) + grid = constructed.run() + renderer = renderers[args.renderer]() + renderer.filter = text_filter.CLIFilter(grid, args.filters) + renderer.render(grid) except exceptions.VolatilityException as excp: self.process_exceptions(excp) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 6e58ee68d..b0ba6baa7 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -10,6 +10,7 @@ import string import sys from functools import wraps from typing import Any, Callable, Dict, List, Tuple +from volatility3.cli import text_filter from volatility3.framework import interfaces, renderers from volatility3.framework.renderers import format_hints @@ -134,6 +135,7 @@ class CLIRenderer(interfaces.renderers.Renderer): name = "unnamed" structured_output = False + filter: text_filter.CLIFilter = None class QuickTextRenderer(CLIRenderer): @@ -172,6 +174,9 @@ class QuickTextRenderer(CLIRenderer): outfd.write("\n{}\n".format("\t".join(line))) def visitor(node: interfaces.renderers.TreeNode, accumulator): + if self.filter and self.filter.filter(node.values): + return accumulator + accumulator.write("\n") # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case accumulator.write( @@ -306,6 +311,10 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths[tree_indent_column] = max( max_column_widths.get(tree_indent_column, 0), node.path_depth ) + + if self.filter and self.filter.filter(node.values): + return accumulator + line = {} for column_index in range(len(grid.columns)): column = grid.columns[column_index] From d8d3e157afab853fbffff394d7d1bc371025664e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 23:46:30 +0000 Subject: [PATCH 30/76] CLI: Don't forget the core text filter class --- volatility3/cli/text_filter.py | 96 ++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 volatility3/cli/text_filter.py diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py new file mode 100644 index 000000000..314af9a7a --- /dev/null +++ b/volatility3/cli/text_filter.py @@ -0,0 +1,96 @@ +import logging +from typing import Any, List, Optional +from volatility3.framework import constants, interfaces +import re + +vollog = logging.getLogger(__name__) + + +class CLIFilter: + def __init__(self, treegrid, filters: List[str]): + self._filters = self._prepare(treegrid, filters) + + def _prepare(self, treegrid: interfaces.renderers.TreeGrid, filters: List[str]): + """Runs through the filter strings and creates the necessary filter objects""" + output = [] + + for filter in filters: + exclude = False + regex = False + pattern = None + column_name = None + if filter.startswith("-"): + exclude = True + filter = filter[1:] + elif filter.startswith("+"): + filter = filter[1:] + components = filter.split(",") + if len(components) < 2: + pattern = components[0] + else: + column_name = components[0] + pattern = ",".join(components[1:]) + if pattern and pattern.endswith("!"): + regex = True + pattern = pattern[:-1] + column_num = None + if column_name: + for num, column in enumerate(treegrid.columns): + if column_name.lower() in column.name.lower(): + column_num = num + break + if pattern: + output.append(ColumnFilter(column_num, pattern, regex, exclude)) + + vollog.log(constants.LOGLEVEL_VVV, "Filters:\n" + repr(output)) + + return output + + def filter( + self, + row: List[Any], + ) -> bool: + """Filters the row based on each of the column_filters""" + found = any([column_filter.found(row) for column_filter in self._filters]) + return not found + + +class ColumnFilter: + def __init__( + self, + column_num: Optional[int], + pattern: str, + regex: bool = False, + exclude: bool = False, + ) -> None: + self.column_num = column_num + self.pattern = pattern + self.exclude = exclude + self.regex = regex + + def find(self, item) -> bool: + """Identifies whether an item is found in the appropriate column""" + try: + if self.regex: + return re.search(self.pattern, f"{item}") + return self.pattern in f"{item}" + except IOError: + return False + + def found(self, row: List[Any]) -> bool: + """Determines whether a row should be filtered + + If the classes exclude value is false, and the necessary pattern is found, the row is not filtered, + otherwise it is filtered. + """ + if self.column_num is None: + found = any([self.find(x) for x in row]) + else: + found = self.find(row[self.column_num]) + if self.exclude: + return not found + return found + + def __repr__(self) -> str: + """Returns a display of a column filter""" + return f"ColumnFilter(column={self.column_num},exclude={self.exclude},regex={self.regex},pattern={self.pattern})" From 865f004f5c6628c230c3dedde0e04573aa4e1339 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 9 Feb 2024 00:00:21 +0000 Subject: [PATCH 31/76] CLI: Ensure no filters still returns results --- volatility3/cli/text_filter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 314af9a7a..948e969df 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -51,6 +51,8 @@ class CLIFilter: row: List[Any], ) -> bool: """Filters the row based on each of the column_filters""" + if not self._filters: + return False found = any([column_filter.found(row) for column_filter in self._filters]) return not found From 0e75140ec5a36e8f9933eee90cc74f0e06789242 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 12 Feb 2024 09:35:06 +0000 Subject: [PATCH 32/76] Linux: update module extension with mod_mem_type property and add try/except for get methods --- .../symbols/linux/extensions/__init__.py | 75 ++++++++++++------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3557f3a17..fb60e3b97 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,6 +15,7 @@ from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATE from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework.constants.linux import CAPABILITIES from volatility3.framework import exceptions, objects, interfaces, symbols +from volatility3.framework.renderers import UnparsableValue from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed @@ -26,9 +27,10 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): - def _get_mod_mem_type(self): - """Attempt to get the mod_mem_type enum once to allow repeated access from other functions""" - if not self.has_member("mod_mem_type"): + @property + def mod_mem_type(self): + """Return the mod_mem_type enum choices if available or None if not""" + if not self.has_member("_mod_mem_type"): try: vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( self._context, self @@ -36,31 +38,41 @@ class module(generic.GenericIntelProcess): # mod_mem_type and module_memory were added in kernel 6.4 which replaces # module_layout for storing the information around core_layout etc. # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information - self.mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices + self._mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices except exceptions.SymbolError: vollog.debug( f"Unable to find mod_mem_type enum. This is expected on kernels <6.4 but may cause issues with later kernels" ) - self.mod_mem_type = None + self._mod_mem_type = None + return self._mod_mem_type def get_module_base(self): - self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base + try: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base + except: + raise AttributeError( + "module -> get_module_base: Unable to get module base. Cannot read base from MOD_TEXT." + ) else: if self.has_member("core_layout"): return self.core_layout.base - else: + elif self.has_member("module_core"): return self.module_core + raise AttributeError("module -> get_module_base: Unable to get module base") def get_init_size(self): - self._get_mod_mem_type() if self.mod_mem_type: - return ( - self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size - + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size - ) + try: + return ( + self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size + + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size + ) + except: + raise AttributeError( + "module -> get_init_size: Unable to determine .init section size of module. Cannot read size of MOD_INIT_TEXT, MOD_INIT_DATA, and MOD_INIT_RODATA" + ) else: if self.has_member("init_layout"): return self.init_layout.size @@ -71,14 +83,19 @@ class module(generic.GenericIntelProcess): ) def get_core_size(self): - self._get_mod_mem_type() if self.mod_mem_type: - return ( - self.mem[self.mod_mem_type["MOD_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_DATA"]].size - + self.mem[self.mod_mem_type["MOD_RODATA"]].size - + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size - ) + try: + return ( + self.mem[self.mod_mem_type["MOD_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_DATA"]].size + + self.mem[self.mod_mem_type["MOD_RODATA"]].size + + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size + ) + except KeyError: + raise AttributeError( + "module -> get_core_size: Unable to determine core size of module. Cannot read size of MOD_TEXT, MOD_DATA, MOD_RODATA, and MOD_RO_AFTER_INIT." + ) + else: if self.has_member("core_layout"): return self.core_layout.size @@ -89,9 +106,13 @@ class module(generic.GenericIntelProcess): ) def get_module_core(self): - self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base + try: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base + except KeyError: + raise AttributeError( + "module -> get_module_core: Unable to get module core. Cannot read base from MOD_TEXT." + ) else: if self.has_member("core_layout"): return self.core_layout.base @@ -100,9 +121,13 @@ class module(generic.GenericIntelProcess): raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base + try: + return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base + except KeyError: + raise AttributeError( + "module -> get_module_core: Unable to get module init. Cannot read base from MOD_INIT_TEXT." + ) else: if self.has_member("init_layout"): return self.init_layout.base From 53d4ec66ebb774db76ef1999832662ecc541e56b Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 12 Feb 2024 09:36:33 +0000 Subject: [PATCH 33/76] Linux: Remove unnecessary import from linux extension. --- volatility3/framework/symbols/linux/extensions/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fb60e3b97..c259d16aa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,7 +15,6 @@ from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATE from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework.constants.linux import CAPABILITIES from volatility3.framework import exceptions, objects, interfaces, symbols -from volatility3.framework.renderers import UnparsableValue from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed From cc35fecf91b1d2704c87031427c623e066ac968a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 19:40:37 +1100 Subject: [PATCH 34/76] Linux: Add library_list plugin and other ELF related code enhacements. - Add library_list plugin - Add ELF dynamic table enum types in elf.json - Update missing program header enum types in elf.json - Add PAGE constants - Add ELF ident and class enums - Replace ELF hardcoded type numbers for enum description matching - Fix unmanaged ValueError exception issue in Elf64Layer::_load_segments() --- .../framework/constants/linux/__init__.py | 26 +++ volatility3/framework/layers/elf.py | 11 +- volatility3/framework/plugins/linux/elfs.py | 34 ++-- .../framework/plugins/linux/library_list.py | 169 ++++++++++++++++++ volatility3/framework/symbols/linux/elf.json | 138 +++++++++++++- .../symbols/linux/extensions/__init__.py | 3 +- .../framework/symbols/linux/extensions/elf.py | 122 +++++++++++-- 7 files changed, 464 insertions(+), 39 deletions(-) create mode 100644 volatility3/framework/plugins/linux/library_list.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 6e8883f19..5e82e580e 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -5,11 +5,15 @@ Linux-specific values that aren't found in debug symbols """ +from enum import IntEnum KERNEL_NAME = "__kernel__" # arch/x86/include/asm/page_types.h PAGE_SHIFT = 12 +PAGE_SIZE = 1 << PAGE_SHIFT +PAGE_MASK = ~(PAGE_SIZE - 1) + """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" # include/linux/sched.h @@ -281,3 +285,25 @@ CAPABILITIES = ( ) ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 + + +class ELF_IDENT(IntEnum): + """ELF header e_ident indexes""" + + EI_MAG0 = 0 + EI_MAG1 = 1 + EI_MAG2 = 2 + EI_MAG3 = 3 + EI_CLASS = 4 + EI_DATA = 5 + EI_VERSION = 6 + EI_OSABI = 7 + EI_PAD = 8 + + +class ELF_CLASS(IntEnum): + """ELF header class types""" + + ELFCLASSNONE = 0 + ELFCLASS32 = 1 + ELFCLASS64 = 2 diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index b2fd6d4d1..6bd5c2d63 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -6,9 +6,11 @@ import struct from typing import Optional from volatility3.framework import exceptions, interfaces, constants +from volatility3.framework.constants.linux import ELF_CLASS from volatility3.framework.layers import segmented from volatility3.framework.symbols import intermed + vollog = logging.getLogger(__name__) @@ -21,7 +23,7 @@ class Elf64Layer(segmented.SegmentedLayer): _header_struct = struct.Struct(" 0 ): diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index e688ecb42..7171a6616 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -14,8 +14,14 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux.extensions import elf +from volatility3.framework.constants.linux import ( + PAGE_SIZE, + PAGE_MASK, + ELF_MAX_EXTRACTION_SIZE, +) from volatility3.plugins.linux import pslist + vollog = logging.getLogger(__name__) @@ -23,7 +29,7 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -87,7 +93,10 @@ class Elfs(plugins.PluginInterface): sections = {} # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? for phdr in elf_object.get_program_headers(): - if phdr.p_type != 1: # PT_LOAD = 1 + try: + if phdr.p_type.description != "PT_LOAD": + continue + except ValueError: continue start = phdr.p_vaddr @@ -95,18 +104,18 @@ class Elfs(plugins.PluginInterface): end = start + size # Use complete memory pages for dumping - # If start isn't a multiple of 4096, stick to the highest multiple < start - # If end isn't a multiple of 4096, stick to the lowest multiple > end - if start % 4096: - start = start & ~0xFFF + # If start isn't a multiple of a page, stick to the highest multiple < start + # If end isn't a multiple of a page, stick to the lowest multiple > end + if start % PAGE_SIZE: + start = start & PAGE_MASK - if end % 4096: - end = (end & ~0xFFF) + 4096 + if end % PAGE_SIZE: + end = (end & PAGE_MASK) + PAGE_SIZE real_size = end - start # Check if ELF has a legitimate size - if real_size < 0 or real_size > constants.linux.ELF_MAX_EXTRACTION_SIZE: + if real_size < 0 or real_size > ELF_MAX_EXTRACTION_SIZE: raise ValueError(f"The claimed size of the ELF is invalid: {real_size}") sections[start] = real_size @@ -140,12 +149,7 @@ class Elfs(plugins.PluginInterface): for vma in task.mm.get_vma_iter(): hdr = proc_layer.read(vma.vm_start, 4, pad=True) - if not ( - hdr[0] == 0x7F - and hdr[1] == 0x45 - and hdr[2] == 0x4C - and hdr[3] == 0x46 - ): + if hdr != b"\x7fELF": continue path = vma.get_name(self.context, task) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py new file mode 100644 index 000000000..ed5545347 --- /dev/null +++ b/volatility3/framework/plugins/linux/library_list.py @@ -0,0 +1,169 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import Iterable, Tuple + +from volatility3.framework import interfaces, renderers, constants, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.objects import utility +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.linux.extensions import elf +from volatility3.plugins.linux import pslist + + +vollog = logging.getLogger(__name__) + + +class LibraryList(interfaces.plugins.PluginInterface): + """Enumerate libraries loaded into processes""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + ), + requirements.ListRequirement( + name="pids", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + ] + + def get_libdl_libraries( + self, proc_layer_name: str, vma_start: int + ) -> interfaces.objects.ObjectInterface: + """Get the ELF link map objects for the given VMA address + + Args: + proc_layer_name (str): Name of the process layer + vma_start (int): VMA start address + + Yields: + ELF link map objects for the given VMA address + """ + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "linux", + "elf", + class_types=elf.class_types, + ) + elf_object = self.context.object( + elf_table_name + constants.BANG + "Elf", + offset=vma_start, + layer_name=proc_layer_name, + ) + + if not elf_object or not elf_object.is_valid(): + return None + + kernel = self.context.modules[self.config["kernel"]] + + try: + for link_map in elf_object.get_link_maps(kernel.symbol_table_name): + if link_map.l_addr and link_map.l_name: + yield link_map + except exceptions.InvalidAddressException: + # Protection against memory smear in this VMA + pass + + def get_libdl_maps( + self, task: interfaces.objects.ObjectInterface, proc_layer_name: str + ) -> interfaces.objects.ObjectInterface: + """Get the ELF link maps objects for a task + + Args: + task (task_struct): A reference task + proc_layer_name (str): Name of the process layer + + Yields: + ELF link map objects + """ + + link_map_seen = set() + for vma in task.mm.get_vma_iter(): + for link_map in self.get_libdl_libraries(proc_layer_name, vma.vm_start): + if link_map.l_addr in link_map_seen: + continue + + yield link_map + link_map_seen.add(link_map.l_addr) + + def get_task_libraries( + self, task: interfaces.objects.ObjectInterface + ) -> Tuple[int, str]: + """Get the task libraries from the ELF headers found within the memory maps + + Args: + task (task_struct): The reference task + + Yields: + Tuples with a ELF link map address and name + """ + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + return + + for elf_link_map in self.get_libdl_maps(task, proc_layer_name): + name = elf_link_map.get_name() + if not name: + continue + yield elf_link_map.l_addr, name + + def get_tasks_libraries( + self, + tasks: Iterable[interfaces.objects.ObjectInterface], + ) -> Iterable[Tuple[str, int, int, str]]: + """Get the task libraries from the ELF headers found within the memory maps for + all the tasks. + + Args: + tasks: An iterable of tasks + + Yields: + Tuples with a task name, task tgid, an ELF link map address and name + """ + for task in tasks: + task_name = utility.array_to_string(task.comm) + for linkmap_addr, linkmap_name in self.get_task_libraries(task): + yield task_name, task.tgid, linkmap_addr, linkmap_name + + def _format_fields(self, fields): + task_name, task_pid, addr, name = fields + return task_name, task_pid, format_hints.Hex(addr), name + + def _generator( + self, tasks: Iterable[interfaces.objects.ObjectInterface] + ) -> Iterable[Tuple[int, Tuple]]: + for fields in self.get_tasks_libraries(tasks): + yield 0, self._format_fields(fields) + + def run(self): + pids = self.config.get("pids") + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=pid_filter + ) + + headers = [ + ("Name", str), + ("Pid", int), + ("LoadAddress", format_hints.Hex), + ("Path", str), + ] + + return renderers.TreeGrid(headers, self._generator(tasks)) diff --git a/volatility3/framework/symbols/linux/elf.json b/volatility3/framework/symbols/linux/elf.json index 76cd8a2ec..e0a95bbba 100644 --- a/volatility3/framework/symbols/linux/elf.json +++ b/volatility3/framework/symbols/linux/elf.json @@ -270,8 +270,8 @@ "d_tag": { "offset": 0, "type": { - "kind": "base", - "name": "long long" + "kind": "enum", + "name": "DtypeEnum64" } }, "d_ptr": { @@ -699,8 +699,8 @@ "d_tag": { "offset": 0, "type": { - "kind": "base", - "name": "long" + "kind": "enum", + "name": "DtypeEnum32" } }, "d_ptr": { @@ -905,11 +905,139 @@ "PT_PHDR": 6, "PT_TLS": 7, "PT_LOOS": 1610612736, + "PT_GNU_EH_FRAME": 1685382480, + "PT_GNU_STACK": 1685382481, + "PT_GNU_RELRO": 1685382482, + "PT_GNU_PROPERTY": 1685382483, "PT_HIOS": 1879048191, "PT_LOWPROC": 1879048192, "PT_HIPROC": 2147483647 }, "size": 4 + }, + "DtypeEnum32": { + "base": "long", + "constants": { + "DT_NULL": 0, + "DT_NEEDED": 1, + "DT_PLTRELSZ": 2, + "DT_PLTGOT": 3, + "DT_HASH": 4, + "DT_STRTAB": 5, + "DT_SYMTAB": 6, + "DT_RELA": 7, + "DT_RELASZ": 8, + "DT_RELAENT": 9, + "DT_STRSZ": 10, + "DT_SYMENT": 11, + "DT_INIT": 12, + "DT_FINI": 13, + "DT_SONAME": 14, + "DT_RPATH": 15, + "DT_SYMBOLIC": 16, + "DT_REL": 17, + "DT_RELSZ": 18, + "DT_RELENT": 19, + "DT_PLTREL": 20, + "DT_DEBUG": 21, + "DT_TEXTREL": 22, + "DT_JMPREL": 23, + "DT_BIND_NOW": 24, + "DT_INIT_ARRAY": 25, + "DT_FINI_ARRAY": 26, + "DT_INIT_ARRAYSZ": 27, + "DT_FINI_ARRAYSZ": 28, + "DT_RUNPATH": 29, + "DT_FLAGS": 30, + "DT_ENCODING": 32, + "DT_PREINIT_ARRAYSZ": 33, + "DT_SYMTAB_SHNDX": 34, + "DT_RELRSZ": 35, + "DT_RELR": 36, + "DT_RELRENT": 37, + "DT_NUM": 38, + "OLD_DT_LOOS": 1610612736, + "DT_LOOS": 1610612749, + "DT_HIOS": 1879044096, + "DT_VALRNGLO": 1879047424, + "DT_VALRNGHI": 1879047679, + "DT_ADDRRNGLO": 1879047680, + "DT_GNU_HASH": 1879047925, + "DT_ADDRRNGHI": 1879047935, + "DT_VERSYM": 1879048176, + "DT_RELACOUNT": 1879048185, + "DT_RELCOUNT": 1879048186, + "DT_FLAGS_1": 1879048187, + "DT_VERDEF": 1879048188, + "DT_VERDEFNUM": 1879048189, + "DT_VERNEED": 1879048190, + "DT_VERNEEDNUM": 1879048191, + "DT_LOPROC": 1879048192, + "DT_HIPROC": 2147483647 + }, + "size": 4 + }, + "DtypeEnum64": { + "base": "long long", + "constants": { + "DT_NULL": 0, + "DT_NEEDED": 1, + "DT_PLTRELSZ": 2, + "DT_PLTGOT": 3, + "DT_HASH": 4, + "DT_STRTAB": 5, + "DT_SYMTAB": 6, + "DT_RELA": 7, + "DT_RELASZ": 8, + "DT_RELAENT": 9, + "DT_STRSZ": 10, + "DT_SYMENT": 11, + "DT_INIT": 12, + "DT_FINI": 13, + "DT_SONAME": 14, + "DT_RPATH": 15, + "DT_SYMBOLIC": 16, + "DT_REL": 17, + "DT_RELSZ": 18, + "DT_RELENT": 19, + "DT_PLTREL": 20, + "DT_DEBUG": 21, + "DT_TEXTREL": 22, + "DT_JMPREL": 23, + "DT_BIND_NOW": 24, + "DT_INIT_ARRAY": 25, + "DT_FINI_ARRAY": 26, + "DT_INIT_ARRAYSZ": 27, + "DT_FINI_ARRAYSZ": 28, + "DT_RUNPATH": 29, + "DT_FLAGS": 30, + "DT_ENCODING": 32, + "DT_PREINIT_ARRAYSZ": 33, + "DT_SYMTAB_SHNDX": 34, + "DT_RELRSZ": 35, + "DT_RELR": 36, + "DT_RELRENT": 37, + "DT_NUM": 38, + "OLD_DT_LOOS": 1610612736, + "DT_LOOS": 1610612749, + "DT_HIOS": 1879044096, + "DT_VALRNGLO": 1879047424, + "DT_VALRNGHI": 1879047679, + "DT_ADDRRNGLO": 1879047680, + "DT_GNU_HASH": 1879047925, + "DT_ADDRRNGHI": 1879047935, + "DT_VERSYM": 1879048176, + "DT_RELACOUNT": 1879048185, + "DT_RELCOUNT": 1879048186, + "DT_FLAGS_1": 1879048187, + "DT_VERDEF": 1879048188, + "DT_VERDEFNUM": 1879048189, + "DT_VERNEED": 1879048190, + "DT_VERNEEDNUM": 1879048191, + "DT_LOPROC": 1879048192, + "DT_HIPROC": 2147483647 + }, + "size": 8 } }, "base_types": { @@ -958,7 +1086,7 @@ }, "metadata": { "producer": { - "version": "0.0.1", + "version": "0.0.2", "name": "ikelos-by-hand", "datetime": "2019-10-21T22:52:00" }, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d73d0cfb9..0a3db9fcd 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,14 +7,13 @@ import logging import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List -from volatility3.framework import constants +from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework.constants.linux import CAPABILITIES -from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index fe85b194f..4b2b29b54 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -6,6 +6,10 @@ from typing import Dict, Tuple import logging from volatility3.framework import constants +from volatility3.framework.constants.linux import ( + ELF_IDENT, + ELF_CLASS, +) from volatility3.framework import objects, interfaces, exceptions vollog = logging.getLogger(__name__) @@ -59,13 +63,15 @@ class elf(objects.StructType): ei_class = self._context.object( symbol_table_name + constants.BANG + "unsigned char", layer_name=layer_name, - offset=object_info.offset + 0x4, + offset=object_info.offset + ELF_IDENT.EI_CLASS, ) - if ei_class == 1: + if ei_class == ELF_CLASS.ELFCLASS32: self._type_prefix = "Elf32_" - elif ei_class == 2: + self._ei_class_size = 32 + elif ei_class == ELF_CLASS.ELFCLASS64: self._type_prefix = "Elf64_" + self._ei_class_size = 64 else: raise ValueError(f"Unsupported ei_class value {ei_class}") @@ -140,36 +146,103 @@ class elf(objects.StructType): ) return section_headers + def get_link_maps(self, kernel_symbol_table_name): + """Get the ELF link map objects for the given VMA address + + Args: + kernel_symbol_table_name (str): Kernel symbol table name + + Yields: + The ELF link map objects + """ + got_entry_size = self._ei_class_size // 8 + + elf_symbol_table = self.get_symbol_table_name() + + link_maps_seen = set() + for phdr in self.get_program_headers(): + try: + if phdr.p_type.description != "PT_DYNAMIC": + continue + except ValueError: + continue + + for dsec in phdr.dynamic_sections(): + try: + if dsec.d_tag.description != "DT_PLTGOT": + continue + except ValueError: + continue + + got_start = dsec.d_ptr + + # link_map is stored at the second GOT entry + link_map_addr = got_start + got_entry_size + + # It needs the kernel symbol table to create a pointer + link_map_ptr = self._context.object( + kernel_symbol_table_name + constants.BANG + "pointer", + offset=link_map_addr, + layer_name=self.vol.layer_name, + ) + if not link_map_ptr: + continue + + linkmap_symname = ( + elf_symbol_table + constants.BANG + self._type_prefix + "LinkMap" + ) + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map_ptr, + layer_name=self.vol.layer_name, + ) + + while link_map and link_map.vol.offset != 0: + if link_map.vol.offset in link_maps_seen: + break + link_maps_seen.add(link_map.vol.offset) + + yield link_map + + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map.l_next, + layer_name=self.vol.layer_name, + ) + def _find_symbols(self): dt_strtab = None dt_symtab = None dt_strent = None for phdr in self.get_program_headers(): + # Find PT_DYNAMIC segment try: - # Find PT_DYNAMIC segment - if str(phdr.p_type.description) != "PT_DYNAMIC": + if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: - # If the p_type value is outside the ones declared in the enumeration, an - # exception is raised - return None + continue # This section contains pointers to the strtab, symtab, and strent sections for dsec in phdr.dynamic_sections(): - if dsec.d_tag == 5: + try: + dtag = dsec.d_tag.description + except ValueError: + continue + + if dtag == "DT_STRTAB": dt_strtab = dsec.d_ptr - elif dsec.d_tag == 6: + elif dtag == "DT_SYMTAB": dt_symtab = dsec.d_ptr - elif dsec.d_tag == 11: + elif dtag == "DT_SYMENT": # Size of the symtab symbol entry dt_strent = dsec.d_ptr break - if dt_strtab is None or dt_symtab is None or dt_strent is None: + if not (dt_strtab and dt_symtab and dt_strent): return None self._cached_symtab = dt_symtab @@ -274,15 +347,18 @@ class elf_phdr(objects.StructType): def get_vaddr(self): offset = self.__getattr__("p_vaddr") - if self._parent_e_type == 3: # ET_DYN - offset = self._parent_offset + offset + try: + if self._parent_e_type.description == "ET_DYN": + offset = self._parent_offset + offset + except ValueError: + pass return offset def dynamic_sections(self): # sanity check try: - if str(self.p_type.description) != "PT_DYNAMIC": + if self.p_type.description != "PT_DYNAMIC": return None except ValueError: # If the value is outside the ones declared in the enumeration, an @@ -314,10 +390,26 @@ class elf_phdr(objects.StructType): break +class elf_linkmap(objects.StructType): + def get_name(self): + try: + buf = self._context.layers.read(self.vol.layer_name, self.l_name, 256) + except exceptions.PagedInvalidAddressException: + # Protection against memory smear + return None + + idx = buf.find(b"\x00") + if idx != -1: + buf = buf[:idx] + return buf.decode() + + class_types = { "Elf": elf, "Elf64_Phdr": elf_phdr, "Elf32_Phdr": elf_phdr, "Elf32_Sym": elf_sym, "Elf64_Sym": elf_sym, + "Elf32_LinkMap": elf_linkmap, + "Elf64_LinkMap": elf_linkmap, } From 6e7b5b59416a0f0830660d6415144ec8ee0039ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 19:41:30 +1100 Subject: [PATCH 35/76] Linux: Add linux_library_list test case --- test/test_volatility.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index aaad615bc..7b151fd6c 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -6,6 +6,7 @@ # import os +import re import subprocess import sys import shutil @@ -331,6 +332,32 @@ def test_linux_tty_check(image, volatility, python): assert rc == 0 +def test_linux_library_list(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.library_list.LibraryList", image, volatility, python + ) + + assert re.search( + rb"NetworkManager\s2363\s0x7f52cdda0000\s/lib/x86_64-linux-gnu/libnss_files.so.2", + out, + ) + assert re.search( + rb"gnome-settings-\s3807\s0x7f7e660b5000\s/lib/x86_64-linux-gnu/libbz2.so.1.0", + out, + ) + assert re.search( + rb"gdu-notificatio\s3878\s0x7f25ce33e000\s/usr/lib/x86_64-linux-gnu/libXau.so.6", + out, + ) + assert re.search( + rb"bash\s8600\s0x7fe78a85f000\s/lib/x86_64-linux-gnu/libnss_files.so.2", + out, + ) + + assert out.count(b"\n") >= 2677 + assert rc == 0 + + # MAC From cf81ceda8262b1bdfc528277cd405c24d572e320 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 20:20:25 +1100 Subject: [PATCH 36/76] Fix CodeQL suggestion --- volatility3/framework/symbols/linux/extensions/elf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 4b2b29b54..828370fe8 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -351,6 +351,8 @@ class elf_phdr(objects.StructType): if self._parent_e_type.description == "ET_DYN": offset = self._parent_offset + offset except ValueError: + # Unknown ELF object file type. Anyway, if the ELF object file type is not a + # shared object (ET_DYN), the virtual address is 'p_vaddr'. pass return offset From cd08cb95fdf9d2503162020694fd4e3eddaf51d3 Mon Sep 17 00:00:00 2001 From: Alejandro Diego Date: Tue, 6 Feb 2024 09:26:49 -0500 Subject: [PATCH 37/76] Windows: Add filtering by offset to psscan Add capability to the psscan plugin to filter by specific offset. The filter would be used by other plugins to find specific offset using the psscan capabilities, like find hidden processes as a result of some dkom for example. The `--offset` flag argument should represent a physical address space. The flag is included in the following plugins: - dlllist - handles --- .../framework/plugins/windows/dlllist.py | 37 +++++++++++++---- .../framework/plugins/windows/handles.py | 41 ++++++++++++++----- .../framework/plugins/windows/psscan.py | 40 +++++++++++++++++- 3 files changed, 97 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index d73cea652..27eddc991 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -13,7 +13,7 @@ from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins import timeliner -from volatility3.plugins.windows import info, pslist +from volatility3.plugins.windows import info, pslist, psscan vollog = logging.getLogger(__name__) @@ -36,6 +36,9 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="psscan", component=psscan.PsScan, version=(1, 1, 0) + ), requirements.VersionRequirement( name="info", component=info.Info, version=(1, 0, 0) ), @@ -45,6 +48,11 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): description="Process IDs to include (all other processes are excluded)", optional=True, ), + requirements.IntRequirement( + name="offset", + description="Process offset in the physical address space", + optional=True, + ), requirements.BooleanRequirement( name="dump", description="Extract listed DLLs", @@ -221,6 +229,24 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) kernel = self.context.modules[self.config["kernel"]] + if self.config["offset"]: + procs = psscan.PsScan.scan_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=psscan.PsScan.create_offset_filter( + self.context.layers[kernel.layer_name], + self.config["offset"], + ), + ) + else: + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + return renderers.TreeGrid( [ ("PID", int), @@ -232,12 +258,5 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ("LoadTime", datetime.datetime), ("File output", str), ], - self._generator( - pslist.PsList.list_processes( - context=self.context, - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, - filter_func=filter_func, - ) - ), + self._generator(procs=procs), ) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index ddd9cb78e..2c5fde1c9 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -9,7 +9,7 @@ from volatility3.framework import constants, exceptions, renderers, interfaces, from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist +from volatility3.plugins.windows import pslist, psscan vollog = logging.getLogger(__name__) @@ -43,14 +43,22 @@ class Handles(interfaces.plugins.PluginInterface): description="Windows kernel", architectures=["Intel32", "Intel64"], ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="psscan", component=psscan.PsScan, version=(1, 1, 0) + ), requirements.ListRequirement( name="pid", element_type=int, description="Process IDs to include (all other processes are excluded)", optional=True, ), - requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + requirements.IntRequirement( + name="offset", + description="Process offset in the physical address space", + optional=True, ), ] @@ -416,6 +424,24 @@ class Handles(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) kernel = self.context.modules[self.config["kernel"]] + if self.config["offset"]: + procs = psscan.PsScan.scan_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=psscan.PsScan.create_offset_filter( + self.context.layers[kernel.layer_name], + self.config["offset"], + ), + ) + else: + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + return renderers.TreeGrid( [ ("PID", int), @@ -426,12 +452,5 @@ class Handles(interfaces.plugins.PluginInterface): ("GrantedAccess", format_hints.Hex), ("Name", str), ], - self._generator( - pslist.PsList.list_processes( - self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func=filter_func, - ) - ), + self._generator(procs=procs), ) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 3d9ae5c1e..8b3afaf7c 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Iterable, Callable, Optional, Tuple +from typing import Iterable, Callable, List, Optional, Tuple from volatility3.framework import renderers, interfaces, layers, exceptions from volatility3.framework.configuration import requirements @@ -59,6 +59,44 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] + @classmethod + def create_offset_filter( + cls, + memory: interfaces.layers.DataLayerInterface, + offset: int = None, + exclude: bool = False, + ) -> Callable[[interfaces.objects.ObjectInterface], bool]: + """A factory for producing filter functions that filter based on the physical offset of the process. + + Args: + offset: A number that is the physical offset to be filtered out + memory: Memory object needed to do the offset mapping to physical. + exclude: Accept only tasks that are not the offset argument + Returns: + Filter function to be passed to the list of processes. + """ + if not isinstance(memory, interfaces.layers.DataLayerInterface): + raise TypeError("memory object requires an instance of DataLayerInterface") + + filter_func = lambda _: False + + # return physical offset in tuple -> (_, _, physical_offset, _, _) + # from the first item in the memory mapping list + virtual_to_physical_offset = lambda virtual_offset, memory: list( + memory.mapping(offset=virtual_offset, length=0) + )[0][2] + + if offset: + if exclude: + filter_func = ( + lambda x: virtual_to_physical_offset(x.vol.offset, memory) == offset + ) + else: + filter_func = ( + lambda x: virtual_to_physical_offset(x.vol.offset, memory) != offset + ) + return filter_func + @classmethod def scan_processes( cls, From 5e38ffa811bcf385c2b470351ed2518118c7d875 Mon Sep 17 00:00:00 2001 From: Alejandro Diego Date: Tue, 20 Feb 2024 15:07:15 -0500 Subject: [PATCH 38/76] Added support for virtual address filtering and validation of ph/v space --- .../framework/plugins/windows/dlllist.py | 3 +- .../framework/plugins/windows/handles.py | 3 +- .../framework/plugins/windows/psscan.py | 67 ++++++++++++++----- 3 files changed, 53 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 27eddc991..f876dc1a2 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -235,7 +235,8 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.layer_name, kernel.symbol_table_name, filter_func=psscan.PsScan.create_offset_filter( - self.context.layers[kernel.layer_name], + self.context, + kernel.layer_name, self.config["offset"], ), ) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 2c5fde1c9..d43d26ef1 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -430,7 +430,8 @@ class Handles(interfaces.plugins.PluginInterface): kernel.layer_name, kernel.symbol_table_name, filter_func=psscan.PsScan.create_offset_filter( - self.context.layers[kernel.layer_name], + self.context, + kernel.layer_name, self.config["offset"], ), ) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 8b3afaf7c..82a481070 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -59,42 +59,73 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] + @classmethod + def physical_offset_from_virtual(cls, context, layer_name, proc): + """Calculate the physical offset from the virtual offset of a process. + + Args: + context: The context containing layers and modules information. + layer_name: The name of the layer containing the process memory. + proc: The process object for which to calculate the physical offset. + + Returns: + int: The physical offset of the process. + Raises: + TypeError: If the primary layer is not an Intel layer. + """ + memory = context.layers[layer_name] + + if not isinstance(memory, layers.intel.Intel): + raise TypeError("Primary layer is not an intel layer") + + (_, _, ph_offset, _, _) = list( + memory.mapping(offset=proc.vol.offset, length=0) + )[0] + + return ph_offset + @classmethod def create_offset_filter( cls, - memory: interfaces.layers.DataLayerInterface, + context: interfaces.context.ContextInterface, + layer_name: str, offset: int = None, + physical: bool = True, exclude: bool = False, ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on the physical offset of the process. Args: offset: A number that is the physical offset to be filtered out - memory: Memory object needed to do the offset mapping to physical. exclude: Accept only tasks that are not the offset argument + Returns: Filter function to be passed to the list of processes. """ - if not isinstance(memory, interfaces.layers.DataLayerInterface): - raise TypeError("memory object requires an instance of DataLayerInterface") - filter_func = lambda _: False - # return physical offset in tuple -> (_, _, physical_offset, _, _) - # from the first item in the memory mapping list - virtual_to_physical_offset = lambda virtual_offset, memory: list( - memory.mapping(offset=virtual_offset, length=0) - )[0][2] - if offset: - if exclude: - filter_func = ( - lambda x: virtual_to_physical_offset(x.vol.offset, memory) == offset - ) + if physical: + if exclude: + filter_func = ( + lambda proc: cls.physical_offset_from_virtual( + context, layer_name, proc + ) + == offset + ) + else: + filter_func = ( + lambda proc: cls.physical_offset_from_virtual( + context, layer_name, proc + ) + != offset + ) else: - filter_func = ( - lambda x: virtual_to_physical_offset(x.vol.offset, memory) != offset - ) + if exclude: + lambda proc: proc.vol.offset == offset + else: + lambda proc: proc.vol.offset != offset + return filter_func @classmethod From 42c6a9ede6bbe3ff52b868144516679753c9fc8c Mon Sep 17 00:00:00 2001 From: Alejandro Diego Date: Tue, 20 Feb 2024 20:07:17 -0500 Subject: [PATCH 39/76] Remove unnecessary import and correctly filter_func var --- volatility3/framework/plugins/windows/psscan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 82a481070..5634298a8 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Iterable, Callable, List, Optional, Tuple +from typing import Iterable, Callable, Optional, Tuple from volatility3.framework import renderers, interfaces, layers, exceptions from volatility3.framework.configuration import requirements @@ -122,9 +122,9 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) else: if exclude: - lambda proc: proc.vol.offset == offset + filter_func = lambda proc: proc.vol.offset == offset else: - lambda proc: proc.vol.offset != offset + filter_func = lambda proc: proc.vol.offset != offset return filter_func From e10cff979af3917b3d0a67ec050c181e37bffed8 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 21 Feb 2024 12:50:05 +0000 Subject: [PATCH 40/76] Linux: update module extension with memoization for _mod_mem_type --- .../symbols/linux/extensions/__init__.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3673b66df..8d11374c9 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -26,27 +26,33 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._mod_mem_type = None # Initialize _mod_mem_type to None for memoization + @property def mod_mem_type(self): - """Return the mod_mem_type enum choices if available or None if not""" - if not self.has_member("_mod_mem_type"): + """Return the mod_mem_type enum choices if available or an empty dict if not""" + # mod_mem_type and module_memory were added in kernel 6.4 which replaces + # module_layout for storing the information around core_layout etc. + # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information + + if self._mod_mem_type is None: try: - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( - self._context, self - ) - # mod_mem_type and module_memory were added in kernel 6.4 which replaces - # module_layout for storing the information around core_layout etc. - # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information - self._mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices + self._mod_mem_type = self._context.symbol_space.get_enumeration( + self.get_symbol_table_name() + constants.BANG + "mod_mem_type" + ).choices except exceptions.SymbolError: vollog.debug( - f"Unable to find mod_mem_type enum. This is expected on kernels <6.4 but may cause issues with later kernels" + f"Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" ) - self._mod_mem_type = None + # set to empty dict to show that the enum was not found, and so shouldn't be searched for again + self._mod_mem_type = {} return self._mod_mem_type def get_module_base(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base except: @@ -61,7 +67,7 @@ class module(generic.GenericIntelProcess): raise AttributeError("module -> get_module_base: Unable to get module base") def get_init_size(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size @@ -82,7 +88,7 @@ class module(generic.GenericIntelProcess): ) def get_core_size(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_TEXT"]].size @@ -105,7 +111,7 @@ class module(generic.GenericIntelProcess): ) def get_module_core(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base except KeyError: @@ -120,7 +126,7 @@ class module(generic.GenericIntelProcess): raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base except KeyError: From 3b5fde94208bbfc98c7b2a147cdd2341427b7097 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 21 Feb 2024 12:58:24 +0000 Subject: [PATCH 41/76] Linux: update module extension logic for choosing the correct values for different kernel versions --- .../symbols/linux/extensions/__init__.py | 60 +++++++++---------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 8d11374c9..751c80cb5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -52,43 +52,41 @@ class module(generic.GenericIntelProcess): return self._mod_mem_type def get_module_base(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base - except: + except KeyError: raise AttributeError( "module -> get_module_base: Unable to get module base. Cannot read base from MOD_TEXT." ) - else: - if self.has_member("core_layout"): - return self.core_layout.base - elif self.has_member("module_core"): - return self.module_core + elif self.has_member("core_layout"): + return self.core_layout.base + elif self.has_member("module_core"): + return self.module_core raise AttributeError("module -> get_module_base: Unable to get module base") def get_init_size(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size ) - except: + except KeyError: raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module. Cannot read size of MOD_INIT_TEXT, MOD_INIT_DATA, and MOD_INIT_RODATA" ) - else: - if self.has_member("init_layout"): - return self.init_layout.size - elif self.has_member("init_size"): - return self.init_size + elif self.has_member("init_layout"): + return self.init_layout.size + elif self.has_member("init_size"): + return self.init_size raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module" ) def get_core_size(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_TEXT"]].size @@ -100,44 +98,40 @@ class module(generic.GenericIntelProcess): raise AttributeError( "module -> get_core_size: Unable to determine core size of module. Cannot read size of MOD_TEXT, MOD_DATA, MOD_RODATA, and MOD_RO_AFTER_INIT." ) - - else: - if self.has_member("core_layout"): - return self.core_layout.size - elif self.has_member("core_size"): - return self.core_size + elif self.has_member("core_layout"): + return self.core_layout.size + elif self.has_member("core_size"): + return self.core_size raise AttributeError( "module -> get_core_size: Unable to determine core size of module" ) def get_module_core(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base except KeyError: raise AttributeError( "module -> get_module_core: Unable to get module core. Cannot read base from MOD_TEXT." ) - else: - if self.has_member("core_layout"): - return self.core_layout.base - elif self.has_member("module_core"): - return self.module_core + elif self.has_member("core_layout"): + return self.core_layout.base + elif self.has_member("module_core"): + return self.module_core raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base except KeyError: raise AttributeError( "module -> get_module_core: Unable to get module init. Cannot read base from MOD_INIT_TEXT." ) - else: - if self.has_member("init_layout"): - return self.init_layout.base - elif self.has_member("module_init"): - return self.module_init + elif self.has_member("init_layout"): + return self.init_layout.base + elif self.has_member("module_init"): + return self.module_init raise AttributeError("module -> get_module_init: Unable to get module init") def get_name(self): From 77675d584d2e07df733c0700c8a3b7dd974252c0 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 21 Feb 2024 14:26:48 +0000 Subject: [PATCH 42/76] Update patch version --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 9aaafb933..7764dc44c 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 6 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From a82b93523de768b68fed529064113340920e52fe Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 22 Feb 2024 09:47:23 +0000 Subject: [PATCH 43/76] Linux: psscan tiny update so that memory_layer_name is reused --- volatility3/framework/plugins/linux/psscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 462577e58..40784a647 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -28,7 +28,7 @@ class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -139,7 +139,7 @@ class PsScan(interfaces.plugins.PluginInterface): kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" ) memory_layer_name = kernel_layer.dependencies[0] - memory_layer = context.layers[kernel_layer.dependencies[0]] + memory_layer = context.layers[memory_layer_name] # scan the memory_layer for these needles for address, _ in memory_layer.scan( From 0899ba8d7fedece67b520c8a2a31c45868177ce1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 28 Feb 2024 23:09:46 +0000 Subject: [PATCH 44/76] Documentation: Fix up syntax errors involving * character --- volatility3/framework/interfaces/plugins.py | 2 +- volatility3/framework/plugins/linux/kmsg.py | 4 +++- volatility3/framework/plugins/linux/pslist.py | 6 +++--- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- volatility3/framework/plugins/windows/mftscan.py | 4 ++-- .../framework/symbols/linux/extensions/__init__.py | 8 ++++---- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 29395aadf..697e4cdc3 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -60,7 +60,7 @@ class FileHandlerInterface(io.RawIOBase): @staticmethod def sanitize_filename(filename: str) -> str: """Sanititizes the filename to ensure only a specific whitelist of characters is allowed through""" - allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]\{\}!$%^:#~?<>,|" + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]{}!$%^:#~?<>,|" result = "" for char in filename: if char in allowed: diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index dd707a7ff..c5e0fc302 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -198,7 +198,9 @@ class ABCKmsg(ABC): class Kmsg_pre_3_5(ABCKmsg): """The kernel ring buffer (log_buf) is a char array that sequentially stores log lines, each separated by newline (LF) characters. i.e: - <6>[ 9565.250411] line1!\n<6>[ 9565.250412] line2\n... + + <6>[ 9565.250411] line1!\\n<6>[ 9565.250412] line2\\n... + """ @classmethod diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 771040dc0..046ee43d8 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -83,11 +83,11 @@ class PsList(interfaces.plugins.PluginInterface): cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False ) -> Tuple[int, int, int, str]: """Extract the fields needed for the final output + Args: task: A task object from where to get the fields. - decorate_comm: If True, it decorates the comm string of - - User threads: in curly brackets, - - Kernel threads: in square brackets + decorate_comm: If True, it decorates the comm string of user threads in curly brackets, + and of Kernel threads in square brackets. Defaults to False. Returns: A tuple with the fields to show in the plugin output. diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index e9c98a227..78217fbec 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -83,7 +83,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock: Kernel generic `sock` object Returns a tuple with: - sock: The respective kernel's \*_sock object for that socket family + sock: The respective kernel's \\*_sock object for that socket family sock_stat: A tuple with the source and destination (address and port) along with its state string socket_filter: A dictionary with information about the socket filter """ @@ -501,7 +501,7 @@ class Sockstat(plugins.PluginInterface): family: Socket family string (AF_UNIX, AF_INET, etc) sock_type: Socket type string (STREAM, DGRAM, etc) protocol: Protocol string (UDP, TCP, etc) - sock_fields: A tuple with the \*_sock object, the sock stats and the extended info dictionary + sock_fields: A tuple with the \\*_sock object, the sock stats and the extended info dictionary """ vmlinux = context.modules[symbol_table] diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 91a2e9152..85c072037 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -38,7 +38,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\*|BAAD/"} + {"yara_rules": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File @@ -197,7 +197,7 @@ class ADS(interfaces.plugins.PluginInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\*|BAAD/"} + {"yara_rules": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 751c80cb5..588ce3874 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1137,17 +1137,17 @@ class vfsmount(objects.StructType): """Helper to make sure it is comparing two pointers to 'vfsmount'. Depending on the kernel version, the calling object (self) could be - a 'vfsmount \*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust + a 'vfsmount \\*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust in the framework "auto" dereferencing ability to assure that when we reach this point 'self' will be a 'vfsmount' already and self.vol.offset - a 'vfsmount \*' and not a 'vfsmount \*\*'. The argument must be a 'vfsmount \*'. + a 'vfsmount \\*' and not a 'vfsmount \\*\\*'. The argument must be a 'vfsmount \\*'. Typically, it's called from do_get_path(). Args: - vfsmount_ptr (vfsmount \*): A pointer to a 'vfsmount' + vfsmount_ptr (vfsmount *): A pointer to a 'vfsmount' Raises: - exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \*' + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \\*' Returns: bool: 'True' if the given argument points to the the same 'vfsmount' From 338106dfe8e0667a07b0ab0ba4d52fbf5f4d51e7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 14:46:35 +1100 Subject: [PATCH 45/76] Move the memory page parameters to the Intel layer --- volatility3/framework/constants/linux/__init__.py | 5 ----- volatility3/framework/layers/intel.py | 12 ++++++++++++ volatility3/framework/plugins/linux/elfs.py | 14 +++++--------- .../framework/symbols/linux/extensions/__init__.py | 2 +- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 5e82e580e..3eabc2341 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -9,11 +9,6 @@ from enum import IntEnum KERNEL_NAME = "__kernel__" -# arch/x86/include/asm/page_types.h -PAGE_SHIFT = 12 -PAGE_SIZE = 1 << PAGE_SHIFT -PAGE_MASK = ~(PAGE_SIZE - 1) - """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" # include/linux/sched.h diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index ae477854d..75e561b33 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -67,6 +67,12 @@ class Intel(linear.LinearlyMappedLayer): math.ceil(math.log2(struct.calcsize(self._entry_format))) ) + @classproperty + @functools.lru_cache() + def page_shift(cls) -> int: + """Page shift for the intel memory layers.""" + return cls._page_size_in_bits + @classproperty @functools.lru_cache() def page_size(cls) -> int: @@ -76,6 +82,12 @@ class Intel(linear.LinearlyMappedLayer): """ return 1 << cls._page_size_in_bits + @classproperty + @functools.lru_cache() + def page_mask(cls) -> int: + """Page mask for the intel memory layers.""" + return ~(cls.page_size - 1) + @classproperty @functools.lru_cache() def bits_per_register(cls) -> int: diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 7171a6616..43cd6bb8b 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -14,11 +14,7 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux.extensions import elf -from volatility3.framework.constants.linux import ( - PAGE_SIZE, - PAGE_MASK, - ELF_MAX_EXTRACTION_SIZE, -) +from volatility3.framework.constants.linux import ELF_MAX_EXTRACTION_SIZE from volatility3.plugins.linux import pslist @@ -106,11 +102,11 @@ class Elfs(plugins.PluginInterface): # Use complete memory pages for dumping # If start isn't a multiple of a page, stick to the highest multiple < start # If end isn't a multiple of a page, stick to the lowest multiple > end - if start % PAGE_SIZE: - start = start & PAGE_MASK + if start % proc_layer.page_size: + start = start & proc_layer.page_mask - if end % PAGE_SIZE: - end = (end & PAGE_MASK) + PAGE_SIZE + if end % proc_layer.page_size: + end = (end & proc_layer.page_mask) + proc_layer.page_size real_size = end - start diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0a3db9fcd..1faafc267 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -640,7 +640,7 @@ class vm_area_struct(objects.StructType): elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True elif proclayer and "x" in flags_str: - for i in range(self.vm_start, self.vm_end, 1 << constants.linux.PAGE_SHIFT): + for i in range(self.vm_start, self.vm_end, proclayer.page_size): try: if proclayer.is_dirty(i): vollog.warning( From d5a0543a2d2d950c6743455f3f0ffa0b5afc2097 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 15:34:20 +1100 Subject: [PATCH 46/76] Use the ELF class constant instead of hardcoding a value. Fixed some f-strings --- volatility3/framework/layers/xen.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index 927b30430..e7aa0ccec 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -5,6 +5,7 @@ from typing import Optional from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.layers import elf from volatility3.framework.symbols import intermed +from volatility3.framework.constants.linux import ELF_CLASS vollog = logging.getLogger(__name__) @@ -14,7 +15,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): _header_struct = struct.Struct(" Date: Thu, 29 Feb 2024 15:34:48 +1100 Subject: [PATCH 47/76] Update author and modification time --- volatility3/framework/symbols/linux/elf.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/elf.json b/volatility3/framework/symbols/linux/elf.json index e0a95bbba..79a96e07a 100644 --- a/volatility3/framework/symbols/linux/elf.json +++ b/volatility3/framework/symbols/linux/elf.json @@ -1087,8 +1087,8 @@ "metadata": { "producer": { "version": "0.0.2", - "name": "ikelos-by-hand", - "datetime": "2019-10-21T22:52:00" + "name": "gcmoreira-by-hand", + "datetime": "2024-02-19T14:37:00" }, "format": "6.1.0" } From 9b0915dc85470df78bc739148093bb72d659297c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 15:36:46 +1100 Subject: [PATCH 48/76] Add logging for unknown ELF types --- volatility3/framework/layers/elf.py | 4 ++ volatility3/framework/plugins/linux/elfs.py | 4 ++ .../framework/symbols/linux/extensions/elf.py | 66 ++++++++++++++++--- 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index 6bd5c2d63..81f3c3634 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -55,6 +55,10 @@ class Elf64Layer(segmented.SegmentedLayer): try: ptype = phdr.p_type.description except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue if ( diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 43cd6bb8b..6820576dc 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -93,6 +93,10 @@ class Elfs(plugins.PluginInterface): if phdr.p_type.description != "PT_LOAD": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue start = phdr.p_vaddr diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 828370fe8..2cf5c3d4e 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -165,6 +165,10 @@ class elf(objects.StructType): if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue for dsec in phdr.dynamic_sections(): @@ -172,6 +176,10 @@ class elf(objects.StructType): if dsec.d_tag.description != "DT_PLTGOT": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF dynamic section type: {dsec.d_tag}", + ) continue got_start = dsec.d_ptr @@ -186,16 +194,27 @@ class elf(objects.StructType): layer_name=self.vol.layer_name, ) if not link_map_ptr: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid ELF link map pointer at 0x{link_map_addr:x}", + ) continue linkmap_symname = ( elf_symbol_table + constants.BANG + self._type_prefix + "LinkMap" ) - link_map = self._context.object( - object_type=linkmap_symname, - offset=link_map_ptr, - layer_name=self.vol.layer_name, - ) + try: + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map_ptr, + layer_name=self.vol.layer_name, + ) + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid ELF link map address at 0x{link_map_ptr:x}", + ) + continue while link_map and link_map.vol.offset != 0: if link_map.vol.offset in link_maps_seen: @@ -204,11 +223,18 @@ class elf(objects.StructType): yield link_map - link_map = self._context.object( - object_type=linkmap_symname, - offset=link_map.l_next, - layer_name=self.vol.layer_name, - ) + try: + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map.l_next, + layer_name=self.vol.layer_name, + ) + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"ELF link map linked list is corrupt at 0x{self.vol.offset:x}", + ) + break def _find_symbols(self): dt_strtab = None @@ -221,6 +247,10 @@ class elf(objects.StructType): if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue # This section contains pointers to the strtab, symtab, and strent sections @@ -228,6 +258,10 @@ class elf(objects.StructType): try: dtag = dsec.d_tag.description except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF dynamic section type: {dsec.d_tag}", + ) continue if dtag == "DT_STRTAB": @@ -353,6 +387,10 @@ class elf_phdr(objects.StructType): except ValueError: # Unknown ELF object file type. Anyway, if the ELF object file type is not a # shared object (ET_DYN), the virtual address is 'p_vaddr'. + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF object type: {self._parent_e_type}", + ) pass return offset @@ -365,6 +403,10 @@ class elf_phdr(objects.StructType): except ValueError: # If the value is outside the ones declared in the enumeration, an # exception is raised + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {self.p_type}", + ) return None # the buffer of array starts at elf_base + our virtual address ( offset ) @@ -398,6 +440,10 @@ class elf_linkmap(objects.StructType): buf = self._context.layers.read(self.vol.layer_name, self.l_name, 256) except exceptions.PagedInvalidAddressException: # Protection against memory smear + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid l_name address for ELF link map at 0x{self.vol.offset:x}", + ) return None idx = buf.find(b"\x00") From 3c9af096e15402a403b41b29a79ab3c74cc617f1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 16:17:47 +1100 Subject: [PATCH 49/76] Add missing PAGE_SHIFT replacement --- volatility3/framework/symbols/linux/extensions/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1faafc267..04cfa099f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -611,7 +611,8 @@ class vm_area_struct(objects.StructType): def get_page_offset(self) -> int: if self.vm_file == 0: return 0 - return self.vm_pgoff << constants.linux.PAGE_SHIFT + parent_layer = self._context.layers[self.vol.layer_name] + return self.vm_pgoff << parent_layer.page_shift def get_name(self, context, task): if self.vm_file != 0: From b681438ddcfdc33defa34d15cf48bd9ce4c9ca58 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 16:21:34 +1100 Subject: [PATCH 50/76] Remove unnecessary 'pass' statement. --- volatility3/framework/symbols/linux/extensions/elf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 2cf5c3d4e..eadcbbae0 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -391,7 +391,6 @@ class elf_phdr(objects.StructType): constants.LOGLEVEL_VVVV, f"Skipping unknown ELF object type: {self._parent_e_type}", ) - pass return offset From a8e10828273fb8cfa74743137ba70fd730e42619 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 29 Feb 2024 14:36:46 +0100 Subject: [PATCH 51/76] restore pslist, add queue_head_t type class --- volatility3/framework/plugins/mac/pslist.py | 8 ++------ volatility3/framework/symbols/mac/__init__.py | 4 +++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 9835644b8..9b570f3f9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -49,9 +49,7 @@ class PsList(interfaces.plugins.PluginInterface): ] @classmethod - def get_list_tasks( - cls, method: str - ) -> Callable[ + def get_list_tasks(cls, method: str) -> Callable[ [interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface], ]: @@ -188,9 +186,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object( - object_type="queue_entry", offset=kernel.get_symbol("tasks").address - ) + queue_entry = kernel.object_from_symbol(symbol_name="tasks") seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index 56ac96633..be4fe8cd1 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -21,12 +21,14 @@ class MacKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("vm_map_object", extensions.vm_map_object) self.set_type_class("socket", extensions.socket) self.set_type_class("inpcb", extensions.inpcb) - self.set_type_class("queue_entry", extensions.queue_entry) self.set_type_class("ifnet", extensions.ifnet) self.set_type_class("sockaddr_dl", extensions.sockaddr_dl) self.set_type_class("sockaddr", extensions.sockaddr) self.set_type_class("sysctl_oid", extensions.sysctl_oid) self.set_type_class("kauth_scope", extensions.kauth_scope) + # https://developer.apple.com/documentation/kernel/queue_head_t + self.set_type_class("queue_entry", extensions.queue_entry) + self.set_type_class("queue_head_t", extensions.queue_entry) class MacUtilities(interfaces.configuration.VersionableInterface): From d56ccbfd0b749d8f2ad69764658bf8f3856b8d21 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 14:18:30 +0100 Subject: [PATCH 52/76] allow specifying integers as 0x in ListRequirement --- volatility3/cli/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 3de7d8f8f..a67762a31 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,7 +827,11 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - additional["type"] = requirement.element_type + # Allow a list of integers, specified with convenient 0x hexadecimal format + if requirement.element_type == int: + additional["type"] = lambda x: int(x, 0) + else: + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 3040bc5fc21f31e27f75ac8c3ec359e43454d098 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 14:20:13 +0100 Subject: [PATCH 53/76] add --dump mechanism, similarly to linux.proc --- .../framework/plugins/mac/proc_maps.py | 173 +++++++++++++++++- 1 file changed, 169 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 781b3ed66..204d414e4 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -2,17 +2,23 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.mac import pslist +from typing import Callable, Generator, Type, Optional +import logging + +vollog = logging.getLogger(__name__) class Maps(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" _required_framework_version = (2, 0, 0) + _version = (1, 1, 0) + MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod def get_requirements(cls): @@ -31,14 +37,155 @@ class Maps(interfaces.plugins.PluginInterface): element_type=int, optional=True, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed memory segments", + default=False, + optional=True, + ), + requirements.ListRequirement( + name="address", + description="Process virtual memory addresses to include " + "(all other VMA sections are excluded). This can be any " + "virtual address within the VMA section. Virtual addresses " + "must be separated by a space.", + element_type=int, + optional=True, + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size for dumped VMA sections " + "(all the bigger sections will be ignored)", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), ] + @classmethod + def list_vmas( + cls, + task: interfaces.objects.ObjectInterface, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: True, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Lists the Virtual Memory Areas of a specific process. + + Args: + task: task object from which to list the vma + filter_func: Function to take a vma and return False if it should be filtered out + + Returns: + Yields vmas based on the task and filtered based on the filter function + """ + for vma in task.get_map_iter(): + if filter_func(vma): + yield vma + else: + vollog.debug( + f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.p_pid} due to filter_func" + ) + + @classmethod + def vma_dump( + cls, + context: interfaces.context.ContextInterface, + task: interfaces.objects.ObjectInterface, + vm_start: int, + vm_end: int, + open_method: Type[interfaces.plugins.FileHandlerInterface], + maxsize: int = MAXSIZE_DEFAULT, + ) -> Optional[interfaces.plugins.FileHandlerInterface]: + """Extracts the complete data for VMA as a FileInterface. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + task: an task_struct instance + vm_start: The start virtual address from the vma to dump + vm_end: The end virtual address from the vma to dump + open_method: class to provide context manager for opening the file + maxsize: Max size of VMA section (default MAXSIZE_DEFAULT) + + Returns: + An open FileInterface object containing the complete data for the task or None in the case of failure + """ + pid = task.p_pid + + try: + proc_layer_name = task.add_process_layer() + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + pid, excp.invalid_address, excp.layer_name + ) + ) + return None + vm_size = vm_end - vm_start + + # check if vm_size is negative, this should never happen. + if vm_size < 0: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is negative." + ) + return None + # check if vm_size is larger than the maxsize limit, and therefore is not saved out. + if maxsize <= vm_size: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is larger than maxsize limit of {maxsize}" + ) + return None + proc_layer = context.layers[proc_layer_name] + file_name = f"pid.{pid}.vma.{vm_start:#x}-{vm_end:#x}.dmp" + try: + file_handle = open_method(file_name) + chunk_size = 1024 * 1024 * 10 + offset = vm_start + while offset < vm_start + vm_size: + to_read = min(chunk_size, vm_start + vm_size - offset) + data = proc_layer.read(offset, to_read, pad=True) + file_handle.write(data) + offset += to_read + except Exception as excp: + vollog.debug(f"Unable to dump virtual memory {file_name}: {excp}") + return None + return file_handle + def _generator(self, tasks): + address_list = self.config.get("address", None) + if not address_list: + # do not filter as no address_list was supplied + vma_filter_func = lambda _: True + else: + # filter for any vm_start that matches the supplied address config + def vma_filter_function(task: interfaces.objects.ObjectInterface) -> bool: + addrs_in_vma = [ + addr + for addr in address_list + if task.links.start <= addr <= task.links.end + ] + + # if any of the user supplied addresses would fall within this vma return true + if addrs_in_vma: + return True + else: + return False + + vma_filter_func = vma_filter_function + for task in tasks: process_name = utility.array_to_string(task.p_comm) process_pid = task.p_pid - for vma in task.get_map_iter(): + for vma in self.list_vmas(task, filter_func=vma_filter_func): + try: + vm_start = vma.links.start + vm_end = vma.links.end + except AttributeError: + vollog.debug( + f"Unable to find the vm_start and vm_end for vma at {vma.vol.offset:#x} for pid {process_pid}" + ) + continue + path = vma.get_path( self.context, self.context.modules[self.config["kernel"]].symbol_table_name, @@ -46,15 +193,32 @@ class Maps(interfaces.plugins.PluginInterface): if path == "": path = vma.get_special_path() + file_output = "Disabled" + if self.config["dump"]: + file_output = "Error outputting file" + file_handle = self.vma_dump( + self.context, + task, + vm_start, + vm_end, + self.open, + self.config["maxsize"], + ) + + if file_handle: + file_handle.close() + file_output = file_handle.preferred_filename + yield ( 0, ( process_pid, process_name, - format_hints.Hex(vma.links.start), - format_hints.Hex(vma.links.end), + format_hints.Hex(vm_start), + format_hints.Hex(vm_end), vma.get_perms(), path, + file_output, ), ) @@ -72,6 +236,7 @@ class Maps(interfaces.plugins.PluginInterface): ("End", format_hints.Hex), ("Protection", str), ("Map Name", str), + ("File output", str), ], self._generator( list_tasks(self.context, self.config["kernel"], filter_func=filter_func) From 965ddcb674d30528792a404b819635c60c207f8e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 19:30:46 +0100 Subject: [PATCH 54/76] make queue_head_t optional, as it might not exist in all ISF --- volatility3/framework/symbols/mac/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index be4fe8cd1..83aebb13b 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -28,7 +28,7 @@ class MacKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("kauth_scope", extensions.kauth_scope) # https://developer.apple.com/documentation/kernel/queue_head_t self.set_type_class("queue_entry", extensions.queue_entry) - self.set_type_class("queue_head_t", extensions.queue_entry) + self.optional_set_type_class("queue_head_t", extensions.queue_entry) class MacUtilities(interfaces.configuration.VersionableInterface): From 2e835099b58b54af2f988355ae04cbfeff9cd13e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 20:19:31 +0100 Subject: [PATCH 55/76] fix wrong list_head comparison + better naming --- .../symbols/mac/extensions/__init__.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index c89b527e6..0a6bfbb90 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -490,22 +490,24 @@ class queue_entry(objects.StructType): for attr in ["next", "prev"]: with contextlib.suppress(exceptions.InvalidAddressException): - n = getattr(self, attr).dereference().cast(type_name) - - while n is not None and n.vol.offset != list_head: - if n.vol.offset in seen: + queue_element = getattr(self, attr).dereference().cast(type_name) + while ( + queue_element is not None + and queue_element.vol.offset != list_head.vol.offset + ): + if queue_element.vol.offset in seen: break - yield n + yield queue_element - seen.add(n.vol.offset) + seen.add(queue_element.vol.offset) yielded = yielded + 1 if yielded == max_size: - return + return None - n = ( - getattr(n.member(attr=member_name), attr) + queue_element = ( + getattr(queue_element.member(attr=member_name), attr) .dereference() .cast(type_name) ) From caf108b604be6925a2e5e1a4afc627da1fa943a5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 17:17:48 +0100 Subject: [PATCH 56/76] macOS dmesg plugin support --- volatility3/framework/plugins/mac/dmesg.py | 79 ++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 volatility3/framework/plugins/mac/dmesg.py diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py new file mode 100644 index 000000000..a006ff854 --- /dev/null +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -0,0 +1,79 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility + +vollog = logging.getLogger(__name__) + + +class Dmesg(interfaces.plugins.PluginInterface): + """Prints the kernel log buffer.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + ] + + @classmethod + def get_kernel_log_buffer( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str + ): + """ + Online documentation : + - https://github.com/apple-open-source/macos/blob/master/xnu/bsd/sys/msgbuf.h + - https://github.com/apple-open-source/macos/blob/ea4cd5a06831aca49e33df829d2976d6de5316ec/xnu/bsd/kern/subr_log.c#L751 + Volatility 2 plugin : + - https://github.com/volatilityfoundation/volatility/blob/master/volatility/plugins/mac/dmesg.py + """ + + kernel = context.modules[kernel_module_name] + if not kernel.has_symbol("msgbufp"): + vollog.error( + 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' + ) + return [] + + msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") + msgbufp = msgbufp_ptr.dereference() + msg_size = msgbufp.msg_size # max buffer size + msg_bufx = msgbufp.msg_bufx # write pointer + msg_bufc = msgbufp.msg_bufc + # msg_bufc is circular, meaning that if its size exceeds msg_size, + # msg_bufx will point to the beginning of the buffer and start overwriting. + msg_bufc_data: str = utility.pointer_to_string(msg_bufc, msg_size) + # Avoid OOB reads + msg_bufx = msg_bufx if msg_bufx <= msg_size else 0 + # We directly take into account the case where the write buffer did a loop, + # as older messages will start at msg_bufx offset (not overwritten yet). + dmesg = msg_bufc_data[msg_bufx:] + dmesg += msg_bufc_data[:msg_bufx] + + # Yield each line + for dmesg_line in dmesg.splitlines(): + yield (dmesg_line.strip(),) + + def _generator(self): + for value in self.get_kernel_log_buffer( + context=self.context, kernel_module_name=self.config["kernel"] + ): + yield (0, value) + + def run(self): + return renderers.TreeGrid( + [ + ("line", str), + ], + self._generator(), + ) From 8d79e3211ba2e8ea751c657b5da9f99a8b3a90c2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 17:30:51 +0100 Subject: [PATCH 57/76] prefer TypeError to vollog.error --- volatility3/framework/plugins/mac/dmesg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index a006ff854..b837b9db6 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -40,10 +40,9 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): - vollog.error( + raise TypeError( 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) - return [] msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") msgbufp = msgbufp_ptr.dereference() From 2f0bb6b297ffe601c563d646f09c80b8b0a4864f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 18:39:48 +0100 Subject: [PATCH 58/76] do not strip each line --- volatility3/framework/plugins/mac/dmesg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index b837b9db6..d4f2c869d 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -61,7 +61,7 @@ class Dmesg(interfaces.plugins.PluginInterface): # Yield each line for dmesg_line in dmesg.splitlines(): - yield (dmesg_line.strip(),) + yield (dmesg_line,) def _generator(self): for value in self.get_kernel_log_buffer( From cb6f8c507268dd399f24e762766b3ba42498799a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 4 Mar 2024 20:47:49 +1100 Subject: [PATCH 59/76] Rename methods to be private --- .../framework/plugins/linux/library_list.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py index ed5545347..062ed078e 100644 --- a/volatility3/framework/plugins/linux/library_list.py +++ b/volatility3/framework/plugins/linux/library_list.py @@ -43,7 +43,7 @@ class LibraryList(interfaces.plugins.PluginInterface): ), ] - def get_libdl_libraries( + def _get_libdl_libraries( self, proc_layer_name: str, vma_start: int ) -> interfaces.objects.ObjectInterface: """Get the ELF link map objects for the given VMA address @@ -81,7 +81,7 @@ class LibraryList(interfaces.plugins.PluginInterface): # Protection against memory smear in this VMA pass - def get_libdl_maps( + def _get_libdl_maps( self, task: interfaces.objects.ObjectInterface, proc_layer_name: str ) -> interfaces.objects.ObjectInterface: """Get the ELF link maps objects for a task @@ -96,14 +96,14 @@ class LibraryList(interfaces.plugins.PluginInterface): link_map_seen = set() for vma in task.mm.get_vma_iter(): - for link_map in self.get_libdl_libraries(proc_layer_name, vma.vm_start): + for link_map in self._get_libdl_libraries(proc_layer_name, vma.vm_start): if link_map.l_addr in link_map_seen: continue yield link_map link_map_seen.add(link_map.l_addr) - def get_task_libraries( + def _get_task_libraries( self, task: interfaces.objects.ObjectInterface ) -> Tuple[int, str]: """Get the task libraries from the ELF headers found within the memory maps @@ -118,13 +118,13 @@ class LibraryList(interfaces.plugins.PluginInterface): if not proc_layer_name: return - for elf_link_map in self.get_libdl_maps(task, proc_layer_name): + for elf_link_map in self._get_libdl_maps(task, proc_layer_name): name = elf_link_map.get_name() if not name: continue yield elf_link_map.l_addr, name - def get_tasks_libraries( + def _get_tasks_libraries( self, tasks: Iterable[interfaces.objects.ObjectInterface], ) -> Iterable[Tuple[str, int, int, str]]: @@ -139,7 +139,7 @@ class LibraryList(interfaces.plugins.PluginInterface): """ for task in tasks: task_name = utility.array_to_string(task.comm) - for linkmap_addr, linkmap_name in self.get_task_libraries(task): + for linkmap_addr, linkmap_name in self._get_task_libraries(task): yield task_name, task.tgid, linkmap_addr, linkmap_name def _format_fields(self, fields): @@ -149,7 +149,7 @@ class LibraryList(interfaces.plugins.PluginInterface): def _generator( self, tasks: Iterable[interfaces.objects.ObjectInterface] ) -> Iterable[Tuple[int, Tuple]]: - for fields in self.get_tasks_libraries(tasks): + for fields in self._get_tasks_libraries(tasks): yield 0, self._format_fields(fields) def run(self): From 373bf9dfaa466fdaba9635a78a6449679af502bf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 4 Mar 2024 18:33:03 +0000 Subject: [PATCH 60/76] Windows: Fix driverirp black issue --- volatility3/framework/plugins/windows/driverirp.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index f7eca0359..433c61ca2 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -115,9 +115,17 @@ class DriverIrp(interfaces.plugins.PluginInterface): ) if not module_found: - yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], - format_hints.Hex(address), renderers.NotAvailableValue(), renderers.NotAvailableValue())) - + yield ( + 0, + ( + format_hints.Hex(driver.vol.offset), + driver_name, + MAJOR_FUNCTIONS[i], + format_hints.Hex(address), + renderers.NotAvailableValue(), + renderers.NotAvailableValue(), + ), + ) def run(self): return renderers.TreeGrid( From 403804431ae0ae5118b13cb0f894edfb9cc03d73 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:11:17 +0100 Subject: [PATCH 61/76] prefer SymbolError to TypeError --- volatility3/framework/plugins/mac/dmesg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index d4f2c869d..12c241641 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -3,7 +3,7 @@ # import logging -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -40,7 +40,7 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): - raise TypeError( + raise exceptions.SymbolError( 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) From 1bf18dbd8a9f68381dfbdb46c4d76ab92cbe0a68 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:11:50 +0100 Subject: [PATCH 62/76] implicit pointer dereference --- volatility3/framework/plugins/mac/dmesg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index 12c241641..daa6617ad 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -44,8 +44,7 @@ class Dmesg(interfaces.plugins.PluginInterface): 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) - msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") - msgbufp = msgbufp_ptr.dereference() + msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") msg_size = msgbufp.msg_size # max buffer size msg_bufx = msgbufp.msg_bufx # write pointer msg_bufc = msgbufp.msg_bufc From 98705e110df4691dc974210f0818621db2f27bbf Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:12:12 +0100 Subject: [PATCH 63/76] more specific msg_bufx comment --- volatility3/framework/plugins/mac/dmesg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index daa6617ad..7f817e605 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -46,7 +46,7 @@ class Dmesg(interfaces.plugins.PluginInterface): msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") msg_size = msgbufp.msg_size # max buffer size - msg_bufx = msgbufp.msg_bufx # write pointer + msg_bufx = msgbufp.msg_bufx # write index of the msg_bufc circular buffer msg_bufc = msgbufp.msg_bufc # msg_bufc is circular, meaning that if its size exceeds msg_size, # msg_bufx will point to the beginning of the buffer and start overwriting. From 69a6c7d5b7cbfb99e8ec70d05dac95a6dea71713 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:16:55 +0100 Subject: [PATCH 64/76] revert int as hex format commit --- volatility3/cli/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index a67762a31..3de7d8f8f 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,11 +827,7 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - # Allow a list of integers, specified with convenient 0x hexadecimal format - if requirement.element_type == int: - additional["type"] = lambda x: int(x, 0) - else: - additional["type"] = requirement.element_type + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 9914f339dfc1cdd511fc3fad676b92b06520243e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:19:15 +0100 Subject: [PATCH 65/76] simplify addrs_in_vma check --- volatility3/framework/plugins/mac/proc_maps.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 204d414e4..fe5179dfa 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -165,10 +165,7 @@ class Maps(interfaces.plugins.PluginInterface): ] # if any of the user supplied addresses would fall within this vma return true - if addrs_in_vma: - return True - else: - return False + return bool(addrs_in_vma) vma_filter_func = vma_filter_function From f4b12c6406e2f1accc147069403e363e7c6a01dc Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:26:18 +0100 Subject: [PATCH 66/76] allow ints in the 0x format in ListRequirement --- volatility3/cli/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 3de7d8f8f..35e977eba 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,7 +827,11 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - additional["type"] = requirement.element_type + # Allow a list of integers, specified with the convenient 0x hexadecimal format + if requirement.element_type == int: + additional["type"] = lambda x: int(x, 0) + else: + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 1f9a983f492ff7983030b8fa27b509106594f735 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:38:01 +0100 Subject: [PATCH 67/76] correct use of SymbolError --- volatility3/framework/plugins/mac/dmesg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index 7f817e605..f9f06a666 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -41,7 +41,9 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): raise exceptions.SymbolError( - 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' + "msgbufp", + kernel.symbol_table_name, + 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.', ) msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") From 9edf33b7212d46682b48dbb40b744f198f8741a8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 11 Mar 2024 21:20:29 +0000 Subject: [PATCH 68/76] Layers: Improve logging on crashdump layer --- volatility3/framework/layers/crash.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 8efd4f7c7..8fd0abbcc 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -261,11 +261,15 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): progress_callback: constants.ProgressCallback = None, ) -> Optional[interfaces.layers.DataLayerInterface]: for layer in [WindowsCrashDump32Layer, WindowsCrashDump64Layer]: - with contextlib.suppress(WindowsCrashDumpFormatException): + try: layer.check_header(context.layers[layer_name]) new_name = context.layers.free_layer_name(layer.__name__) context.config[ interfaces.configuration.path_join(new_name, "base_layer") ] = layer_name return layer(context, new_name, new_name) + except (WindowsCrashDump32Layer, WindowsCrashDump64Layer) as excp: + vollog.log( + constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" + ) return None From 084ea38f84b36e8db2594fc31f1c2d61b3b7c6ce Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 12 Mar 2024 00:13:02 +0000 Subject: [PATCH 69/76] Layers: Fix up typo in recent crashdump patch --- volatility3/framework/layers/crash.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 8fd0abbcc..5598fc2e4 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -268,8 +268,8 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): interfaces.configuration.path_join(new_name, "base_layer") ] = layer_name return layer(context, new_name, new_name) - except (WindowsCrashDump32Layer, WindowsCrashDump64Layer) as excp: + except WindowsCrashDumpFormatException as excp: vollog.log( constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" - ) + )\ return None From 8dbc64f4a8678455adbac80ac716dfa62b3aecb2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 12 Mar 2024 00:14:30 +0000 Subject: [PATCH 70/76] Layers: Fix up typo in recent crashdump patch - take 2 --- volatility3/framework/layers/crash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 5598fc2e4..042b18ddc 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -271,5 +271,5 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): except WindowsCrashDumpFormatException as excp: vollog.log( constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" - )\ + ) return None From f6495d3d986e01fb96789a88aa89cc2c8e40cb71 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 16 Mar 2024 08:42:53 +0000 Subject: [PATCH 71/76] Documentation: Improve logging level docstrings --- volatility3/framework/constants/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 921f602fd..3c5e0eb2f 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -59,14 +59,18 @@ PACKAGE_VERSION = ( AUTOMAGIC_CONFIG_PATH = "automagic" """The root section within the context configuration for automagic values""" +LOGLEVEL_INFO = 20 +"""Logging level for information data, showed when use the requests any logging: -v""" +LOGLEVEL_DEBUG = 10 +"""Logging level for debugging data, showed when the user requests more logging detail: -vv""" LOGLEVEL_V = 9 -"""Logging level for a single -v""" +"""Logging level for the lowest "extra" level of logging: -vvv""" LOGLEVEL_VV = 8 -"""Logging level for -vv""" +"""Logging level for two levels of detail: -vvvv""" LOGLEVEL_VVV = 7 -"""Logging level for -vvv""" +"""Logging level for three levels of detail: -vvvvv""" LOGLEVEL_VVVV = 6 -"""Logging level for -vvvv""" +"""Logging level for four levels of detail: -vvvvvv""" CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" From 436d355ef301d9cd73883b5129f8b9f744c2ecb7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 18:44:09 +0100 Subject: [PATCH 72/76] incrementally order extra log levels --- volatility3/cli/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 35e977eba..ec9918a65 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -264,6 +264,17 @@ class CommandLine: file_logger.setFormatter(file_formatter) rootlog.addHandler(file_logger) vollog.info("Logging started") + + for level, level_value in enumerate( + [ + constants.LOGLEVEL_V, + constants.LOGLEVEL_VV, + constants.LOGLEVEL_VVV, + constants.LOGLEVEL_VVVV, + ] + ): + logging.addLevelName(level_value, f"DETAIL {level+1}") + if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None From 33d653e7a66b8b96350a3568c6d156eadb0cf878 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 18:44:19 +0100 Subject: [PATCH 73/76] use logging explicit constants --- volatility3/cli/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index ec9918a65..116cde114 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -278,9 +278,9 @@ class CommandLine: if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None - console.setLevel(30 - (partial_args.verbosity * 10)) + console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: - console.setLevel(10 - (partial_args.verbosity - 2)) + console.setLevel(logging.DEBUG - (partial_args.verbosity - 2)) for level, msg in delayed_logs: vollog.log(level, msg) From 745491c846314fb37231c892e2cfab9eed4275f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:02 +0100 Subject: [PATCH 74/76] put extra log level ordering in a function --- volatility3/cli/__init__.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 116cde114..457a49311 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -265,16 +265,7 @@ class CommandLine: rootlog.addHandler(file_logger) vollog.info("Logging started") - for level, level_value in enumerate( - [ - constants.LOGLEVEL_V, - constants.LOGLEVEL_VV, - constants.LOGLEVEL_VVV, - constants.LOGLEVEL_VVVV, - ] - ): - logging.addLevelName(level_value, f"DETAIL {level+1}") - + self.order_extra_verbose_levels() if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None @@ -706,6 +697,17 @@ class CommandLine: ) context.config[extended_path] = value + def order_extra_verbose_levels(self): + for level, level_value in enumerate( + [ + constants.LOGLEVEL_V, + constants.LOGLEVEL_VV, + constants.LOGLEVEL_VVV, + constants.LOGLEVEL_VVVV, + ] + ): + logging.addLevelName(level_value, f"DETAIL {level+1}") + def file_handler_class_factory(self, direct=True): output_dir = self.output_dir From 7ff5f57e5d4485ad2e5f2ad4dc78017880442c05 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:28 +0100 Subject: [PATCH 75/76] use logging explicit constants --- volatility3/cli/volshell/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 9e74acfec..998c8245b 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -198,9 +198,9 @@ class VolShell(cli.CommandLine): vollog.info("Logging started") if partial_args.verbosity < 3: - console.setLevel(30 - (partial_args.verbosity * 10)) + console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: - console.setLevel(10 - (partial_args.verbosity - 2)) + console.setLevel(logging.DEBUG - (partial_args.verbosity - 2)) for level, msg in delayed_logs: vollog.log(level, msg) From de4a3359982b67b74a354001d380502ff41804d1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:45 +0100 Subject: [PATCH 76/76] call extra verbose level ordering --- volatility3/cli/volshell/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 998c8245b..035ed9b2e 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -197,6 +197,7 @@ class VolShell(cli.CommandLine): vollog.addHandler(file_logger) vollog.info("Logging started") + self.order_extra_verbose_levels() if partial_args.verbosity < 3: console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: