From b5c3ab171e4d3554e39efd8a07dbd9f5dceafccc Mon Sep 17 00:00:00 2001 From: cpuu Date: Sat, 19 Nov 2022 15:01:39 +0900 Subject: [PATCH 001/101] Add macos tutorial --- doc/source/getting-started-macos-tutorial.rst | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 doc/source/getting-started-macos-tutorial.rst diff --git a/doc/source/getting-started-macos-tutorial.rst b/doc/source/getting-started-macos-tutorial.rst new file mode 100644 index 000000000..6dd144ac3 --- /dev/null +++ b/doc/source/getting-started-macos-tutorial.rst @@ -0,0 +1,152 @@ +macOS Tutorial +============== + +This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite. + +Acquiring memory +---------------- + +Volatility3 does not provide the ability to acquire memory. The example below is an open source tool. Other commercial tools are also available. + +* `osxpmem `_ + + + +Procedure to create symbol tables for macOS +-------------------------------------------- + +To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. + +.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , + which is built and maintained by `volatilityfoundation `_. + After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/mac``. + If necessary create a mac directory under the symbols directory (this will become unnecessary in future versions). + + +Listing plugins +--------------- + +The following is a sample of the macOS plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. + +.. code-block:: shell-session + + $ python3 vol.py --help | grep -i mac. | head -n 5 + mac.bash.Bash Recovers bash command history from memory. + mac.check_syscall.Check_syscall + mac.check_sysctl.Check_sysctl + mac.check_trap_table.Check_trap_table + +.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of macOS plugins. + + +Using plugins +------------- + +The following is the syntax to run the volatility CLI. + +.. code-block:: shell-session + + $ python3 vol.py -f + + +Example +------- + +banners +~~~~~~~ + +In this example we will be using a memory dump from the Securinets CTF Quals 2019 Challenge called Contact_me. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge. +Thanks go to `stuxnet `_ for providing this memory dump and `writeup `_. + + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me banners.Banners + + Volatility 3 Framework 2.1.0 + + Progress: 100.00 PDB scanning finished + Offset Banner + + 0x4d2c7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0xb42b180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0xcda9100 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x1275e7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x1284fba4 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x34ad0180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + + +The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols/mac`` directory. + +linux.pslist +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.pslist + + Volatility 3 Framework 2.1.0 Stacking attempts finished + + PID PPID COMM + + 0 0 kernel_task + 1 0 launchd + 35 1 UserEventAgent + 38 1 kextd + 39 1 fseventsd + 37 1 uninstalld + 45 1 configd + 46 1 powerd + 52 1 logd + 58 1 warmd + ..... + +``mac.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. + +mac.pstree +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.pstree + Volatility 3 Framework 2.1.0 + Progress: 100.00 Stacking attempts finished + PID PPID COMM + + 35 1 UserEventAgent + 38 1 kextd + 39 1 fseventsd + 37 1 uninstalld + 204 1 softwareupdated + * 449 204 SoftwareUpdateCo + 337 1 system_installd + * 455 337 update_dyld_shar + +``mac.pstree`` helps us to display the parent child relationships between processes. + +mac.ifconfig +~~~~~~~~~~ + +we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.ifconfig + + Volatility 3 Framework 2.1.0 + Progress: 100.00 Stacking attempts finished + Interface IP Address Mac Address Promiscuous + + lo0 False + lo0 127.0.0.1 False + lo0 ::1 False + lo0 fe80:1::1 False + gif0 False + stf0 False + en0 00:0C:29:89:8B:F0 00:0C:29:89:8B:F0 False + en0 fe80:4::10fb:c89d:217f:52ae 00:0C:29:89:8B:F0 False + en0 192.168.140.128 00:0C:29:89:8B:F0 False + utun0 False + utun0 fe80:5::2a95:bb15:87e3:977c False From 17bcc8d47e372e5066d07b98b0059d1d2b7ee548 Mon Sep 17 00:00:00 2001 From: cpuu Date: Sat, 19 Nov 2022 23:11:45 +0900 Subject: [PATCH 002/101] typo --- doc/source/getting-started-macos-tutorial.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/getting-started-macos-tutorial.rst b/doc/source/getting-started-macos-tutorial.rst index 6dd144ac3..0d95a1700 100644 --- a/doc/source/getting-started-macos-tutorial.rst +++ b/doc/source/getting-started-macos-tutorial.rst @@ -131,6 +131,8 @@ mac.ifconfig ~~~~~~~~~~ we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. + + .. code-block:: shell-session $ python3 vol.py -f contact_me mac.ifconfig From 53870c64d1553e43e283bff3fbc08cc0249d4d0e Mon Sep 17 00:00:00 2001 From: cpuu Date: Sat, 19 Nov 2022 23:15:29 +0900 Subject: [PATCH 003/101] Add macos tutorial --- doc/source/getting-started-macos-tutorial.rst | 2 +- doc/source/index.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/getting-started-macos-tutorial.rst b/doc/source/getting-started-macos-tutorial.rst index 0d95a1700..76f77d0f3 100644 --- a/doc/source/getting-started-macos-tutorial.rst +++ b/doc/source/getting-started-macos-tutorial.rst @@ -17,7 +17,7 @@ Procedure to create symbol tables for macOS To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. -.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , +.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , which is built and maintained by `volatilityfoundation `_. After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/mac``. If necessary create a mac directory under the symbols directory (this will become unnecessary in future versions). diff --git a/doc/source/index.rst b/doc/source/index.rst index 9b1d05858..e096731c7 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -26,6 +26,7 @@ There is also some information to get you started quickly: getting-started-linux-tutorial getting-started-windows-tutorial + getting-started-macos-tutorial .. toctree:: From fc0fa30f9e3fda0479f0fd9761b33550d705ecc2 Mon Sep 17 00:00:00 2001 From: cpuu Date: Sun, 20 Nov 2022 10:18:47 +0900 Subject: [PATCH 004/101] Update doc/source/getting-started-macos-tutorial.rst Co-authored-by: Donghyun Kim --- doc/source/getting-started-macos-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting-started-macos-tutorial.rst b/doc/source/getting-started-macos-tutorial.rst index 76f77d0f3..bc0cb1b92 100644 --- a/doc/source/getting-started-macos-tutorial.rst +++ b/doc/source/getting-started-macos-tutorial.rst @@ -81,7 +81,7 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols/mac`` directory. -linux.pslist +mac.pslist ~~~~~~~~~~~~ .. code-block:: shell-session From 53b24d33e0d3c63fec59d23bf553f35bfff92580 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 13 Dec 2022 09:59:09 +0000 Subject: [PATCH 005/101] First attempt at adding a --dump option to linux.proc, aim to be similar to windows.vadinfo --dump --- volatility3/framework/plugins/linux/proc.py | 143 +++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 9d8af482e..6d182ff9e 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -4,18 +4,23 @@ """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" -from volatility3.framework import renderers +import logging +from typing import Callable, List, Generator, Iterable, Type, Optional + +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) + MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod def get_requirements(cls): @@ -35,16 +40,138 @@ class Maps(plugins.PluginInterface): element_type=int, optional=True, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed memory segments", + default=False, + optional=True, + ), + requirements.ListRequirement( + name="address", + description="Process virtual memory address to include " + "(all other address ranges are excluded). This must be " + "a base address, not an address within the desired range.", + element_type=int, + optional=True, + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size for dumped VMA sections " + "(all the bigger sections will be ignored)", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), ] + @classmethod + def list_vmas( + cls, + task: interfaces.objects.ObjectInterface, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: False, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Lists the Virtual Memory Areas of a specific process. + + Args: + task: task object from which to list the vma + filter_func: Function to take a vma and return True if it should be filtered out + + Returns: + A list of vmas based on the task and filtered based on the filter function + """ + if task.mm: + for vma in task.mm.get_mmap_iter(): + if not filter_func(vma): + yield vma + + @classmethod + def vma_dump( + cls, + context: interfaces.context.ContextInterface, + task: interfaces.objects.ObjectInterface, + vma: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + maxsize: int = MAXSIZE_DEFAULT, + ) -> Optional[interfaces.plugins.FileHandlerInterface]: + """Extracts the complete data for VMA as a FileInterface. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + task: an task_struct instance + vma: The suspected VMA to extract (ObjectInterface) + open_method: class to provide context manager for opening the file + maxsize: Max size of VMA section (default MAXSIZE_DEFAULT) + + Returns: + An open FileInterface object containing the complete data for the task or None in the case of failure + """ + try: + vm_start = vma.vm_start + vm_end = vma.vm_end + except AttributeError: + vollog.debug("Unable to find the vm_start and vm_end") + return None + + vm_size = vm_end - vm_start + if 0 < maxsize < vm_size: + vollog.debug( + f"Skip virtual memory dump {vm_start:#x}-{vm_end:#x} due to maxsize limit" + ) + return None + + pid = "Unknown" + try: + pid = task.tgid + proc_layer_name = task.add_process_layer() + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + pid, excp.invalid_address, excp.layer_name + ) + ) + return None + + proc_layer = context.layers[proc_layer_name] + file_name = f"pid.{pid}.vma.{vm_start:#x}-{vm_end:#x}.dmp" + try: + file_handle = open_method(file_name) + chunk_size = 1024 * 1024 * 10 + offset = vm_start + while offset < vm_start + vm_size: + to_read = min(chunk_size, vm_start + vm_size - offset) + data = proc_layer.read(offset, to_read, pad=True) + if not data: + break + file_handle.write(data) + offset += to_read + + except Exception as excp: + vollog.debug(f"Unable to dump virtual memory {file_name}: {excp}") + return None + + return file_handle + def _generator(self, tasks): + # build filter for addresses if required + address_list = self.config.get("address", []) + if address_list == []: + # do not filter as no address_list was supplied + filter_func = lambda _: False + else: + # filter for any vm_start that matches the supplied address config + def filter_function(x: interfaces.objects.ObjectInterface) -> bool: + return x.vm_start not in address_list + + filter_func = filter_function + for task in tasks: if not task.mm: continue name = utility.array_to_string(task.comm) - for vma in task.mm.get_mmap_iter(): + for vma in self.list_vmas(task, filter_func=filter_func): flags = vma.get_protection() page_offset = vma.get_page_offset() major = 0 @@ -61,6 +188,16 @@ class Maps(plugins.PluginInterface): path = vma.get_name(self.context, task) + file_output = "Disabled" + if self.config["dump"]: + file_handle = self.vma_dump( + self.context, task, vma, self.open, self.config["maxsize"] + ) + file_output = "Error outputting file" + if file_handle: + file_handle.close() + file_output = file_handle.preferred_filename + yield ( 0, ( @@ -74,6 +211,7 @@ class Maps(plugins.PluginInterface): minor, inode, path, + file_output, ), ) @@ -92,6 +230,7 @@ class Maps(plugins.PluginInterface): ("Minor", int), ("Inode", int), ("File Path", str), + ("File output", str), ], self._generator( pslist.PsList.list_tasks( From 4c0a0b923bc1b7c13e753e90f84db44d95b95368 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 4 Jan 2023 16:05:00 +0000 Subject: [PATCH 006/101] Update linux.proc --dump changes based on comments from ikelos --- volatility3/framework/plugins/linux/proc.py | 64 ++++++++++++--------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 6d182ff9e..d8a17ae38 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -5,7 +5,7 @@ found in Linux's /proc file system.""" import logging -from typing import Callable, List, Generator, Iterable, Type, Optional +from typing import Callable, Generator, Type, Optional from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements @@ -16,6 +16,7 @@ from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) + class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" @@ -48,9 +49,9 @@ class Maps(plugins.PluginInterface): ), requirements.ListRequirement( name="address", - description="Process virtual memory address to include " - "(all other address ranges are excluded). This must be " - "a base address, not an address within the desired range.", + description="Process virtual memory addresses to include " + "(all other VMA sections are excluded). This can be any " + "virtual address within the VMA section.", element_type=int, optional=True, ), @@ -69,21 +70,25 @@ class Maps(plugins.PluginInterface): task: interfaces.objects.ObjectInterface, filter_func: Callable[ [interfaces.objects.ObjectInterface], bool - ] = lambda _: False, + ] = lambda _: True, ) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Lists the Virtual Memory Areas of a specific process. Args: task: task object from which to list the vma - filter_func: Function to take a vma and return True if it should be filtered out + filter_func: Function to take a vma and return False if it should be filtered out Returns: - A list of vmas based on the task and filtered based on the filter function + Yields vmas based on the task and filtered based on the filter function """ if task.mm: for vma in task.mm.get_mmap_iter(): - if not filter_func(vma): + if filter_func(vma): yield vma + else: + vollog.debug( + f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.pid} due to filter_func" + ) @classmethod def vma_dump( @@ -106,23 +111,15 @@ class Maps(plugins.PluginInterface): Returns: An open FileInterface object containing the complete data for the task or None in the case of failure """ + pid = task.pid try: vm_start = vma.vm_start vm_end = vma.vm_end except AttributeError: - vollog.debug("Unable to find the vm_start and vm_end") - return None - - vm_size = vm_end - vm_start - if 0 < maxsize < vm_size: - vollog.debug( - f"Skip virtual memory dump {vm_start:#x}-{vm_end:#x} due to maxsize limit" - ) + vollog.debug(f"Unable to find the vm_start and vm_end for pid {pid}") return None - pid = "Unknown" try: - pid = task.tgid proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( @@ -132,6 +129,13 @@ class Maps(plugins.PluginInterface): ) return None + vm_size = vm_end - vm_start + if 0 < maxsize < vm_size: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is larger than maxsize limit of {maxsize}" + ) + return None + proc_layer = context.layers[proc_layer_name] file_name = f"pid.{pid}.vma.{vm_start:#x}-{vm_end:#x}.dmp" try: @@ -141,8 +145,6 @@ class Maps(plugins.PluginInterface): while offset < vm_start + vm_size: to_read = min(chunk_size, vm_start + vm_size - offset) data = proc_layer.read(offset, to_read, pad=True) - if not data: - break file_handle.write(data) offset += to_read @@ -154,16 +156,24 @@ class Maps(plugins.PluginInterface): def _generator(self, tasks): # build filter for addresses if required - address_list = self.config.get("address", []) - if address_list == []: + address_list = self.config.get("address", None) + if not address_list: # do not filter as no address_list was supplied - filter_func = lambda _: False + vma_filter_func = lambda _: True else: # filter for any vm_start that matches the supplied address config - def filter_function(x: interfaces.objects.ObjectInterface) -> bool: - return x.vm_start not in address_list + def vma_filter_function(x: interfaces.objects.ObjectInterface) -> bool: + addrs_in_vma = [ + addr for addr in address_list if x.vm_start <= addr <= x.vm_end + ] - filter_func = filter_function + # if any of the user supplied addresses would fall within this vma return true + if addrs_in_vma: + return True + else: + return False + + vma_filter_func = vma_filter_function for task in tasks: if not task.mm: @@ -171,7 +181,7 @@ class Maps(plugins.PluginInterface): name = utility.array_to_string(task.comm) - for vma in self.list_vmas(task, filter_func=filter_func): + for vma in self.list_vmas(task, filter_func=vma_filter_func): flags = vma.get_protection() page_offset = vma.get_page_offset() major = 0 From 560569e03e5e8f7f6f29d6695b57745b11e2afee Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 6 Jan 2023 22:12:20 +0000 Subject: [PATCH 007/101] update linux.proc --dump so that vma object is not passed to dump func --- volatility3/framework/plugins/linux/proc.py | 41 ++++++++++++++------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index d8a17ae38..99c8761c1 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -95,7 +95,8 @@ class Maps(plugins.PluginInterface): cls, context: interfaces.context.ContextInterface, task: interfaces.objects.ObjectInterface, - vma: interfaces.objects.ObjectInterface, + vm_start: int, + vm_end: int, open_method: Type[interfaces.plugins.FileHandlerInterface], maxsize: int = MAXSIZE_DEFAULT, ) -> Optional[interfaces.plugins.FileHandlerInterface]: @@ -105,6 +106,8 @@ class Maps(plugins.PluginInterface): context: The context to retrieve required elements (layers, symbol tables) from task: an task_struct instance vma: The suspected VMA to extract (ObjectInterface) + vm_start: The start virtual address from the vma to dump + vm_end: The end virtual address from the vma to dump open_method: class to provide context manager for opening the file maxsize: Max size of VMA section (default MAXSIZE_DEFAULT) @@ -112,12 +115,6 @@ class Maps(plugins.PluginInterface): An open FileInterface object containing the complete data for the task or None in the case of failure """ pid = task.pid - try: - vm_start = vma.vm_start - vm_end = vma.vm_end - except AttributeError: - vollog.debug(f"Unable to find the vm_start and vm_end for pid {pid}") - return None try: proc_layer_name = task.add_process_layer() @@ -200,13 +197,31 @@ class Maps(plugins.PluginInterface): file_output = "Disabled" if self.config["dump"]: - file_handle = self.vma_dump( - self.context, task, vma, self.open, self.config["maxsize"] - ) file_output = "Error outputting file" - if file_handle: - file_handle.close() - file_output = file_handle.preferred_filename + try: + vm_start = vma.vm_start + vm_end = vma.vm_end + except AttributeError: + vollog.debug( + f"Unable to find the vm_start and vm_end for vma at {vma.vol.offset:#x} for pid {pid}" + ) + vm_start = None + vm_end = None + + if vm_start and vm_end: + # only attempt to dump the memory if we have vm_start and vm_end + file_handle = self.vma_dump( + self.context, + task, + vm_start, + vm_end, + self.open, + self.config["maxsize"], + ) + + if file_handle: + file_handle.close() + file_output = file_handle.preferred_filename yield ( 0, From a60b91b6e18cb5979f2872ec61dc60af4a10faa1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 9 Jan 2023 21:33:07 +0000 Subject: [PATCH 008/101] Layers: Add in Xen CoreDump format support --- volatility3/framework/layers/xen.py | 171 +++++++++++++++++++ volatility3/framework/symbols/linux/xen.json | 115 +++++++++++++ 2 files changed, 286 insertions(+) create mode 100644 volatility3/framework/layers/xen.py create mode 100644 volatility3/framework/symbols/linux/xen.json diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py new file mode 100644 index 000000000..33050ea4b --- /dev/null +++ b/volatility3/framework/layers/xen.py @@ -0,0 +1,171 @@ +import logging +import struct +from typing import Optional + +from volatility3.framework import constants, interfaces, exceptions +from volatility3.framework.layers import elf +from volatility3.framework.symbols import intermed + +vollog = logging.getLogger(__name__) + + +class XenCoreDumpLayer(elf.Elf64Layer): + """A layer that supports the Xen Dump-Core format as documented at: https://xenbits.xen.org/docs/4.6-testing/misc/dump-core-format.txt""" + + _header_struct = struct.Struct(" None: + # Create a custom SymbolSpace + self._elf_table_name = intermed.IntermediateSymbolTable.create( + context, config_path, "linux", "elf" + ) + self._xen_table_name = intermed.IntermediateSymbolTable.create( + context, config_path, "linux", "xen" + ) + + super().__init__(context, config_path, name) + + def _load_segments(self) -> None: + """Load the segments from based on the PT_LOAD segments of the Elf64 format""" + ehdr = self.context.object( + self._elf_table_name + constants.BANG + "Elf64_Ehdr", + layer_name=self._base_layer, + offset=0, + ) + + segments = [] + segment_headers = [] + + for sindex in range(ehdr.e_shnum): + shdr = self.context.object( + self._elf_table_name + constants.BANG + "Elf64_Shdr", + layer_name=self._base_layer, + offset=ehdr.e_shoff + (sindex * ehdr.e_shentsize), + ) + + segment_headers.append(shdr) + + if sindex == ehdr.e_shstrndx: + segment_names = self.context.layers[self._base_layer].read( + shdr.sh_offset, shdr.sh_size + ) + segment_names = segment_names.split(b"\x00") + + if not segment_names: + raise elf.ElfFormatException("No segment names, not a Xen Core Dump") + + p2m_data = None + pfn_data = None + + for varname, pattern, outvar in [ + ("xen_p2m", b".xen_p2m", p2m_data), + ("xen_pfn", b".xen_pfn", pfn_data), + ]: + if pattern in segment_names: + hdr = segment_headers[segment_names.index(pattern)] + result = self.context.object( + self._xen_table_name + constants.BANG + varname, + layer_name=self._base_layer, + offset=hdr.sh_offset, + size=hdr.sh_size, + ) + result.entries.count = hdr.sh_size // result.entries.vol.subtype.size + outvar = result + + pages_hdr = segment_headers[segment_names.index(b".xen_pages")] + page_size = 0x1000 + + if pfn_data and not p2m_data: + for entry_index in range(len(pfn_data.entries)): + entry = pfn_data.entries[entry_index] + # TODO: Don't hardcode the maximum value here + if entry and entry != 0xFFFFFFFF: + segments.append( + ( + entry * page_size, + pages_hdr.sh_offset + (entry_index * page_size), + page_size, + page_size, + ) + ) + elif p2m_data and not pfn_data: + for entry_index in range(len(p2m_data.entries)): + entry = p2m_data.entries[entry_index] + # TODO: Don't hardcode the maximum value here + if entry.pfn != 0xFFFFFFFF: + segments.append( + ( + entry.pfn * page_size, + pages_hdr.sh_offset + (entry_index * page_size), + page_size, + page_size, + ) + ) + elif p2m_data and pfn_data: + raise elf.ElfFormatException( + self.name, f"Both P2M and PFN in Xen Core Dump" + ) + else: + raise elf.ElfFormatException( + self.name, f"Neither P2M nor PFN in Xen Core Dump" + ) + + if len(segments) == 0: + raise elf.ElfFormatException( + self.name, f"No ELF segments defined in {self._base_layer}" + ) + + self._segments = segments + + @classmethod + def _check_header( + cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0 + ) -> bool: + try: + header_data = base_layer.read(offset, cls._header_struct.size) + except exceptions.InvalidAddressException: + raise elf.ElfFormatException( + base_layer.name, + f"Offset 0x{offset:0x} does not exist within the base layer", + ) + (magic, elf_class, elf_data_encoding, elf_version) = cls._header_struct.unpack( + header_data + ) + if magic != cls.MAGIC: + raise elf.ElfFormatException( + base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}" + ) + if elf_class != cls.ELF_CLASS: + raise elf.ElfFormatException( + base_layer.name, f"ELF class is not 64-bit (2): {elf_class:d}" + ) + # Virtualbox uses an ELF version of 0, which isn't to specification, but is ok to deal with + return True + + +class XenCoreDumpStacker(elf.Elf64Stacker): + stack_order = 10 + + @classmethod + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: + try: + if not XenCoreDumpLayer._check_header(context.layers[layer_name]): + return None + except elf.ElfFormatException as excp: + vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") + return None + new_name = context.layers.free_layer_name("XenCoreDumpLayer") + context.config[ + interfaces.configuration.path_join(new_name, "base_layer") + ] = layer_name + + return XenCoreDumpLayer(context, new_name, new_name) diff --git a/volatility3/framework/symbols/linux/xen.json b/volatility3/framework/symbols/linux/xen.json new file mode 100644 index 000000000..8e843e728 --- /dev/null +++ b/volatility3/framework/symbols/linux/xen.json @@ -0,0 +1,115 @@ +{ + "symbols": { + }, + "user_types": { + "xen_p2m": { + "fields":{ + "entries": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned long long" + } + } + } + }, + "kind": "struct", + "size": 8 + }, + "xen_pfn":{ + "fields":{ + "entries": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned long long" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "xen_pfn_entry":{ + "fields":{ + "pfn":{ + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "gmfn":{ + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long long" + } + } + }, + "kind": "struct", + "size": 16 + + } + }, + "enums": { + }, + "base_types": { + "unsigned char": { + "endian": "little", + "kind": "char", + "signed": false, + "size": 1 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + }, + "long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 4 + }, + "char": { + "endian": "little", + "kind": "char", + "signed": true, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "ikelos-by-hand", + "datetime": "2023-01-09T00:51:00" + }, + "format": "6.1.0" + } +} From d963aa3afb702980552653e40d3b090c57bf06bf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 10 Jan 2023 16:39:43 +0000 Subject: [PATCH 009/101] Layers: Fix trying to be too clever with Xen --- volatility3/framework/layers/xen.py | 49 +++++++++++++++++------------ 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index 33050ea4b..f7881a091 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -26,9 +26,23 @@ class XenCoreDumpLayer(elf.Elf64Layer): self._xen_table_name = intermed.IntermediateSymbolTable.create( context, config_path, "linux", "xen" ) + self._segment_headers = {} super().__init__(context, config_path, name) + def _extract_result_array( + self, varname: str, segment_index: int + ) -> interfaces.objects.ObjectInterface: + hdr = self._segment_headers[segment_index] + result = self.context.object( + self._xen_table_name + constants.BANG + varname, + layer_name=self._base_layer, + offset=hdr.sh_offset, + size=hdr.sh_size, + ) + result.entries.count = hdr.sh_size // result.entries.vol.subtype.size + return result + def _load_segments(self) -> None: """Load the segments from based on the PT_LOAD segments of the Elf64 format""" ehdr = self.context.object( @@ -38,7 +52,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): ) segments = [] - segment_headers = [] + self._segment_headers = [] for sindex in range(ehdr.e_shnum): shdr = self.context.object( @@ -47,7 +61,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): offset=ehdr.e_shoff + (sindex * ehdr.e_shentsize), ) - segment_headers.append(shdr) + self._segment_headers.append(shdr) if sindex == ehdr.e_shstrndx: segment_names = self.context.layers[self._base_layer].read( @@ -58,25 +72,20 @@ class XenCoreDumpLayer(elf.Elf64Layer): if not segment_names: raise elf.ElfFormatException("No segment names, not a Xen Core Dump") - p2m_data = None - pfn_data = None + try: + p2m_data = self._extract_result_array( + "xen_p2m", segment_names.index(b".xen_p2m") + ) + except ValueError: + p2m_data = None + try: + pfn_data = self._extract_result_array( + "xen_pfn", segment_names.index(b".xen_pfn") + ) + except ValueError: + pfn_data = None - for varname, pattern, outvar in [ - ("xen_p2m", b".xen_p2m", p2m_data), - ("xen_pfn", b".xen_pfn", pfn_data), - ]: - if pattern in segment_names: - hdr = segment_headers[segment_names.index(pattern)] - result = self.context.object( - self._xen_table_name + constants.BANG + varname, - layer_name=self._base_layer, - offset=hdr.sh_offset, - size=hdr.sh_size, - ) - result.entries.count = hdr.sh_size // result.entries.vol.subtype.size - outvar = result - - pages_hdr = segment_headers[segment_names.index(b".xen_pages")] + pages_hdr = self._segment_headers[segment_names.index(b".xen_pages")] page_size = 0x1000 if pfn_data and not p2m_data: From f3dea3619cc2521c9616a530222a47bff74f738b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 5 Mar 2023 23:14:09 +0000 Subject: [PATCH 010/101] Windows: Memoize part of the pool handling code --- volatility3/framework/contexts/__init__.py | 7 ++++- .../symbols/windows/extensions/pool.py | 29 +++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 226a303dd..ecce5041c 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -381,6 +381,7 @@ class ModuleCollection(interfaces.context.ModuleContainer): def __init__( self, modules: Optional[List[interfaces.context.ModuleInterface]] = None ) -> None: + self._prefix_count = {} super().__init__(modules) def deduplicate(self) -> "ModuleCollection": @@ -400,9 +401,13 @@ class ModuleCollection(interfaces.context.ModuleContainer): def free_module_name(self, prefix: str = "module") -> str: """Returns an unused module name""" - count = 1 + if prefix not in self._prefix_count: + self._prefix_count[prefix] = 1 + return prefix + count = self._prefix_count[prefix] while prefix + str(count) in self: count += 1 + self._prefix_count[prefix] = count return prefix + str(count) @property diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index ac7f36a99..052ae4dd6 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -396,13 +396,9 @@ class OBJECT_HEADER(objects.StructType): ) symbol_table_name = self.vol.type_name.split(constants.BANG)[0] - - try: - header_offset = self.NameInfoOffset - except AttributeError: - # http://codemachine.com/article_objectheader.html (Windows 7 and later) - name_info_bit = 0x2 - + if symbol_table_name in self._context.modules: + ntkrnlmp = self._context.modules[symbol_table_name] + else: layer = self._context.layers[self.vol.native_layer_name] kvo = layer.config.get("kernel_virtual_offset", None) @@ -411,16 +407,25 @@ class OBJECT_HEADER(objects.StructType): f"Could not find kernel_virtual_offset for layer: {self.vol.layer_name}" ) + # We know this symbol table name can't exist because we checked for it earlier ntkrnlmp = self._context.module( symbol_table_name, layer_name=self.vol.layer_name, offset=kvo ) + self._context.add_module(ntkrnlmp) + + try: + header_offset = self.NameInfoOffset + except AttributeError: + # http://codemachine.com/article_objectheader.html (Windows 7 and later) + name_info_bit = 0x2 + address = ntkrnlmp.get_symbol("ObpInfoMaskToOffset").address calculated_index = self.InfoMask & (name_info_bit | (name_info_bit - 1)) - header_offset = self._context.object( - symbol_table_name + constants.BANG + "unsigned char", + header_offset = ntkrnlmp.object( + "unsigned char", layer_name=self.vol.native_layer_name, - offset=kvo + address + calculated_index, + offset=address + calculated_index, ) if header_offset == 0: @@ -430,8 +435,8 @@ class OBJECT_HEADER(objects.StructType): ) ) - header = self._context.object( - symbol_table_name + constants.BANG + "_OBJECT_HEADER_NAME_INFO", + header = ntkrnlmp.object( + "_OBJECT_HEADER_NAME_INFO", layer_name=self.vol.layer_name, offset=self.vol.offset - header_offset, native_layer_name=self.vol.native_layer_name, From 21e34ec605d001a7f272907a13fe30210eeab278 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 5 Mar 2023 23:25:42 +0000 Subject: [PATCH 011/101] Windows: Fix up double adding the symbol from context.module --- volatility3/framework/symbols/windows/extensions/pool.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index 052ae4dd6..b761ddad8 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -396,6 +396,7 @@ class OBJECT_HEADER(objects.StructType): ) symbol_table_name = self.vol.type_name.split(constants.BANG)[0] + if symbol_table_name in self._context.modules: ntkrnlmp = self._context.modules[symbol_table_name] else: @@ -411,7 +412,6 @@ class OBJECT_HEADER(objects.StructType): ntkrnlmp = self._context.module( symbol_table_name, layer_name=self.vol.layer_name, offset=kvo ) - self._context.add_module(ntkrnlmp) try: header_offset = self.NameInfoOffset @@ -440,5 +440,6 @@ class OBJECT_HEADER(objects.StructType): layer_name=self.vol.layer_name, offset=self.vol.offset - header_offset, native_layer_name=self.vol.native_layer_name, + absolute=True, ) return header From a35afd4f343c10d7f8d1df2cb5eec8364c3dbd5a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 8 Mar 2023 20:40:04 +0000 Subject: [PATCH 012/101] Core: Bump framwork version after release branch --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 8f1163fe1..3a6b24ea8 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 4 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From b8755ac574e8226321ed169a1d6cac3a39505617 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 8 Mar 2023 20:42:24 +0000 Subject: [PATCH 013/101] Linux: fix black lint issues --- volatility3/framework/plugins/linux/envars.py | 1 - volatility3/framework/plugins/linux/iomem.py | 1 - 2 files changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 028eb2a57..5cbf0f502 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -70,7 +70,6 @@ class Envars(plugins.PluginInterface): # if mm exists attempt to get envars if mm: - # get process layer to read envars from proc_layer_name = task.add_process_layer() if proc_layer_name is None: diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index fddea4668..8efbf3b57 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -128,7 +128,6 @@ class IOMem(interfaces.plugins.PluginInterface): # only continue if iomem_root address was located if iomem_root_offset is not None: - # recursively parse the resources starting from the root resource at 'iomem_resource' for depth, (name, start, end) in self.parse_resource( self.context, vmlinux_module_name, iomem_root_offset From 99417cdfcc87d93d82b7288d8ee06ec4e06ada07 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 14:17:37 +0000 Subject: [PATCH 014/101] Linux: Psscan check parent pointer is valid --- volatility3/framework/plugins/linux/psscan.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index f4bfd347e..ca9d30586 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -52,7 +52,10 @@ class PsScan(interfaces.plugins.PluginInterface): """ pid = task.tgid tid = task.pid - ppid = task.parent.tgid if task.parent else 0 + ppid = 0 + + if task.parent.is_readable(): + ppid = task.parent.tgid name = utility.array_to_string(task.comm) exit_state = DescExitStateEnum(task.exit_state).name From 36ec5164703d2b5eaf0b3ff0d5f3a5f59572a5ff Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 15:17:24 +0000 Subject: [PATCH 015/101] Linux: Fix psscan task native_layer --- volatility3/framework/plugins/linux/psscan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index ca9d30586..462577e58 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -138,6 +138,7 @@ class PsScan(interfaces.plugins.PluginInterface): raise exceptions.LayerException( kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" ) + memory_layer_name = kernel_layer.dependencies[0] memory_layer = context.layers[kernel_layer.dependencies[0]] # scan the memory_layer for these needles @@ -148,7 +149,8 @@ class PsScan(interfaces.plugins.PluginInterface): ptask = context.object( vmlinux.symbol_table_name + constants.BANG + "task_struct", offset=address - sched_class_offset, - layer_name="memory_layer", + layer_name=memory_layer_name, + native_layer_name=kernel_layer_name, ) # sanity check exit_state From 46f56770af1785f5f0bcfb887849bb96e164f23c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 15:18:58 +0000 Subject: [PATCH 016/101] Core: Pointer is_readable should check the native layer --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index a04eedd87..3b1745718 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -442,7 +442,7 @@ class Pointer(Integer): def is_readable(self, layer_name: Optional[str] = None) -> bool: """Determines whether the address of this pointer can be read from memory.""" - layer_name = layer_name or self.vol.layer_name + layer_name = layer_name or self.vol.native_layer_name return self._context.layers[layer_name].is_valid(self, self.vol.subtype.size) def __getattr__(self, attr: str) -> Any: From 9216bab61b3187fc248760ffdda5b86c8a694c9a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 17:32:46 +0000 Subject: [PATCH 017/101] Core: Improve import exception reporting --- volatility3/framework/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index ec0edc2fe..479925fb7 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -26,6 +26,7 @@ import importlib import inspect import logging import os +import traceback from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces @@ -183,7 +184,11 @@ def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str try: importlib.import_module(module) except ImportError as e: - vollog.debug(str(e)) + vollog.debug( + "".join( + traceback.TracebackException.from_exception(e).format(chain=True) + ) + ) vollog.debug( "Failed to import module {} based on file: {}".format(module, path) ) From d9a365d96fcd990c7faba32ab7aa63523203e9f8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 17:33:20 +0000 Subject: [PATCH 018/101] Linux: Rename linux.envars to linux.envvars --- volatility3/framework/plugins/linux/envars.py | 121 +----------------- .../framework/plugins/linux/envvars.py | 121 ++++++++++++++++++ 2 files changed, 127 insertions(+), 115 deletions(-) create mode 100644 volatility3/framework/plugins/linux/envvars.py diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 5cbf0f502..c4e3ed3c9 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -1,121 +1,12 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - +from volatility3.plugins import envvars import logging -from volatility3.framework import exceptions, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins -from volatility3.framework.objects import utility -from volatility3.plugins.linux import pslist - vollog = logging.getLogger(__name__) -class Envars(plugins.PluginInterface): - """Lists processes with their environment variables""" - - _required_framework_version = (2, 0, 0) - - @classmethod - def get_requirements(cls): - # Since we're calling the plugin, make sure we have the plugin's requirements - return [ - requirements.ModuleRequirement( - name="kernel", - description="Linux kernel", - architectures=["Intel32", "Intel64"], - ), - requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) - ), - requirements.ListRequirement( - name="pid", - description="Filter on specific process IDs", - element_type=int, - optional=True, - ), - ] - - def _generator(self, tasks): - """Generates a listing of processes along with environment variables""" - - # walk the process list and return the envars - for task in tasks: - pid = task.pid - - # get process name as string - name = utility.array_to_string(task.comm) - - # try and get task parent - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." - ) - ppid = 0 - - # kernel threads never have an mm as they do not have userland mappings - try: - mm = task.mm - except exceptions.InvalidAddressException: - # no mm so cannot get envars - vollog.debug( - f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." - ) - mm = None - continue - - # if mm exists attempt to get envars - if mm: - # get process layer to read envars from - proc_layer_name = task.add_process_layer() - if proc_layer_name is None: - vollog.debug( - f"Unable to construct process layer for task {pid} {name}, will not extract any envars." - ) - continue - proc_layer = self.context.layers[proc_layer_name] - - # get the size of the envars with sanity checking - envars_size = task.mm.env_end - task.mm.env_start - if not (0 < envars_size <= 8192): - vollog.debug( - f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." - ) - continue - - # attempt to read all envars data - try: - envar_data = proc_layer.read(task.mm.env_start, envars_size) - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." - ) - continue - - # parse envar data, envars are null terminated, keys and values are separated by '=' - envar_data = envar_data.rstrip(b"\x00") - for envar_pair in envar_data.split(b"\x00"): - try: - key, value = envar_pair.decode().split("=", 1) - except ValueError: - vollog.debug( - f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" - ) - continue - yield (0, (pid, ppid, name, key, value)) - - def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - - return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], - self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ), +class Envars(envvars.Envvars): + def run(self, *args, **kwargs): + vollog.warning( + "The linux.envars plugin has been renamed to linux.envvars and will only be accessible through the new name in a future release" ) + return super().run(*args, **kwargs) diff --git a/volatility3/framework/plugins/linux/envvars.py b/volatility3/framework/plugins/linux/envvars.py new file mode 100644 index 000000000..1d6c8b784 --- /dev/null +++ b/volatility3/framework/plugins/linux/envvars.py @@ -0,0 +1,121 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from volatility3.framework import exceptions, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class Envvars(plugins.PluginInterface): + """Lists processes with their environment variables""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + ] + + def _generator(self, tasks): + """Generates a listing of processes along with environment variables""" + + # walk the process list and return the envars + for task in tasks: + pid = task.pid + + # get process name as string + name = utility.array_to_string(task.comm) + + # try and get task parent + try: + ppid = task.parent.pid + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." + ) + ppid = 0 + + # kernel threads never have an mm as they do not have userland mappings + try: + mm = task.mm + except exceptions.InvalidAddressException: + # no mm so cannot get envars + vollog.debug( + f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." + ) + mm = None + continue + + # if mm exists attempt to get envars + if mm: + # get process layer to read envars from + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + vollog.debug( + f"Unable to construct process layer for task {pid} {name}, will not extract any envars." + ) + continue + proc_layer = self.context.layers[proc_layer_name] + + # get the size of the envars with sanity checking + envars_size = task.mm.env_end - task.mm.env_start + if not (0 < envars_size <= 8192): + vollog.debug( + f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." + ) + continue + + # attempt to read all envars data + try: + envar_data = proc_layer.read(task.mm.env_start, envars_size) + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." + ) + continue + + # parse envar data, envars are null terminated, keys and values are separated by '=' + envar_data = envar_data.rstrip(b"\x00") + for envar_pair in envar_data.split(b"\x00"): + try: + key, value = envar_pair.decode().split("=", 1) + except ValueError: + vollog.debug( + f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" + ) + continue + yield (0, (pid, ppid, name, key, value)) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + return renderers.TreeGrid( + [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), + ) From 3d51b4a41a7110670826557ea82f3bd0ecf55e5c Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 24 Mar 2023 15:08:59 +0000 Subject: [PATCH 019/101] Add basic support for linux maple tree struct --- .../framework/symbols/linux/__init__.py | 2 + .../symbols/linux/extensions/__init__.py | 142 ++++++++++++++++++ 2 files changed, 144 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index ce07167e5..7a22241a5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -50,6 +50,8 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.optional_set_type_class("bt_sock", extensions.bt_sock) self.optional_set_type_class("xdp_sock", extensions.xdp_sock) + # Only found in 6.1+ kernels + self.optional_set_type_class("maple_tree", extensions.maple_tree) class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 5ab8f1aa0..72173f471 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -294,11 +294,128 @@ class fs_struct(objects.StructType): raise AttributeError("Unable to find the root mount") +class maple_tree(objects.StructType): + # include/linux/maple_tree.h + # Mask for Maple Tree Flags + MT_FLAGS_HEIGHT_MASK = 0x7C + MT_FLAGS_HEIGHT_OFFSET = 0x02 + + # Shift and mask to extract information from maple tree node pointers + MAPLE_NODE_TYPE_SHIFT = 0x03 + MAPLE_NODE_TYPE_MASK = 0x0F + MAPLE_NODE_POINTER_MASK = 0xFF + + # types of Maple Tree Nodes + MAPLE_DENSE = 0 + MAPLE_LEAF_64 = 1 + MAPLE_RANGE_64 = 2 + MAPLE_ARANGE_64 = 3 + + def get_slot_iter(self): + """Parse the Maple Tree and return every non zero slot.""" + maple_tree_offset, _, _ = self._parse_maple_tree_entry(self.vol.offset) + maple_tree_depth = ( + self.ma_flags & self.MT_FLAGS_HEIGHT_MASK + ) >> self.MT_FLAGS_HEIGHT_OFFSET + yield from self._parse_maple_tree_node( + self.ma_root, maple_tree_offset, maple_tree_depth + ) + + def _parse_maple_tree_node( + self, maple_tree_entry, parent, maple_tree_depth, seen=set(), depth=1 + ): + """Recursively parse Maple Tree Nodes and yield all non empty slots""" + + # protect against unlikely loop + if maple_tree_entry in seen: + vollog.warning( + f"The mte {hex(maple_tree_entry)} has all ready been seen, no further results will be produced for this node." + ) + return + else: + seen.add(maple_tree_entry) + if maple_tree_depth < depth: + vollog.warning( + f"The depth for the maple tree at {hex(self.vol.offset)} is {maple_tree_depth}, however when parsing the nodes " + f"a depth of {depth} was reached. This is unexpected and may lead to incorrect results." + ) + # parse the mte to extract the pointer value, node type, and leaf status + pointer, node_type, is_leaf = self._parse_maple_tree_entry(maple_tree_entry) + + # create a pointer object for the node parent mte (note this will include flags in the low bits) + symbol_table_name = self.get_symbol_table_name() + node_parent_mte = self._context.object( + symbol_table_name + constants.BANG + "pointer", + layer_name=self.vol.layer_name, + offset=pointer, + ) + + # extract the actual pointer to the parent of this node + node_parent_pointer, _, _ = self._parse_maple_tree_entry(node_parent_mte) + + # verify that the node_parent_pointer correctly points to the parent + assert node_parent_pointer == parent + + # create a node object + node = self._context.object( + symbol_table_name + constants.BANG + "maple_node", + layer_name=self.vol.layer_name, + offset=pointer, + ) + + # parse the slots based on the node type + if node_type == self.MAPLE_DENSE: + assert is_leaf == True + for slot in node.alloc.slot: + if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: + yield slot + elif node_type == self.MAPLE_LEAF_64: + assert is_leaf == True + for slot in node.mr64.slot: + if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: + yield slot + elif node_type == self.MAPLE_RANGE_64: + assert is_leaf == False + for slot in node.mr64.slot: + if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: + yield from self._parse_maple_tree_node( + slot, pointer, maple_tree_depth, seen, depth + 1 + ) + elif node_type == self.MAPLE_ARANGE_64: + assert is_leaf == False + for slot in node.ma64.slot: + if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: + yield from self._parse_maple_tree_node( + slot, pointer, maple_tree_depth, seen, depth + 1 + ) + else: + # unkown maple node type + raise AttributeError( + f"Unkown Maple Tree node type {node_type} at offset {hex(pointer)}." + ) + + def _parse_maple_tree_entry(self, maple_tree_entry): + """Parse a Maple Tree Entry and return the pointer, node type, if the node is a leaf, if the node is the root""" + # Extract the node type + node_type = ( + maple_tree_entry >> self.MAPLE_NODE_TYPE_SHIFT + ) & self.MAPLE_NODE_TYPE_MASK + + # Determine if it's a leaf node or not + is_leaf = node_type < self.MAPLE_RANGE_64 + + # Clear the lower bits to get the true pointer value + pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) + + return pointer, node_type, is_leaf class mm_struct(objects.StructType): def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct.""" + if not self.has_member('mmap'): + raise AttributeError("get_mmap_iter called on mm_struct where no mmap member exists.") + if not self.mmap: return @@ -312,7 +429,32 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next + def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns an iterator for the mm_mt member of an mm_struct.""" + + if not self.has_member('mm_mt'): + raise AttributeError("get_maple_tree_iter called on mm_struct where no mm_mt member exists.") + symbol_table_name = self.get_symbol_table_name() + for vma_pointer in self.mm_mt.get_slot_iter(): + # convert pointer to vm_area_struct and yield + vma = self._context.object( + symbol_table_name + constants.BANG + "vm_area_struct", + layer_name=self.vol.layer_name, + offset=vma_pointer + ) + yield vma + + def get_vma_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" + + if self.has_member('mmap'): + yield from self.get_mmap_iter() + elif self.has_member('mm_mt'): + yield from self.get_maple_tree_iter() + else: + raise AttributeError("Unable to find mmap or mm_mt in mm_struct") + class super_block(objects.StructType): # include/linux/kdev_t.h MINORBITS = 20 From 5207cfcb93cf7f8955b5a499ac13a46ba962da78 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 24 Mar 2023 15:10:34 +0000 Subject: [PATCH 020/101] Modify linux plugins to support both mmap and mm_mt --- volatility3/framework/plugins/linux/elfs.py | 2 +- volatility3/framework/plugins/linux/malfind.py | 2 +- volatility3/framework/plugins/linux/proc.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 822a69dd6..2ff5eb591 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -48,7 +48,7 @@ class Elfs(plugins.PluginInterface): name = utility.array_to_string(task.comm) - for vma in task.mm.get_mmap_iter(): + for vma in task.mm.get_vma_iter(): hdr = proc_layer.read(vma.vm_start, 4, pad=True) if not ( hdr[0] == 0x7F diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 18237b80c..1fd005de8 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -46,7 +46,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] - for vma in task.mm.get_mmap_iter(): + for vma in task.mm.get_vma_iter(): if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]": data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 9d8af482e..8979f6f63 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -44,7 +44,7 @@ class Maps(plugins.PluginInterface): name = utility.array_to_string(task.comm) - for vma in task.mm.get_mmap_iter(): + for vma in task.mm.get_vma_iter(): flags = vma.get_protection() page_offset = vma.get_page_offset() major = 0 From 89e477a068092cad50acfaa0fd965097ae5da8c7 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 24 Mar 2023 20:07:32 +0000 Subject: [PATCH 021/101] Remove _parse_maple_tree_entry function and make parsing easier to read --- .../symbols/linux/extensions/__init__.py | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 72173f471..9375a288b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -313,7 +313,7 @@ class maple_tree(objects.StructType): def get_slot_iter(self): """Parse the Maple Tree and return every non zero slot.""" - maple_tree_offset, _, _ = self._parse_maple_tree_entry(self.vol.offset) + maple_tree_offset = self.vol.offset & ~(self.MAPLE_NODE_POINTER_MASK) maple_tree_depth = ( self.ma_flags & self.MT_FLAGS_HEIGHT_MASK ) >> self.MT_FLAGS_HEIGHT_OFFSET @@ -340,7 +340,11 @@ class maple_tree(objects.StructType): f"a depth of {depth} was reached. This is unexpected and may lead to incorrect results." ) # parse the mte to extract the pointer value, node type, and leaf status - pointer, node_type, is_leaf = self._parse_maple_tree_entry(maple_tree_entry) + pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) + node_type = ( + maple_tree_entry >> self.MAPLE_NODE_TYPE_SHIFT + ) & self.MAPLE_NODE_TYPE_MASK + is_leaf = node_type < self.MAPLE_RANGE_64 # create a pointer object for the node parent mte (note this will include flags in the low bits) symbol_table_name = self.get_symbol_table_name() @@ -351,7 +355,7 @@ class maple_tree(objects.StructType): ) # extract the actual pointer to the parent of this node - node_parent_pointer, _, _ = self._parse_maple_tree_entry(node_parent_mte) + node_parent_pointer = node_parent_mte & ~(self.MAPLE_NODE_POINTER_MASK) # verify that the node_parent_pointer correctly points to the parent assert node_parent_pointer == parent @@ -394,21 +398,6 @@ class maple_tree(objects.StructType): f"Unkown Maple Tree node type {node_type} at offset {hex(pointer)}." ) - def _parse_maple_tree_entry(self, maple_tree_entry): - """Parse a Maple Tree Entry and return the pointer, node type, if the node is a leaf, if the node is the root""" - # Extract the node type - node_type = ( - maple_tree_entry >> self.MAPLE_NODE_TYPE_SHIFT - ) & self.MAPLE_NODE_TYPE_MASK - - # Determine if it's a leaf node or not - is_leaf = node_type < self.MAPLE_RANGE_64 - - # Clear the lower bits to get the true pointer value - pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) - - return pointer, node_type, is_leaf - class mm_struct(objects.StructType): def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct.""" From 089234671c70b746c91c25ac2980a6a4f35607b0 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 27 Mar 2023 06:29:30 +0100 Subject: [PATCH 022/101] Remove checks for leaf node in linux maple tree parsing, node type is enough. --- volatility3/framework/symbols/linux/extensions/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9375a288b..a8bf1b6aa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -344,7 +344,6 @@ class maple_tree(objects.StructType): node_type = ( maple_tree_entry >> self.MAPLE_NODE_TYPE_SHIFT ) & self.MAPLE_NODE_TYPE_MASK - is_leaf = node_type < self.MAPLE_RANGE_64 # create a pointer object for the node parent mte (note this will include flags in the low bits) symbol_table_name = self.get_symbol_table_name() @@ -369,24 +368,20 @@ class maple_tree(objects.StructType): # parse the slots based on the node type if node_type == self.MAPLE_DENSE: - assert is_leaf == True for slot in node.alloc.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield slot elif node_type == self.MAPLE_LEAF_64: - assert is_leaf == True for slot in node.mr64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield slot elif node_type == self.MAPLE_RANGE_64: - assert is_leaf == False for slot in node.mr64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( slot, pointer, maple_tree_depth, seen, depth + 1 ) elif node_type == self.MAPLE_ARANGE_64: - assert is_leaf == False for slot in node.ma64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( From 18a9f898325bf84ad48500a043a4c3b49d4afcdd Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 29 Mar 2023 10:00:55 +0100 Subject: [PATCH 023/101] update logic for checking if a vma should be saved to disk in linux.proc plugin --- volatility3/framework/plugins/linux/proc.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 99c8761c1..e885978b1 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -105,7 +105,6 @@ class Maps(plugins.PluginInterface): Args: context: The context to retrieve required elements (layers, symbol tables) from task: an task_struct instance - vma: The suspected VMA to extract (ObjectInterface) vm_start: The start virtual address from the vma to dump vm_end: The end virtual address from the vma to dump open_method: class to provide context manager for opening the file @@ -127,7 +126,16 @@ class Maps(plugins.PluginInterface): return None vm_size = vm_end - vm_start - if 0 < maxsize < vm_size: + + # check if vm_size is negative, this should never happen. + if vm_size < 0: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is negative." + ) + return None + + # check if vm_size is larger than the maxsize limit, and therefore is not saved out. + if maxsize <= vm_size: vollog.warning( f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is larger than maxsize limit of {maxsize}" ) From 61a2f78baaaa6bc7a957ab5c811494e2923ebf0d Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 29 Mar 2023 10:05:37 +0100 Subject: [PATCH 024/101] fix black linting in linux.proc plugin --- volatility3/framework/plugins/linux/proc.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index e885978b1..2d7348fff 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -124,7 +124,6 @@ class Maps(plugins.PluginInterface): ) ) return None - vm_size = vm_end - vm_start # check if vm_size is negative, this should never happen. @@ -133,14 +132,12 @@ class Maps(plugins.PluginInterface): f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is negative." ) return None - # check if vm_size is larger than the maxsize limit, and therefore is not saved out. if maxsize <= vm_size: vollog.warning( f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is larger than maxsize limit of {maxsize}" ) return None - proc_layer = context.layers[proc_layer_name] file_name = f"pid.{pid}.vma.{vm_start:#x}-{vm_end:#x}.dmp" try: @@ -152,11 +149,9 @@ class Maps(plugins.PluginInterface): data = proc_layer.read(offset, to_read, pad=True) file_handle.write(data) offset += to_read - except Exception as excp: vollog.debug(f"Unable to dump virtual memory {file_name}: {excp}") return None - return file_handle def _generator(self, tasks): @@ -179,11 +174,9 @@ class Maps(plugins.PluginInterface): return False vma_filter_func = vma_filter_function - for task in tasks: if not task.mm: continue - name = utility.array_to_string(task.comm) for vma in self.list_vmas(task, filter_func=vma_filter_func): @@ -200,7 +193,6 @@ class Maps(plugins.PluginInterface): major = inode_object.i_sb.major minor = inode_object.i_sb.minor inode = inode_object.i_ino - path = vma.get_name(self.context, task) file_output = "Disabled" @@ -215,7 +207,6 @@ class Maps(plugins.PluginInterface): ) vm_start = None vm_end = None - if vm_start and vm_end: # only attempt to dump the memory if we have vm_start and vm_end file_handle = self.vma_dump( @@ -230,7 +221,6 @@ class Maps(plugins.PluginInterface): if file_handle: file_handle.close() file_output = file_handle.preferred_filename - yield ( 0, ( From 60292c2da1efa886d0f46bd720af537a6069a623 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 Mar 2023 20:46:06 +0100 Subject: [PATCH 025/101] Linux: Fix slight issue in envvars renaming --- volatility3/framework/plugins/linux/envars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index c4e3ed3c9..758943312 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -1,4 +1,4 @@ -from volatility3.plugins import envvars +from volatility3.plugins.linux import envvars import logging vollog = logging.getLogger(__name__) From 90b9157dcf2ababcd1c52128d72aa9732319701a Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 30 Mar 2023 08:54:56 +0100 Subject: [PATCH 026/101] Linux.proc: Fix broken variable in debug msg. --- volatility3/framework/plugins/linux/proc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 2d7348fff..c1a834bbe 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,6 +21,7 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod @@ -203,7 +204,7 @@ class Maps(plugins.PluginInterface): vm_end = vma.vm_end except AttributeError: vollog.debug( - f"Unable to find the vm_start and vm_end for vma at {vma.vol.offset:#x} for pid {pid}" + f"Unable to find the vm_start and vm_end for vma at {vma.vol.offset:#x} for pid {task.pid}" ) vm_start = None vm_end = None From 8d47f89eddbdf958406290f2da5affeb934dfc7a Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 30 Mar 2023 08:57:43 +0100 Subject: [PATCH 027/101] Linux.proc: Add debug msg when task has no mm member. --- volatility3/framework/plugins/linux/proc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index c1a834bbe..3ce9216fa 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -90,6 +90,10 @@ class Maps(plugins.PluginInterface): vollog.debug( f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.pid} due to filter_func" ) + else: + vollog.debug( + f"Excluded pid {task.pid} as there is no mm member. It is likely a kernel thread." + ) @classmethod def vma_dump( From 41edab23099b1e011d1b0acc4ef538f9ad406ec1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 4 Apr 2023 23:23:27 +0100 Subject: [PATCH 028/101] Plugins: Support yara-4.3.0 and above --- volatility3/framework/plugins/yarascan.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 1c548e5a6..1c8467689 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -34,13 +34,27 @@ class YaraScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules + self.st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( + 4, + 3, + ) def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: for match in self._rules.match(data=data): - for offset, name, value in match.strings: - yield (offset + data_offset, match.rule, name, value) + if self.st_object: + for match_string in match.strings: + for instance in match_string.instances: + yield ( + instance.offset + data_offset, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield (offset + data_offset, match.rule, name, value) class YaraScan(plugins.PluginInterface): From f33ea67e860837f579f7af2d60861baa68f350d5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 4 Apr 2023 23:57:35 +0100 Subject: [PATCH 029/101] Automagic: Update clear-cache and do removals first --- volatility3/framework/__init__.py | 6 +----- volatility3/framework/automagic/symbol_cache.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 479925fb7..c7b23a9c3 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -224,8 +224,4 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: def clear_cache(complete=False): - glob_pattern = "*.cache" - if not complete: - glob_pattern = "data_" + glob_pattern - for cache_filename in glob.glob(os.path.join(constants.CACHE_PATH, glob_pattern)): - os.unlink(cache_filename) + os.unlink(os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index a58bf0091..29f2cfd08 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -310,6 +310,14 @@ class SqliteCache(CacheManagerInterface): new_locations = on_disk_locations.difference(cached_locations) missing_locations = cached_locations.difference(on_disk_locations) + # Missing entries + if missing_locations: + self._database.cursor().execute( + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", + [x for x in missing_locations], + ) + self._database.commit() + cache_update = set() files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: @@ -437,15 +445,6 @@ class SqliteCache(CacheManagerInterface): progress_callback(100, "Reading remote ISF list") self._database.commit() - # Missing entries - - if missing_locations: - self._database.cursor().execute( - f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", - [x for x in missing_locations], - ) - self._database.commit() - def get_identifier_dictionary( self, operating_system: Optional[str] = None, local_only: bool = False ) -> Dict[bytes, str]: From 2e0ecdd770b84aab991aba85ae2d33143212ba43 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 7 Apr 2023 18:04:56 +0900 Subject: [PATCH 030/101] Fix: typo for linux iomem plugin --- volatility3/framework/plugins/linux/iomem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index 8efbf3b57..2056851aa 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -42,7 +42,7 @@ class IOMem(interfaces.plugins.PluginInterface): Args: context: The context to retrieve required elements (layers, symbol tables) from vmlinux_module_name: The name of the kernel module on which to operate - resource_offset: The offset to the resouce to be parsed + resource_offset: The offset to the resource to be parsed seen: The set of resource offsets that have already been parsed depth: How deep into the resource structure we are @@ -57,7 +57,7 @@ class IOMem(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.warning( f"Unable to create resource object at {resource_offset:#x}. This resource, " - "its sibling, and any of it's childern and will be missing from the output." + "its sibling, and any of it's children and will be missing from the output." ) return None From ebb74e2f9292b7cdd211c4886b06fb407e751c80 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Apr 2023 19:57:15 +0100 Subject: [PATCH 031/101] Core: Bump the copyright year of the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 502e26f10..471735af8 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ The latest generated copy of the documentation can be found at: Date: Fri, 14 Apr 2023 08:08:27 +0100 Subject: [PATCH 032/101] Volshell: Mark the script as executable in the repo --- volshell.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 volshell.py diff --git a/volshell.py b/volshell.py old mode 100644 new mode 100755 From 58af80df3c9adfbe8df382b2242087d385c319a7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Apr 2023 13:29:41 +0100 Subject: [PATCH 033/101] Requirements: Shift location_from_file to the requirement from the CLI class --- volatility3/cli/__init__.py | 21 +++++---------- volatility3/framework/automagic/windows.py | 3 +++ .../framework/configuration/requirements.py | 26 +++++++++++++++++++ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 336902d50..99052d82e 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -353,7 +353,9 @@ class CommandLine: ### if args.file: try: - single_location = self.location_from_file(args.file) + single_location = requirements.URLRequirement.location_from_file( + args.file + ) ctx.config["automagic.LayerStacker.single_location"] = single_location except ValueError as excp: parser.error(str(excp)) @@ -456,19 +458,10 @@ class CommandLine: Returns: The URL for the location of the file """ - # We want to work in URLs, but we need to accept absolute and relative files (including on windows) - single_location = parse.urlparse(filename, "") - if single_location.scheme == "" or len(single_location.scheme) == 1: - single_location = parse.urlparse( - parse.urljoin("file:", request.pathname2url(os.path.abspath(filename))) - ) - if single_location.scheme == "file": - if not os.path.exists(request.url2pathname(single_location.path)): - filename = request.url2pathname(single_location.path) - if not filename: - raise ValueError("File URL looks incorrect (potentially missing /)") - raise ValueError(f"File does not exist: {filename}") - return parse.urlunparse(single_location) + vollog.debug( + f"{__name__}.location_from_file has been deprecated and moved to requirements.URIRequirement.location_from_file" + ) + return requirements.URIRequirement.location_from_file(filename) def process_exceptions(self, excp): """Provide useful feedback if an exception occurs during a run of a plugin.""" diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 986eeae22..ccc8de2eb 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -367,6 +367,7 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): progress_callback: constants.ProgressCallback = None, ) -> None: """Finds translation layers that can have swap layers added.""" + path_join = interfaces.configuration.path_join self._translation_requirement = self.find_requirements( context, @@ -382,11 +383,13 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): swap_sub_config, swap_req = self.find_swap_requirement( trans_sub_config, trans_req ) + counter = 0 swap_config = interfaces.configuration.parent_path(swap_sub_config) if swap_req and swap_req.unsatisfied(context, swap_config): # See if any of them need constructing + for swap_location in self.config.get("single_swap_locations", []): # Setup config locations/paths current_layer_name = swap_req.name + str(counter) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 6b64b1cb9..abdffdbe4 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -10,7 +10,9 @@ expect to be in the context (such as particular layers or symboltables). """ import abc import logging +import os from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type +from urllib import parse, request from volatility3.framework import constants, interfaces @@ -55,6 +57,30 @@ class URIRequirement(StringRequirement): # TODO: Maybe a a check that to unsatisfied that the path really is a URL? + @classmethod + def location_from_file(cls, filename: str) -> str: + """Returns the URL location from a file parameter (which may be a URL) + + Args: + filename: The path to the file (either an absolute, relative, or URL path) + + Returns: + The URL for the location of the file + """ + # We want to work in URLs, but we need to accept absolute and relative files (including on windows) + single_location = parse.urlparse(filename, "") + if single_location.scheme == "" or len(single_location.scheme) == 1: + single_location = parse.urlparse( + parse.urljoin("file:", request.pathname2url(os.path.abspath(filename))) + ) + if single_location.scheme == "file": + if not os.path.exists(request.url2pathname(single_location.path)): + filename = request.url2pathname(single_location.path) + if not filename: + raise ValueError("File URL looks incorrect (potentially missing /)") + raise ValueError(f"File does not exist: {filename}") + return parse.urlunparse(single_location) + class BytesRequirement(interfaces.configuration.SimpleTypeRequirement): """A requirement type that contains a byte string.""" From a5e6c550e3fbaa0110d6398b57a570c16914f8aa Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 16 Apr 2023 13:30:55 +0100 Subject: [PATCH 034/101] Automagic: Handle file swap locations and throw a warning if they don't exist --- volatility3/framework/automagic/windows.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index ccc8de2eb..a8530829b 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -401,7 +401,17 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): # Fill in the config if swap_location: context.config[current_layer_path] = current_layer_name - context.config[layer_loc_path] = swap_location + try: + context.config[ + layer_loc_path + ] = requirements.URIRequirement.location_from_file( + swap_location + ) + except ValueError: + vollog.warning( + f"Volatility swap_location {swap_location} could not be validated - swap layer disabled" + ) + continue context.config[ layer_class_path ] = "volatility3.framework.layers.physical.FileLayer" From 33f54f8cf7be2039c3e976917f66ced49ce8365b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 26 Apr 2023 00:33:35 +1000 Subject: [PATCH 035/101] Replace LinuxUtilities._do_get_path() with the new mountinfo _do_get_path() avoiding duplicate code. It also fixes issue #930 --- .../framework/plugins/linux/mountinfo.py | 40 +---- .../framework/symbols/linux/__init__.py | 149 ++++++++++-------- 2 files changed, 84 insertions(+), 105 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index ebd6e55a0..1de776412 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -9,6 +9,7 @@ from typing import Tuple, List, Iterable, Union from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins +from volatility3.framework.symbols import linux from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) @@ -71,40 +72,9 @@ class MountInfo(plugins.PluginInterface): ), ] - @classmethod - def _do_get_path(cls, mnt, fs_root) -> Union[None, str]: - """It mimics the Linux kernel prepend_path function.""" - vfsmnt = mnt.mnt - dentry = vfsmnt.get_mnt_root() - - path_reversed = [] - while dentry != fs_root.dentry or vfsmnt.vol.offset != fs_root.mnt: - if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): - parent = mnt.get_mnt_parent().dereference() - # Escaped? - if dentry != vfsmnt.get_mnt_root(): - return None - - # Global root? - if mnt.vol.offset != parent.vol.offset: - dentry = mnt.get_mnt_mountpoint() - mnt = parent - vfsmnt = mnt.mnt - continue - - return None - - parent = dentry.d_parent - dname = dentry.d_name.name_as_str() - path_reversed.append(dname.strip("/")) - dentry = parent - - path = "/" + "/".join(reversed(path_reversed)) - return path - @classmethod def get_mountinfo( - cls, mnt, task + cls, mnt, task, context ) -> Union[ None, Tuple[int, int, str, str, str, List[str], List[str], str, str, List[str]] ]: @@ -115,8 +85,8 @@ class MountInfo(plugins.PluginInterface): if not mnt_root: return None - path_root = cls._do_get_path(mnt, task.fs.root) - if path_root is None: + path_root = linux.LinuxUtilities._get_path_root(context, mnt, task.fs.root) + if not path_root: return None mnt_root_path = mnt_root.path() @@ -207,7 +177,7 @@ class MountInfo(plugins.PluginInterface): if mnt_ns_ids and mnt_ns_id not in mnt_ns_ids: continue - mnt_info = self.get_mountinfo(mnt, task) + mnt_info = MountInfo.get_mountinfo(mnt, task, self.context) if mnt_info is None: continue diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index ce07167e5..80f5e9990 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,11 +1,11 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Iterator, List, Tuple, Optional +from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects -from volatility3.framework.objects import utility +from volatility3.framework.objects import utility, Pointer from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -59,83 +59,92 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) - # based on __d_path from the Linux kernel @classmethod - def _do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> str: - ret_path: List[str] = [] - - while dentry != rdentry or vfsmnt != rmnt: - dname = dentry.path() - if dname == "": - break - - ret_path.insert(0, dname.strip("/")) - if dentry == vfsmnt.get_mnt_root() or dentry == dentry.d_parent: - if vfsmnt.get_mnt_parent() == vfsmnt: - break - - dentry = vfsmnt.get_mnt_mountpoint() - vfsmnt = vfsmnt.get_mnt_parent() - - continue - - parent = dentry.d_parent - dentry = parent - - # if we did not gather any valid dentrys in the path, then the entire file is - # either 1) smeared out of memory or 2) de-allocated and corresponding structures overwritten - # we return an empty string in this case to avoid confusion with something like a handle to the root - # directory (e.g., "/") - if not ret_path: - return "" - - ret_val = "/".join([str(p) for p in ret_path if p != ""]) - - if ret_val.startswith(("socket:", "pipe:")): - if ret_val.find("]") == -1: - try: - inode = dentry.d_inode - ino = inode.i_ino - except exceptions.InvalidAddressException: - ino = 0 - - ret_val = ret_val[:-1] + f":[{ino}]" - else: - ret_val = ret_val.replace("/", "") - - elif ret_val != "inotify": - ret_val = "/" + ret_val - - return ret_val - - # method used by 'older' kernels - # TODO: lookup when dentry_operations->d_name was merged into the mainline kernel for exact version - @classmethod - def _get_path_file(cls, task, filp) -> str: + def _get_path_file(cls, context, task, filp) -> str: rdentry = task.fs.get_root_dentry() rmnt = task.fs.get_root_mnt() - dentry = filp.get_dentry() vfsmnt = filp.get_vfsmnt() + dentry = filp.get_dentry() - return LinuxUtilities._do_get_path(rdentry, rmnt, dentry, vfsmnt) + return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt, context) + + @classmethod + def _get_path_root(cls, context, mnt, fs_root) -> str: + rdentry = fs_root.dentry + rmnt = fs_root.mnt + vfsmnt = mnt.mnt + dentry = vfsmnt.mnt_root + + return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt, context) + + @classmethod + def _get_vmlinux_from_volobj(cls, volobj, context): + symbol_table_arr = volobj.vol.type_name.split("!", 1) + symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None + + module_names = context.modules.get_modules_by_symbol_tables(symbol_table) + module_names = list(module_names) + + if not module_names: + raise ValueError(f"No module using the symbol table '{symbol_table}'") + + kernel_module_name = module_names[0] + kernel = context.modules[kernel_module_name] + + return kernel + + @classmethod + def _get_mnt_from_vfsmnt(cls, vfsmnt, dentry, context): + vmlinux = cls._get_vmlinux_from_volobj(dentry, context) + + # When it's called from _get_path_file(), 'vfsmnt' is a Pointer + # struct file->f_path->mnt is "struct vfsmount *". + # However, when called from _get_path_root() + # struct mount -> mnt is "struct vfsmount" + vfsmnt_ptr = vfsmnt if type(vfsmnt) == Pointer else vfsmnt.vol.offset + + mnt = cls.container_of(vfsmnt_ptr, "mount", "mnt", vmlinux) + + return mnt + + @classmethod + def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt, context) -> Union[None, str]: + """It mimics the Linux kernel prepend_path function.""" + + mnt = cls._get_mnt_from_vfsmnt(vfsmnt, dentry, context) + + path_reversed = [] + while dentry != rdentry or vfsmnt.vol.offset != rmnt: + if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): + parent = mnt.get_mnt_parent().dereference() + # Escaped? + if dentry != vfsmnt.get_mnt_root(): + break + + # Global root? + if mnt.vol.offset != parent.vol.offset: + dentry = mnt.get_mnt_mountpoint() + mnt = parent + vfsmnt = mnt.mnt + continue + + break + + parent = dentry.d_parent + dname = dentry.d_name.name_as_str() + path_reversed.append(dname.strip("/")) + dentry = parent + + path = "/" + "/".join(reversed(path_reversed)) + return path @classmethod def _get_new_sock_pipe_path(cls, context, task, filp) -> str: dentry = filp.get_dentry() + kernel_module = cls._get_vmlinux_from_volobj(dentry, context) + sym_addr = dentry.d_op.d_dname - - symbol_table_arr = sym_addr.vol.type_name.split("!") - symbol_table = None - if len(symbol_table_arr) == 2: - symbol_table = symbol_table_arr[0] - - for module_name in context.modules.get_modules_by_symbol_tables(symbol_table): - kernel_module = context.modules[module_name] - break - else: - raise ValueError(f"No module using the symbol table {symbol_table}") - symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) if len(symbs) == 1: @@ -151,7 +160,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): pre_name = "pipe" elif sym == "simple_dname": - pre_name = cls._get_path_file(task, filp) + pre_name = cls._get_path_file(context, task, filp) else: pre_name = f"" @@ -192,7 +201,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): if dname_is_valid: ret = LinuxUtilities._get_new_sock_pipe_path(context, task, filp) else: - ret = LinuxUtilities._get_path_file(task, filp) + ret = LinuxUtilities._get_path_file(context, task, filp) return ret From b2d33c2cb0cf8535647f77feac35c44bc124ace5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 26 Apr 2023 00:36:58 +1000 Subject: [PATCH 036/101] Check if 'mnt_namespace' has the 'ns' member before using it --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 5ab8f1aa0..da5bf5dad 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -825,7 +825,7 @@ class mnt_namespace(objects.StructType): def get_inode(self): if self.has_member("proc_inum"): return self.proc_inum - elif self.ns.has_member("inum"): + elif self.has_member("ns") and self.ns.has_member("inum"): return self.ns.inum else: raise AttributeError("Unable to find mnt_namespace inode") From 317f565685c4d63b26d315cc77f74afcb65a525b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 30 Apr 2023 08:28:22 +0100 Subject: [PATCH 037/101] Core: Renable the import protection for volatility3.framework.plugins In commit 21d916b (about 10 months ago) the logic for the warning about volatility3.framework importing was disabled. This re-enables it. Also updates to support pyinstaller 5.10 function stack depth. Closes #944 --- volatility3/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/__init__.py b/volatility3/__init__.py index 28df7da5a..94a6721e1 100644 --- a/volatility3/__init__.py +++ b/volatility3/__init__.py @@ -38,10 +38,13 @@ class WarningFindSpec(abc.MetaPathFinder): """Mock find_spec method that just checks the name, this must go first.""" if fullname.startswith("volatility3.framework.plugins."): - warning = "Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins" + warning = f"Import {fullname}: Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins" # Pyinstaller uses walk_packages/_collect_submodules to import, but needs to read the modules to figure out dependencies # As such, we only print the warning when directly imported rather than from within walk_packages/_collect_submodules - if inspect.stack()[-2].function in ["walk_packages", "_collect_submodules"]: + if inspect.stack()[-2].function not in [ + "walk_packages", + "_collect_submodules", + ] and inspect.stack()[-3].function not in ["_collect_submodules"]: raise Warning(warning) From 77081d3f6832d187eeff5fc0c2d7b64c2655a4b8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 May 2023 05:51:24 +0900 Subject: [PATCH 038/101] Add: code comment for plugins --- volatility3/framework/plugins/windows/crashinfo.py | 2 ++ volatility3/framework/plugins/windows/ldrmodules.py | 2 ++ volatility3/plugins/windows/statistics.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/volatility3/framework/plugins/windows/crashinfo.py b/volatility3/framework/plugins/windows/crashinfo.py index a9f32f63d..4ecd85087 100644 --- a/volatility3/framework/plugins/windows/crashinfo.py +++ b/volatility3/framework/plugins/windows/crashinfo.py @@ -14,6 +14,8 @@ vollog = logging.getLogger(__name__) class Crashinfo(interfaces.plugins.PluginInterface): + """Lists the information from a Windows crash dump.""" + _required_framework_version = (2, 0, 0) @classmethod diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 9642810a5..3c2b8a42d 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -7,6 +7,8 @@ from volatility3.plugins.windows import pslist, vadinfo class LdrModules(interfaces.plugins.PluginInterface): + """Lists the loaded modules in a particular windows memory image.""" + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index e921b3565..4cc05440c 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -13,6 +13,8 @@ vollog = logging.getLogger(__name__) class Statistics(plugins.PluginInterface): + """Lists statistics about the memory space.""""" + _required_framework_version = (2, 0, 0) @classmethod From 79cd3fe2feddf4f1041f2941eb34149c5083ab58 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 May 2023 05:57:56 +0900 Subject: [PATCH 039/101] Lint: black issue for windows.statistics --- volatility3/plugins/windows/statistics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 4cc05440c..9915312e3 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -13,7 +13,7 @@ vollog = logging.getLogger(__name__) class Statistics(plugins.PluginInterface): - """Lists statistics about the memory space.""""" + """Lists statistics about the memory space.""" _required_framework_version = (2, 0, 0) From c06836e45dfe56dfc5c5dc9977cdc638aed4785e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 2 May 2023 05:58:57 +0900 Subject: [PATCH 040/101] Lint: black issue for windows.ldrmodules --- volatility3/framework/plugins/windows/ldrmodules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 3c2b8a42d..4c8456fa9 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -8,7 +8,7 @@ from volatility3.plugins.windows import pslist, vadinfo class LdrModules(interfaces.plugins.PluginInterface): """Lists the loaded modules in a particular windows memory image.""" - + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) From 159e5a2fbd7393934f73a50b6512df9334357e27 Mon Sep 17 00:00:00 2001 From: cpuu Date: Thu, 4 May 2023 11:59:30 +0900 Subject: [PATCH 041/101] Fix requirements.URLRequirement to requirements.URIRequirement [Refactor] Fix requirements.URLRequirement to requirements.URIRequirement in single_location assignment --- volatility3/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 99052d82e..9bfd14c6c 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -353,7 +353,7 @@ class CommandLine: ### if args.file: try: - single_location = requirements.URLRequirement.location_from_file( + single_location = requirements.URIRequirement.location_from_file( args.file ) ctx.config["automagic.LayerStacker.single_location"] = single_location From 14f449367d68c38211c5cba095dee41d38c60f0a Mon Sep 17 00:00:00 2001 From: cpuu Date: Thu, 4 May 2023 12:52:00 +0900 Subject: [PATCH 042/101] Delete getting-started-macos-tutorial.rst --- doc/source/getting-started-macos-tutorial.rst | 154 ------------------ 1 file changed, 154 deletions(-) delete mode 100644 doc/source/getting-started-macos-tutorial.rst diff --git a/doc/source/getting-started-macos-tutorial.rst b/doc/source/getting-started-macos-tutorial.rst deleted file mode 100644 index bc0cb1b92..000000000 --- a/doc/source/getting-started-macos-tutorial.rst +++ /dev/null @@ -1,154 +0,0 @@ -macOS Tutorial -============== - -This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite. - -Acquiring memory ----------------- - -Volatility3 does not provide the ability to acquire memory. The example below is an open source tool. Other commercial tools are also available. - -* `osxpmem `_ - - - -Procedure to create symbol tables for macOS --------------------------------------------- - -To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. - -.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , - which is built and maintained by `volatilityfoundation `_. - After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/mac``. - If necessary create a mac directory under the symbols directory (this will become unnecessary in future versions). - - -Listing plugins ---------------- - -The following is a sample of the macOS plugins available for volatility3, it is not complete and more more plugins may -be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. -For plugin requests, please create an issue with a description of the requested plugin. - -.. code-block:: shell-session - - $ python3 vol.py --help | grep -i mac. | head -n 5 - mac.bash.Bash Recovers bash command history from memory. - mac.check_syscall.Check_syscall - mac.check_sysctl.Check_sysctl - mac.check_trap_table.Check_trap_table - -.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of macOS plugins. - - -Using plugins -------------- - -The following is the syntax to run the volatility CLI. - -.. code-block:: shell-session - - $ python3 vol.py -f - - -Example -------- - -banners -~~~~~~~ - -In this example we will be using a memory dump from the Securinets CTF Quals 2019 Challenge called Contact_me. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge. -Thanks go to `stuxnet `_ for providing this memory dump and `writeup `_. - - -.. code-block:: shell-session - - $ python3 vol.py -f contact_me banners.Banners - - Volatility 3 Framework 2.1.0 - - Progress: 100.00 PDB scanning finished - Offset Banner - - 0x4d2c7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - 0xb42b180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - 0xcda9100 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - 0x1275e7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - 0x1284fba4 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - 0x34ad0180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 - - -The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols/mac`` directory. - -mac.pslist -~~~~~~~~~~~~ - -.. code-block:: shell-session - - $ python3 vol.py -f contact_me mac.pslist - - Volatility 3 Framework 2.1.0 Stacking attempts finished - - PID PPID COMM - - 0 0 kernel_task - 1 0 launchd - 35 1 UserEventAgent - 38 1 kextd - 39 1 fseventsd - 37 1 uninstalld - 45 1 configd - 46 1 powerd - 52 1 logd - 58 1 warmd - ..... - -``mac.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. - -mac.pstree -~~~~~~~~~~~~ - -.. code-block:: shell-session - - $ python3 vol.py -f contact_me mac.pstree - Volatility 3 Framework 2.1.0 - Progress: 100.00 Stacking attempts finished - PID PPID COMM - - 35 1 UserEventAgent - 38 1 kextd - 39 1 fseventsd - 37 1 uninstalld - 204 1 softwareupdated - * 449 204 SoftwareUpdateCo - 337 1 system_installd - * 455 337 update_dyld_shar - -``mac.pstree`` helps us to display the parent child relationships between processes. - -mac.ifconfig -~~~~~~~~~~ - -we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. - - -.. code-block:: shell-session - - $ python3 vol.py -f contact_me mac.ifconfig - - Volatility 3 Framework 2.1.0 - Progress: 100.00 Stacking attempts finished - Interface IP Address Mac Address Promiscuous - - lo0 False - lo0 127.0.0.1 False - lo0 ::1 False - lo0 fe80:1::1 False - gif0 False - stf0 False - en0 00:0C:29:89:8B:F0 00:0C:29:89:8B:F0 False - en0 fe80:4::10fb:c89d:217f:52ae 00:0C:29:89:8B:F0 False - en0 192.168.140.128 00:0C:29:89:8B:F0 False - utun0 False - utun0 fe80:5::2a95:bb15:87e3:977c False From c9e0b6695f2fe88eb2021df504c3ed5eece559e7 Mon Sep 17 00:00:00 2001 From: cpuu Date: Thu, 4 May 2023 12:52:16 +0900 Subject: [PATCH 043/101] Update index.rst --- doc/source/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index e096731c7..9b1d05858 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -26,7 +26,6 @@ There is also some information to get you started quickly: getting-started-linux-tutorial getting-started-windows-tutorial - getting-started-macos-tutorial .. toctree:: From 1ad8c950dab7005e7f603bbf1c34c962c27eb943 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 4 May 2023 13:47:57 +0300 Subject: [PATCH 044/101] Added modules name flag --- volatility3/framework/plugins/windows/modules.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index eba6d1ce7..e25d0425e 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -42,6 +42,7 @@ class Modules(interfaces.plugins.PluginInterface): default=False, optional=True, ), + requirements.StringRequirement(name="name", description="module name/sub string", optional=True, default=""), ] def _generator(self): @@ -64,6 +65,9 @@ class Modules(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: FullDllName = "" + if self.config['name'] and self.config['name'] not in BaseDllName: + continue + file_output = "Disabled" if self.config["dump"]: file_handle = dlllist.DllList.dump_pe( From 9249758dde5c7a1325f25b16ca49b0577d00d714 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 4 May 2023 15:15:23 +0300 Subject: [PATCH 045/101] formatting --- volatility3/framework/plugins/windows/modules.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index e25d0425e..a1d480bb4 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -42,7 +42,12 @@ class Modules(interfaces.plugins.PluginInterface): default=False, optional=True, ), - requirements.StringRequirement(name="name", description="module name/sub string", optional=True, default=""), + requirements.StringRequirement( + name="name", + description="module name/sub string", + optional=True, + default="", + ), ] def _generator(self): @@ -66,7 +71,7 @@ class Modules(interfaces.plugins.PluginInterface): FullDllName = "" if self.config['name'] and self.config['name'] not in BaseDllName: - continue + continue file_output = "Disabled" if self.config["dump"]: From 8952c6c985c8cafd038ae4ed8db0ea4db6b04517 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Thu, 4 May 2023 15:18:37 +0300 Subject: [PATCH 046/101] formatting --- volatility3/framework/plugins/windows/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index a1d480bb4..879da4a5f 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -70,7 +70,7 @@ class Modules(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: FullDllName = "" - if self.config['name'] and self.config['name'] not in BaseDllName: + if self.config["name"] and self.config["name"] not in BaseDllName: continue file_output = "Disabled" From 093fc9ab797e2953ecdfb8cd17c096b3bf573a29 Mon Sep 17 00:00:00 2001 From: cpuu Date: Fri, 5 May 2023 23:39:33 +0900 Subject: [PATCH 047/101] Add tutorial for macOS Analysis Add tutorial for macOS Analysis --- doc/source/getting-started-mac-tutorial.rst | 155 ++++++++++++++++++++ doc/source/index.rst | 1 + 2 files changed, 156 insertions(+) create mode 100644 doc/source/getting-started-mac-tutorial.rst diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst new file mode 100644 index 000000000..3e650fede --- /dev/null +++ b/doc/source/getting-started-mac-tutorial.rst @@ -0,0 +1,155 @@ +macOS Tutorial +============== + +This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite. + +Acquiring memory +---------------- + +Volatility3 does not provide the ability to acquire memory. The example below is an open source tool. Other commercial tools are also available. + +* `osxpmem `_ + + + +Procedure to create symbol tables for macOS +-------------------------------------------- + +To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. + +.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , + which is built and maintained by `volatilityfoundation `_. + After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/mac``. + If necessary create a mac directory under the symbols directory (this will become unnecessary in future versions). + + +Listing plugins +--------------- + +The following is a sample of the macOS plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. + +.. code-block:: shell-session + + $ python3 vol.py --help | grep -i mac. | head -n 4 + mac.bash.Bash Recovers bash command history from memory. + mac.check_syscall.Check_syscall + mac.check_sysctl.Check_sysctl + mac.check_trap_table.Check_trap_table + +.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of macOS plugins. + + +Using plugins +------------- + +The following is the syntax to run the volatility CLI. + +.. code-block:: shell-session + + $ python3 vol.py -f + + +Example +------- + +banners +~~~~~~~ + +In this example we will be using a memory dump from the Securinets CTF Quals 2019 Challenge called Contact_me. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge. +Thanks go to `stuxnet `_ for providing this memory dump and `writeup `_. + + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me banners.Banners + + Volatility 3 Framework 2.4.2 + + Progress: 100.00 PDB scanning finished + Offset Banner + + 0x4d2c7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0xb42b180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0xcda9100 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x1275e7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x1284fba4 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + 0x34ad0180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64 + + +The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols/mac`` directory. + +mac.pslist +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.pslist.PsList + + Volatility 3 Framework 2.4.2 + Progress: 100.00 Stacking attempts finished + + PID PPID COMM + + 0 0 kernel_task + 1 0 launchd + 35 1 UserEventAgent + 38 1 kextd + 39 1 fseventsd + 37 1 uninstalld + 45 1 configd + 46 1 powerd + 52 1 logd + 58 1 warmd + ..... + +``mac.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. + +mac.pstree +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.pstree.PsTree + Volatility 3 Framework 2.4.2 + Progress: 100.00 Stacking attempts finished + PID PPID COMM + + 35 1 UserEventAgent + 38 1 kextd + 39 1 fseventsd + 37 1 uninstalld + 204 1 softwareupdated + * 449 204 SoftwareUpdateCo + 337 1 system_installd + * 455 337 update_dyld_shar + +``mac.pstree`` helps us to display the parent child relationships between processes. + +mac.ifconfig +~~~~~~~~~~ + +we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. + + +.. code-block:: shell-session + + $ python3 vol.py -f contact_me mac.ifconfig.Ifconfig + + Volatility 3 Framework 2.4.2 + Progress: 100.00 Stacking attempts finished + Interface IP Address Mac Address Promiscuous + + lo0 False + lo0 127.0.0.1 False + lo0 ::1 False + lo0 fe80:1::1 False + gif0 False + stf0 False + en0 00:0C:29:89:8B:F0 00:0C:29:89:8B:F0 False + en0 fe80:4::10fb:c89d:217f:52ae 00:0C:29:89:8B:F0 False + en0 192.168.140.128 00:0C:29:89:8B:F0 False + utun0 False + utun0 fe80:5::2a95:bb15:87e3:977c False \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 9b1d05858..7f35e9bcb 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -25,6 +25,7 @@ There is also some information to get you started quickly: :caption: Getting Started getting-started-linux-tutorial + getting-started-mac-tutorial getting-started-windows-tutorial From 7063e6094481e940026b9654f4a52580aa6805cb Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 6 May 2023 14:12:09 +0200 Subject: [PATCH 048/101] mountinfo improvements: + Fixes issue with repeated path suffixes. + Adds support for older kernels (<3.3.8) and + Adds a function which simplify the way the framework internally can gets a context and a vmlinux. This is obtaining the context and symbol space from the same vol object instead of drag a context everywhere. This changes also affects other plugins such as elfs, malfind and proc.maps. It also adds doc strings to some of the existent functions. --- volatility3/framework/plugins/linux/elfs.py | 2 +- .../framework/plugins/linux/malfind.py | 2 +- .../framework/plugins/linux/mountinfo.py | 4 +- volatility3/framework/plugins/linux/proc.py | 2 +- .../framework/symbols/linux/__init__.py | 161 ++++++++----- .../symbols/linux/extensions/__init__.py | 216 ++++++++++++++++-- 6 files changed, 308 insertions(+), 79 deletions(-) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 822a69dd6..fa14dcd49 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -58,7 +58,7 @@ class Elfs(plugins.PluginInterface): ): continue - path = vma.get_name(self.context, task) + path = vma.get_name(task) yield ( 0, diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 18237b80c..552fb8f53 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -47,7 +47,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.mm.get_mmap_iter(): - if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]": + if vma.is_suspicious() and vma.get_name(task) != "[vdso]": data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 1de776412..dfb2e23b4 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -74,7 +74,7 @@ class MountInfo(plugins.PluginInterface): @classmethod def get_mountinfo( - cls, mnt, task, context + cls, mnt, task ) -> Union[ None, Tuple[int, int, str, str, str, List[str], List[str], str, str, List[str]] ]: @@ -85,7 +85,7 @@ class MountInfo(plugins.PluginInterface): if not mnt_root: return None - path_root = linux.LinuxUtilities._get_path_root(context, mnt, task.fs.root) + path_root = linux.LinuxUtilities._get_path_mnt(task, mnt) if not path_root: return None diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 9d8af482e..fa7bc1629 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -59,7 +59,7 @@ class Maps(plugins.PluginInterface): minor = inode_object.i_sb.minor inode = inode_object.i_ino - path = vma.get_name(self.context, task) + path = vma.get_name(task) yield ( 0, diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 80f5e9990..486314dd5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -5,7 +5,7 @@ from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects -from volatility3.framework.objects import utility, Pointer +from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -60,75 +60,74 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) @classmethod - def _get_path_file(cls, context, task, filp) -> str: + def _get_path_file(cls, task, filp) -> str: + """Returns the file pathname relative to the task's root directory. + + Args: + task (task_struct): A reference task + filp (file *): A pointer to an open file + + Returns: + str: File pathname relative to the task's root directory. + """ rdentry = task.fs.get_root_dentry() rmnt = task.fs.get_root_mnt() vfsmnt = filp.get_vfsmnt() dentry = filp.get_dentry() - return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt, context) + return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt) @classmethod - def _get_path_root(cls, context, mnt, fs_root) -> str: - rdentry = fs_root.dentry - rmnt = fs_root.mnt - vfsmnt = mnt.mnt - dentry = vfsmnt.mnt_root + def _get_path_mnt(cls, task, mnt) -> str: + """Returns the mount point pathname relative to the task's root directory. - return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt, context) + Args: + task (task_struct): A reference task + mnt (vfsmount or mount): A mounted filesystem or a mount point. + - kernels < 3.3.8 type is 'vfsmount' + - kernels >= 3.3.8 type is 'mount' + + Returns: + str: Pathname of the mount point relative to the task's root directory. + """ + rdentry = task.fs.get_root_dentry() + rmnt = task.fs.get_root_mnt() + + vfsmnt = mnt.get_vfsmnt_current() + dentry = mnt.get_dentry_current() + + return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt) @classmethod - def _get_vmlinux_from_volobj(cls, volobj, context): - symbol_table_arr = volobj.vol.type_name.split("!", 1) - symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None + def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> Union[None, str]: + """Returns a pathname of the mount point or file + It mimics the Linux kernel prepend_path function. - module_names = context.modules.get_modules_by_symbol_tables(symbol_table) - module_names = list(module_names) + Args: + rdentry (dentry *): A pointer to the root dentry + rmnt (vfsmount *): A pointer to the root vfsmount + dentry (dentry *): A pointer to the dentry + vfsmnt (vfsmount *): A pointer to the vfsmount - if not module_names: - raise ValueError(f"No module using the symbol table '{symbol_table}'") - - kernel_module_name = module_names[0] - kernel = context.modules[kernel_module_name] - - return kernel - - @classmethod - def _get_mnt_from_vfsmnt(cls, vfsmnt, dentry, context): - vmlinux = cls._get_vmlinux_from_volobj(dentry, context) - - # When it's called from _get_path_file(), 'vfsmnt' is a Pointer - # struct file->f_path->mnt is "struct vfsmount *". - # However, when called from _get_path_root() - # struct mount -> mnt is "struct vfsmount" - vfsmnt_ptr = vfsmnt if type(vfsmnt) == Pointer else vfsmnt.vol.offset - - mnt = cls.container_of(vfsmnt_ptr, "mount", "mnt", vmlinux) - - return mnt - - @classmethod - def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt, context) -> Union[None, str]: - """It mimics the Linux kernel prepend_path function.""" - - mnt = cls._get_mnt_from_vfsmnt(vfsmnt, dentry, context) + Returns: + str: Pathname of the mount point or file + """ path_reversed = [] - while dentry != rdentry or vfsmnt.vol.offset != rmnt: + while dentry != rdentry or not vfsmnt.is_equal(rmnt): if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): - parent = mnt.get_mnt_parent().dereference() # Escaped? if dentry != vfsmnt.get_mnt_root(): break # Global root? - if mnt.vol.offset != parent.vol.offset: - dentry = mnt.get_mnt_mountpoint() - mnt = parent - vfsmnt = mnt.mnt - continue + if not vfsmnt.has_parent(): + break - break + dentry = vfsmnt.get_dentry_parent() + vfsmnt = vfsmnt.get_vfsmnt_parent() + + continue parent = dentry.d_parent dname = dentry.d_name.name_as_str() @@ -139,10 +138,19 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return path @classmethod - def _get_new_sock_pipe_path(cls, context, task, filp) -> str: + def _get_new_sock_pipe_path(cls, task, filp) -> str: + """Returns the sock pipe pathname relative to the task's root directory. + + Args: + task (task_struct): A reference task + filp (file *): A pointer to a sock pipe open file + + Returns: + str: Sock pipe pathname relative to the task's root directory. + """ dentry = filp.get_dentry() - kernel_module = cls._get_vmlinux_from_volobj(dentry, context) + kernel_module = cls.get_vmlinux_from_volobj(dentry) sym_addr = dentry.d_op.d_dname symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) @@ -160,7 +168,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): pre_name = "pipe" elif sym == "simple_dname": - pre_name = cls._get_path_file(context, task, filp) + pre_name = cls._get_path_file(task, filp) else: pre_name = f"" @@ -172,10 +180,20 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return ret - # a 'file' structure doesn't have enough information to properly restore its full path - # we need the root mount information from task_struct to determine this @classmethod - def path_for_file(cls, context, task, filp) -> str: + def path_for_file(cls, task, filp) -> str: + """Returns a file (or sock pipe) pathname relative to the task's root directory. + + A 'file' structure doesn't have enough information to properly restore its + full path we need the root mount information from task_struct to determine this + + Args: + task (task_struct): A reference task + filp (file *): A pointer to an open file + + Returns: + str: A file (or sock pipe) pathname relative to the task's root directory. + """ try: dentry = filp.get_dentry() except exceptions.InvalidAddressException: @@ -199,9 +217,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): dname_is_valid = False if dname_is_valid: - ret = LinuxUtilities._get_new_sock_pipe_path(context, task, filp) + ret = LinuxUtilities._get_new_sock_pipe_path(task, filp) else: - ret = LinuxUtilities._get_path_file(context, task, filp) + ret = LinuxUtilities._get_path_file(task, filp) return ret @@ -234,7 +252,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): for fd_num, filp in enumerate(fds): if filp != 0: - full_path = LinuxUtilities.path_for_file(context, task, filp) + full_path = LinuxUtilities.path_for_file(task, filp) yield fd_num, filp, full_path @@ -357,3 +375,30 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return vmlinux.object( object_type=type_name, offset=container_addr, absolute=True ) + + @classmethod + def get_vmlinux_from_volobj(cls, volobj): + """Get the vmlinux from a vol obj + + Args: + volobj (vol object): A vol object + + Raises: + ValueError: If it cannot obtain any module from the symbol table + + Returns: + volatility3.framework.contexts.Module: A kernel object (vmlinux) + """ + symbol_table_arr = volobj.vol.type_name.split("!", 1) + symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None + + module_names = volobj._context.modules.get_modules_by_symbol_tables(symbol_table) + module_names = list(module_names) + + if not module_names: + raise ValueError(f"No module using the symbol table '{symbol_table}'") + + kernel_module_name = module_names[0] + kernel = volobj._context.modules[kernel_module_name] + + return kernel diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index da5bf5dad..b2225f764 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -435,9 +435,9 @@ class vm_area_struct(objects.StructType): return self.vm_pgoff << constants.linux.PAGE_SHIFT - def get_name(self, context, task): + def get_name(self, task): if self.vm_file != 0: - fname = linux.LinuxUtilities.path_for_file(context, task, self.vm_file) + fname = linux.LinuxUtilities.path_for_file(task, self.vm_file) elif self.vm_start <= task.mm.start_brk and self.vm_end >= task.mm.brk: fname = "[heap]" elif self.vm_start <= task.mm.start_stack <= self.vm_end: @@ -544,6 +544,7 @@ class struct_file(objects.StructType): raise AttributeError("Unable to find file -> dentry") def get_vfsmnt(self) -> interfaces.objects.ObjectInterface: + """Returns the fs (vfsmount) where this file is mounted""" if self.has_member("f_vfsmnt"): return self.f_vfsmnt elif self.has_member("f_path"): @@ -675,11 +676,70 @@ class mount(objects.StructType): raise AttributeError("Unable to find mount -> mount flags") def get_mnt_parent(self): + """Gets the fs where we are mounted on + + Returns: + A 'mount *' + """ return self.mnt_parent def get_mnt_mountpoint(self): + """Gets the dentry of the mountpoint + + Returns: + A 'dentry *' + """ + return self.mnt_mountpoint + def get_parent_mount(self): + return self.mnt.get_parent_mount() + + def has_parent(self) -> bool: + """Checks if this mount has a parent + + Returns: + bool: 'True' if this mount has a parent + """ + return self.mnt_parent != self.vol.offset + + def get_vfsmnt_current(self): + """Returns the fs where we are mounted on + + Returns: + A 'vfsmount' + """ + return self.mnt + + def get_vfsmnt_parent(self): + """Gets the parent fs (vfsmount) to where it's mounted on + + Returns: + A 'vfsmount' + """ + + return self.get_mnt_parent().get_vfsmnt_current() + + def get_dentry_current(self): + """Returns the root of the mounted tree + + Returns: + A 'dentry *' + """ + vfsmnt = self.get_vfsmnt_current() + dentry = vfsmnt.mnt_root + + return dentry + + def get_dentry_parent(self): + """Returns the parent root of the mounted tree + + Returns: + A 'dentry *' + """ + + return self.get_mnt_parent().get_dentry_current() + def get_flags_access(self) -> str: return "ro" if self.get_mnt_flags() & self.MNT_READONLY else "rw" @@ -703,9 +763,6 @@ class mount(objects.StructType): def get_devname(self) -> str: return utility.pointer_to_string(self.mnt_devname, count=255) - def has_parent(self) -> bool: - return self.vol.offset != self.mnt_parent - def get_dominating_id(self, root) -> int: """Get ID of closest dominating peer group having a representative under the given root.""" mnt_seen = set() @@ -783,24 +840,117 @@ class vfsmount(objects.StructType): and self.get_mnt_parent() != 0 ) - def _get_real_mnt(self): - table_name = self.vol.type_name.split(constants.BANG)[0] - mount_struct = f"{table_name}{constants.BANG}mount" - offset = self._context.symbol_space.get_type( - mount_struct - ).relative_child_offset("mnt") + def _is_kernel_prior_to_struct_mount(self) -> bool: + """Helper to distinguish between kernels prior to version 3.3.8 that + lacked the 'mount' structure and later versions that have it. - return self._context.object( - mount_struct, self.vol.layer_name, offset=self.vol.offset - offset - ) + The 'mnt_parent' member was moved from struct 'vfsmount' to struct + 'mount' when the latter was introduced. + + Alternatively, vmlinux.has_type('mount') can be used here but it is faster. + + Returns: + bool: 'True' if the kernel + """ + + return self.has_member("mnt_parent") + + def is_equal(self, vfsmount_ptr) -> bool: + """Helper to make sure it is comparing two pointers to 'vfsmount'. + + Depending on the kernel version, the calling object (self) could be + a 'vfsmount *' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust + in the framework "auto" dereferencing ability to assure that when we + reach this point 'self' will be a 'vfsmount' already and self.vol.offset + a 'vfsmount *' and not a 'vfsmount **'. The argument must be a 'vfsmount *'. + Typically, it's called from do_get_path(). + + Args: + vfsmount_ptr (vfsmount *): A pointer to a 'vfsmount' + + Raises: + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount *' + + Returns: + bool: 'True' if the given argument points to the the same 'vfsmount' + as 'self'. + """ + if type(vfsmount_ptr) == objects.Pointer: + return self.vol.offset == vfsmount_ptr + else: + raise exceptions.VolatilityException("Unexpected argument type. It has to be a 'vfsmount *'") + + def _get_real_mnt(self): + """Gets the struct 'mount' containing this 'vfsmount'. + + It should be only called from kernels >= 3.3.8 when 'struct mount' was introduced. + + Returns: + mount: the struct 'mount' containing this 'vfsmount'. + """ + vmlinux = linux.LinuxUtilities.get_vmlinux_from_volobj(self) + return linux.LinuxUtilities.container_of(self.vol.offset, "mount", "mnt", vmlinux) + + def get_vfsmnt_current(self): + """Returns the current fs where we are mounted on + + Returns: + A 'vfsmount *' + """ + return self.get_mnt_parent() + + def get_vfsmnt_parent(self): + """Gets the parent fs (vfsmount) to where it's mounted on + + Returns: + For kernels < 3.3.8: A 'vfsmount *' + For kernels >= 3.3.8: A 'vfsmount' + """ + if self._is_kernel_prior_to_struct_mount(): + return self.get_mnt_parent() + else: + return self._get_real_mnt().get_vfsmnt_parent() + + def get_dentry_current(self): + """Returns the root of the mounted tree + + Returns: + A 'dentry *' + """ + if self._is_kernel_prior_to_struct_mount(): + return self.get_mnt_mountpoint() + else: + return self._get_real_mnt().get_dentry_current() + + def get_dentry_parent(self): + """Returns the parent root of the mounted tree + + Returns: + A 'dentry *' + """ + if self._is_kernel_prior_to_struct_mount(): + return self.get_mnt_mountpoint() + else: + return self._get_real_mnt().get_mnt_mountpoint() def get_mnt_parent(self): - if self.has_member("mnt_parent"): + """Gets the mnt_parent member. + + Returns: + For kernels < 3.3.8: A 'vfsmount *' + For kernels >= 3.3.8: A 'mount *' + """ + if self._is_kernel_prior_to_struct_mount(): return self.mnt_parent else: - return self._get_real_mnt().mnt_parent + return self._get_real_mnt().get_mnt_parent() def get_mnt_mountpoint(self): + """Gets the dentry of the mountpoint + + Returns: + A 'dentry *' + """ if self.has_member("mnt_mountpoint"): return self.mnt_mountpoint else: @@ -809,6 +959,40 @@ class vfsmount(objects.StructType): def get_mnt_root(self): return self.mnt_root + def has_parent(self) -> bool: + if self._is_kernel_prior_to_struct_mount(): + return self.mnt_parent != self.vol.offset + else: + return self._get_real_mnt().has_parent() + + def get_mnt_sb(self): + return self.mnt_sb + + def get_flags_access(self) -> str: + return "ro" if self.mnt_flags & mount.MNT_READONLY else "rw" + + def get_flags_opts(self) -> Iterable[str]: + flags = [ + mntflagtxt + for mntflag, mntflagtxt in mount.MNT_FLAGS.items() + if mntflag & self.mnt_flags != 0 + ] + return flags + + def get_mnt_flags(self): + return self.mnt_flags + + def is_shared(self) -> bool: + return self.get_mnt_flags() & mount.MNT_SHARED + + def is_unbindable(self) -> bool: + return self.get_mnt_flags() & mount.MNT_UNBINDABLE + + def is_slave(self) -> bool: + return self.mnt_master and self.mnt_master.vol.offset != 0 + + def get_devname(self) -> str: + return utility.pointer_to_string(self.mnt_devname, count=255) class kobject(objects.StructType): def reference_count(self): From 2d922a8cd645c318e4e10c5d286bc86589f64f8b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 6 May 2023 14:18:33 +0200 Subject: [PATCH 049/101] Improves the way the mount points and the namespaces are filtered. Previously, it was filtering by mount namespaces id. Even that approach was working as expected, it's not viable for older kernels were there was not a mount namespace ID. With this changes we filtered the mount points individually by the mount ID which is unique system wide, no mather the namespace to which it belongs to. --- .../framework/plugins/linux/mountinfo.py | 70 ++++++++++++------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index dfb2e23b4..e03659aec 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -12,6 +12,7 @@ from volatility3.framework.interfaces import plugins from volatility3.framework.symbols import linux from volatility3.plugins.linux import pslist + vollog = logging.getLogger(__name__) MountInfoData = namedtuple( @@ -140,30 +141,38 @@ class MountInfo(plugins.PluginInterface): ) def _get_tasks_mountpoints( - self, tasks: Iterable[interfaces.objects.ObjectInterface], per_namespace: bool + self, tasks: Iterable[interfaces.objects.ObjectInterface], filtered_by_pids: bool ): - seen_namespaces = set() + seen_mountpoints = set() for task in tasks: if not ( - task - and task.fs - and task.fs.root - and task.nsproxy - and task.nsproxy.mnt_ns + task and + task.fs and + task.fs.root and + task.nsproxy and + task.nsproxy.mnt_ns ): - # This task doesn't have all the information required + # This task doesn't have all the information required. + # It should be a kernel < 2.6.30 continue mnt_namespace = task.nsproxy.mnt_ns - mnt_ns_id = mnt_namespace.get_inode() - - if per_namespace: - if mnt_ns_id in seen_namespaces: - continue - else: - seen_namespaces.add(mnt_ns_id) + try: + mnt_ns_id = str(mnt_namespace.get_inode()) + except AttributeError: + mnt_ns_id = renderers.NotAvailableValue() for mount in mnt_namespace.get_mount_points(): + # When PIDs are filtered, it makes sense that the user want to + # see each of those processes mount points. So we don't filter + # by mount id in this case. + if not filtered_by_pids: + mnt_id = int(mount.mnt_id) + if mnt_id in seen_mountpoints: + continue + else: + seen_mountpoints.add(mnt_id) + yield task, mount, mnt_ns_id def _generator( @@ -171,13 +180,26 @@ class MountInfo(plugins.PluginInterface): tasks: Iterable[interfaces.objects.ObjectInterface], mnt_ns_ids: List[int], mount_format: bool, - per_namespace: bool, + filtered_by_pids: bool, ) -> Iterable[Tuple[int, Tuple]]: - for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, per_namespace): - if mnt_ns_ids and mnt_ns_id not in mnt_ns_ids: + warning_shown = False + for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, filtered_by_pids): + if ( + not warning_shown and + mnt_ns_ids and + isinstance(mnt_ns_id, renderers.NotAvailableValue) + ): + vollog.warning("Cannot filter by namespace id, it is not available in this kernel.") + warning_shown = True + + if ( + not isinstance(mnt_ns_id, renderers.NotAvailableValue) and + mnt_ns_ids and + mnt_ns_id not in mnt_ns_ids + ): continue - mnt_info = MountInfo.get_mountinfo(mnt, task, self.context) + mnt_info = self.get_mountinfo(mnt, task) if mnt_info is None: continue @@ -212,7 +234,7 @@ class MountInfo(plugins.PluginInterface): ] fields_values = [mnt_ns_id] - if not per_namespace: + if filtered_by_pids: fields_values.append(task.pid) fields_values.extend(extra_fields_values) @@ -228,14 +250,14 @@ class MountInfo(plugins.PluginInterface): self.context, self.config["kernel"], filter_func=pid_filter ) - columns = [("MNT_NS_ID", int)] + columns = [("MNT_NS_ID", str)] # The PID column does not make sense when a PID filter is not specified. In that case, the default behavior is # to displays the mountpoints per namespace. if pids: columns.append(("PID", int)) - per_namespace = False + filtered_by_pids = True else: - per_namespace = True + filtered_by_pids = False if self.config.get("mount-format"): extra_columns = [ @@ -262,5 +284,5 @@ class MountInfo(plugins.PluginInterface): columns.extend(extra_columns) return renderers.TreeGrid( - columns, self._generator(tasks, mount_ns_ids, mount_format, per_namespace) + columns, self._generator(tasks, mount_ns_ids, mount_format, filtered_by_pids) ) From a09f77897b24b4b0da06a8b45a353ee730cadd08 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 6 May 2023 14:21:58 +0200 Subject: [PATCH 050/101] sockstat improvements: + Fixes issues with netlink sockets for older kernels, supporting now kernels < 3.7.10 + Fixes issue with network namespace id for older kernels. --- .../framework/plugins/linux/sockstat.py | 26 +++++++++++++++---- .../symbols/linux/extensions/__init__.py | 24 ++++++++++++++++- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index f03a2ad8e..e37b8a1bb 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -28,7 +28,11 @@ class SockHandlers(interfaces.configuration.VersionableInterface): self._vmlinux = vmlinux self._task = task - netns_id = task.nsproxy.net_ns.get_inode() + try: + netns_id = task.nsproxy.net_ns.get_inode() + except AttributeError: + netns_id = NotAvailableValue() + self._netdevices = self._build_network_devices_map(netns_id) self._sock_family_handlers = { @@ -61,7 +65,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): self._vmlinux.symbol_table_name + constants.BANG + "net_device" ) for net_dev in net.dev_base_head.to_list(net_device_symname, "dev_list"): - if net.get_inode() != netns_id: + if isinstance(netns_id, NotAvailableValue) or net.get_inode() != netns_id: continue dev_name = utility.array_to_string(net_dev.name) netdevices_map[net_dev.ifindex] = dev_name @@ -227,14 +231,22 @@ class SockHandlers(interfaces.configuration.VersionableInterface): if netlink_sock.groups: groups_bitmap = netlink_sock.groups.dereference() src_addr = f"groups:0x{groups_bitmap:08x}" - src_port = netlink_sock.portid + + try: + # Kernel >= 3.7.10 + src_port = netlink_sock.get_portid() + except AttributeError: + src_port = NotAvailableValue() dst_addr = f"group:0x{netlink_sock.dst_group:08x}" module = netlink_sock.module if module and module.name: module_name_str = utility.array_to_string(module.name) dst_addr = f"{dst_addr},lkm:{module_name_str}" - dst_port = netlink_sock.dst_portid + try: + dst_port = netlink_sock.get_dst_portid() + except AttributeError: + dst_port = NotAvailableValue() state = netlink_sock.get_state() @@ -518,7 +530,11 @@ class Sockstat(plugins.PluginInterface): protocol = child_sock.get_protocol() net = task.nsproxy.net_ns - netns_id = net.get_inode() + try: + netns_id = net.get_inode() + except AttributeError: + netns_id = NotAvailableValue() + yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields def _format_fields(self, sock_stat, protocol): diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b2225f764..d97916b74 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1028,10 +1028,13 @@ class mnt_namespace(objects.StructType): class net(objects.StructType): def get_inode(self): if self.has_member("proc_inum"): + # 3.8.13 <= kernel < 3.19.8 return self.proc_inum - elif self.ns.has_member("inum"): + elif self.has_member("ns") and self.ns.has_member("inum"): + # kernel >= 3.19.8 return self.ns.inum else: + # kernel < 3.8.13 raise AttributeError("Unable to find net_namespace inode") @@ -1239,6 +1242,25 @@ class netlink_sock(objects.StructType): # Return the generic socket state return self.sk.sk_socket.get_state() + def get_portid(self): + if self.has_member("pid"): + # kernel < 3.7.10 + return self.pid + if self.has_member("portid"): + # kernel >= 3.7.10 + return self.portid + else: + raise AttributeError("Unable to find a source port id") + + def get_dst_portid(self): + if self.has_member("dst_pid"): + # kernel < 3.7.10 + return self.dst_pid + if self.has_member("dst_portid"): + # kernel >= 3.7.10 + return self.dst_portid + else: + raise AttributeError("Unable to find a destination port id") class vsock_sock(objects.StructType): def get_protocol(self): From c40b3a043f2c91506514c9fa7cdc7d3bcce81116 Mon Sep 17 00:00:00 2001 From: Paul Kermann Date: Mon, 8 May 2023 05:07:19 +0300 Subject: [PATCH 051/101] CR fixes --- volatility3/framework/plugins/windows/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 879da4a5f..c1be1b0a6 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -46,7 +46,7 @@ class Modules(interfaces.plugins.PluginInterface): name="name", description="module name/sub string", optional=True, - default="", + default=None, ), ] From ea09c4732843f27830d2afe859203d825fa15b76 Mon Sep 17 00:00:00 2001 From: cpuu Date: Mon, 8 May 2023 17:32:14 +0900 Subject: [PATCH 052/101] Update getting-started-mac-tutorial.rst --- doc/source/getting-started-mac-tutorial.rst | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index 3e650fede..cb0dda845 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -17,16 +17,11 @@ Procedure to create symbol tables for macOS To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. -.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , - which is built and maintained by `volatilityfoundation `_. - After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/mac``. - If necessary create a mac directory under the symbols directory (this will become unnecessary in future versions). - Listing plugins --------------- -The following is a sample of the macOS plugins available for volatility3, it is not complete and more more plugins may +The following is a sample of the macOS plugins available for volatility3, it is not complete and more plugins may be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. For plugin requests, please create an issue with a description of the requested plugin. @@ -79,7 +74,7 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols/mac`` directory. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. mac.pslist ~~~~~~~~~~~~ @@ -131,9 +126,6 @@ mac.pstree mac.ifconfig ~~~~~~~~~~ -we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. - - .. code-block:: shell-session $ python3 vol.py -f contact_me mac.ifconfig.Ifconfig @@ -152,4 +144,6 @@ we can use the ``mac.ifconfig`` plugin to get information about the configuratio en0 fe80:4::10fb:c89d:217f:52ae 00:0C:29:89:8B:F0 False en0 192.168.140.128 00:0C:29:89:8B:F0 False utun0 False - utun0 fe80:5::2a95:bb15:87e3:977c False \ No newline at end of file + utun0 fe80:5::2a95:bb15:87e3:977c False + + we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. From e86ff963e2325d33bf2fff612e9f494afbea836b Mon Sep 17 00:00:00 2001 From: cpuu Date: Mon, 8 May 2023 17:43:24 +0900 Subject: [PATCH 053/101] Update getting-started-mac-tutorial.rst --- doc/source/getting-started-mac-tutorial.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index cb0dda845..d70d260ca 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -17,6 +17,10 @@ Procedure to create symbol tables for macOS To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. +.. tip:: It may be possible to locate pre-made ISF files from the `download link `_ , + which is built and maintained by `volatilityfoundation `_. + After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/``. + Listing plugins --------------- From 6b2ae6bd653b384d7e5001272878450370da74d0 Mon Sep 17 00:00:00 2001 From: cpuu Date: Mon, 8 May 2023 17:44:17 +0900 Subject: [PATCH 054/101] Update getting-started-mac-tutorial.rst --- doc/source/getting-started-mac-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index d70d260ca..cfb0afa9a 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -150,4 +150,4 @@ mac.ifconfig utun0 False utun0 fe80:5::2a95:bb15:87e3:977c False - we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. +we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. From ced6ff346cd8694ed02f835da8bec62f60ff9b56 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 8 May 2023 11:54:48 +0200 Subject: [PATCH 055/101] Apply 'Black' suggestions --- .../framework/plugins/linux/mountinfo.py | 37 +++++++++++-------- .../framework/plugins/linux/sockstat.py | 5 ++- .../framework/symbols/linux/__init__.py | 4 +- .../symbols/linux/extensions/__init__.py | 10 ++++- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index e03659aec..0606884ff 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -141,16 +141,18 @@ class MountInfo(plugins.PluginInterface): ) def _get_tasks_mountpoints( - self, tasks: Iterable[interfaces.objects.ObjectInterface], filtered_by_pids: bool + self, + tasks: Iterable[interfaces.objects.ObjectInterface], + filtered_by_pids: bool, ): seen_mountpoints = set() for task in tasks: if not ( - task and - task.fs and - task.fs.root and - task.nsproxy and - task.nsproxy.mnt_ns + task + and task.fs + and task.fs.root + and task.nsproxy + and task.nsproxy.mnt_ns ): # This task doesn't have all the information required. # It should be a kernel < 2.6.30 @@ -183,19 +185,23 @@ class MountInfo(plugins.PluginInterface): filtered_by_pids: bool, ) -> Iterable[Tuple[int, Tuple]]: warning_shown = False - for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, filtered_by_pids): + for task, mnt, mnt_ns_id in self._get_tasks_mountpoints( + tasks, filtered_by_pids + ): if ( - not warning_shown and - mnt_ns_ids and - isinstance(mnt_ns_id, renderers.NotAvailableValue) + not warning_shown + and mnt_ns_ids + and isinstance(mnt_ns_id, renderers.NotAvailableValue) ): - vollog.warning("Cannot filter by namespace id, it is not available in this kernel.") + vollog.warning( + "Cannot filter by namespace id, it is not available in this kernel." + ) warning_shown = True if ( - not isinstance(mnt_ns_id, renderers.NotAvailableValue) and - mnt_ns_ids and - mnt_ns_id not in mnt_ns_ids + not isinstance(mnt_ns_id, renderers.NotAvailableValue) + and mnt_ns_ids + and mnt_ns_id not in mnt_ns_ids ): continue @@ -284,5 +290,6 @@ class MountInfo(plugins.PluginInterface): columns.extend(extra_columns) return renderers.TreeGrid( - columns, self._generator(tasks, mount_ns_ids, mount_format, filtered_by_pids) + columns, + self._generator(tasks, mount_ns_ids, mount_format, filtered_by_pids), ) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index e37b8a1bb..72a1e453e 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -65,7 +65,10 @@ class SockHandlers(interfaces.configuration.VersionableInterface): self._vmlinux.symbol_table_name + constants.BANG + "net_device" ) for net_dev in net.dev_base_head.to_list(net_device_symname, "dev_list"): - if isinstance(netns_id, NotAvailableValue) or net.get_inode() != netns_id: + if ( + isinstance(netns_id, NotAvailableValue) + or net.get_inode() != netns_id + ): continue dev_name = utility.array_to_string(net_dev.name) netdevices_map[net_dev.ifindex] = dev_name diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 486314dd5..858845125 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -392,7 +392,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): symbol_table_arr = volobj.vol.type_name.split("!", 1) symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None - module_names = volobj._context.modules.get_modules_by_symbol_tables(symbol_table) + module_names = volobj._context.modules.get_modules_by_symbol_tables( + symbol_table + ) module_names = list(module_names) if not module_names: diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d97916b74..3dbf560c6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -878,7 +878,9 @@ class vfsmount(objects.StructType): if type(vfsmount_ptr) == objects.Pointer: return self.vol.offset == vfsmount_ptr else: - raise exceptions.VolatilityException("Unexpected argument type. It has to be a 'vfsmount *'") + raise exceptions.VolatilityException( + "Unexpected argument type. It has to be a 'vfsmount *'" + ) def _get_real_mnt(self): """Gets the struct 'mount' containing this 'vfsmount'. @@ -889,7 +891,9 @@ class vfsmount(objects.StructType): mount: the struct 'mount' containing this 'vfsmount'. """ vmlinux = linux.LinuxUtilities.get_vmlinux_from_volobj(self) - return linux.LinuxUtilities.container_of(self.vol.offset, "mount", "mnt", vmlinux) + return linux.LinuxUtilities.container_of( + self.vol.offset, "mount", "mnt", vmlinux + ) def get_vfsmnt_current(self): """Returns the current fs where we are mounted on @@ -994,6 +998,7 @@ class vfsmount(objects.StructType): def get_devname(self) -> str: return utility.pointer_to_string(self.mnt_devname, count=255) + class kobject(objects.StructType): def reference_count(self): refcnt = self.kref.refcount @@ -1262,6 +1267,7 @@ class netlink_sock(objects.StructType): else: raise AttributeError("Unable to find a destination port id") + class vsock_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for vsocks From 200099f8b3f92e2ad228e0a481a835f1fbc9d8b6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 8 May 2023 14:58:51 +0200 Subject: [PATCH 056/101] Improve support for socket filters in kernels < 4.1.52 --- .../framework/plugins/linux/sockstat.py | 23 +++++++++++++++---- .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 17 ++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 72a1e453e..f06b3ad8e 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -150,19 +150,32 @@ class SockHandlers(interfaces.configuration.VersionableInterface): return bpfprog = sock_filter.prog - if bpfprog.type == 0: - # BPF_PROG_TYPE_UNSPEC = 0 + + BPF_PROG_TYPE_UNSPEC = 0 # cBPF filter + try: + bpfprog_type = bpfprog.get_type() + if bpfprog_type == BPF_PROG_TYPE_UNSPEC: + return # cBPF filter + except AttributeError: + # kernel < 3.18.140, it's a cBPF filter + return + + BPF_PROG_TYPE_SOCKET_FILTER = 1 # eBPF filter + if bpfprog_type != BPF_PROG_TYPE_SOCKET_FILTER: + socket_filter["bpf_filter_type"] = f"UNK({bpfprog_type})" + vollog.warning(f"Unexpected BPF type {bpfprog_type} for a socket") return socket_filter["bpf_filter_type"] = "eBPF" if not bpfprog.has_member("aux") or not bpfprog.aux: - return + return # kernel < 3.18.140 bpfprog_aux = bpfprog.aux + if bpfprog_aux.has_member("id"): - # `id` member was added to `bpf_prog_aux` in kernels 4.13 + # `id` member was added to `bpf_prog_aux` in kernels 4.13.16 socket_filter["bpf_filter_id"] = str(bpfprog_aux.id) if bpfprog_aux.has_member("name"): - # `name` was added to `bpf_prog_aux` in kernels 4.15 + # `name` was added to `bpf_prog_aux` in kernels 4.15.18 bpfprog_name = utility.array_to_string(bpfprog_aux.name) if bpfprog_name: socket_filter["bpf_filter_name"] = bpfprog_name diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 858845125..3780a86f0 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -30,6 +30,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("kobject", extensions.kobject) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) + self.optional_set_type_class("bpf_prog", extensions.bpf_prog) # Mount self.set_type_class("vfsmount", extensions.vfsmount) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3dbf560c6..87bd76554 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1313,3 +1313,20 @@ class xdp_sock(objects.StructType): def get_state(self): # xdp_sock.state is an enum return self.state.lookup() + + +class bpf_prog(objects.StructType): + def get_type(self): + # The program type was in `bpf_prog_aux::prog_type` from 3.18.140 to + # 4.1.52 before it was moved to `bpf_prog::type` + if self.has_member("type"): + # kernel >= 4.1.52 + return self.type + + if self.has_member("aux") and self.aux: + if self.aux.has_member("prog_type"): + # 3.18.140 <= kernel < 4.1.52 + return self.aux.prog_type + + # kernel < 3.18.140 + raise AttributeError("Unable to find the BPF type") From 7d65c20cc0b971c862889065d8a90d98e24819c9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 8 May 2023 14:59:58 +0200 Subject: [PATCH 057/101] Fix f-string --- volatility3/framework/plugins/linux/iomem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index 2056851aa..785405ef3 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -66,7 +66,7 @@ class IOMem(interfaces.plugins.PluginInterface): name = utility.pointer_to_string(resource.name, 128) except exceptions.InvalidAddressException: vollog.warning( - "Unable to follow pointer to name for resource object at {resource_offset:#x}, " + f"Unable to follow pointer to name for resource object at {resource_offset:#x}, " "replaced with UnreadableValue" ) name = renderers.UnreadableValue() From de28b5ab7077c04ac776018264c22484d1766c37 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 8 May 2023 19:59:53 +0200 Subject: [PATCH 058/101] Rollback explicit context --- volatility3/framework/plugins/linux/elfs.py | 2 +- .../framework/plugins/linux/malfind.py | 2 +- volatility3/framework/plugins/linux/proc.py | 2 +- .../framework/symbols/linux/__init__.py | 27 +++++++++++-------- .../symbols/linux/extensions/__init__.py | 6 ++--- 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index fa14dcd49..822a69dd6 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -58,7 +58,7 @@ class Elfs(plugins.PluginInterface): ): continue - path = vma.get_name(task) + path = vma.get_name(self.context, task) yield ( 0, diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 552fb8f53..18237b80c 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -47,7 +47,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.mm.get_mmap_iter(): - if vma.is_suspicious() and vma.get_name(task) != "[vdso]": + if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]": data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index fa7bc1629..9d8af482e 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -59,7 +59,7 @@ class Maps(plugins.PluginInterface): minor = inode_object.i_sb.minor inode = inode_object.i_ino - path = vma.get_name(task) + path = vma.get_name(self.context, task) yield ( 0, diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3780a86f0..3f51d3c0c 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -139,10 +139,11 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return path @classmethod - def _get_new_sock_pipe_path(cls, task, filp) -> str: + def _get_new_sock_pipe_path(cls, context, task, filp) -> str: """Returns the sock pipe pathname relative to the task's root directory. Args: + context: The context to retrieve required elements (layers, symbol tables) from task (task_struct): A reference task filp (file *): A pointer to a sock pipe open file @@ -151,7 +152,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): """ dentry = filp.get_dentry() - kernel_module = cls.get_vmlinux_from_volobj(dentry) + kernel_module = cls.get_vmlinux_from_volobj(context, dentry) sym_addr = dentry.d_op.d_dname symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) @@ -182,13 +183,14 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return ret @classmethod - def path_for_file(cls, task, filp) -> str: + def path_for_file(cls, context, task, filp) -> str: """Returns a file (or sock pipe) pathname relative to the task's root directory. A 'file' structure doesn't have enough information to properly restore its full path we need the root mount information from task_struct to determine this Args: + context: The context to retrieve required elements (layers, symbol tables) from task (task_struct): A reference task filp (file *): A pointer to an open file @@ -218,7 +220,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): dname_is_valid = False if dname_is_valid: - ret = LinuxUtilities._get_new_sock_pipe_path(task, filp) + ret = LinuxUtilities._get_new_sock_pipe_path(context, task, filp) else: ret = LinuxUtilities._get_path_file(task, filp) @@ -253,7 +255,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): for fd_num, filp in enumerate(fds): if filp != 0: - full_path = LinuxUtilities.path_for_file(task, filp) + full_path = LinuxUtilities.path_for_file(context, task, filp) yield fd_num, filp, full_path @@ -378,30 +380,33 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ) @classmethod - def get_vmlinux_from_volobj(cls, volobj): + def get_vmlinux_from_volobj( + cls, + context: interfaces.context.ContextInterface, + volobj: interfaces.objects.ObjectInterface, + ) -> interfaces.context.ModuleInterface: """Get the vmlinux from a vol obj Args: + context: The context to retrieve required elements (layers, symbol tables) from volobj (vol object): A vol object Raises: ValueError: If it cannot obtain any module from the symbol table Returns: - volatility3.framework.contexts.Module: A kernel object (vmlinux) + A kernel object (vmlinux) """ symbol_table_arr = volobj.vol.type_name.split("!", 1) symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None - module_names = volobj._context.modules.get_modules_by_symbol_tables( - symbol_table - ) + module_names = context.modules.get_modules_by_symbol_tables(symbol_table) module_names = list(module_names) if not module_names: raise ValueError(f"No module using the symbol table '{symbol_table}'") kernel_module_name = module_names[0] - kernel = volobj._context.modules[kernel_module_name] + kernel = context.modules[kernel_module_name] return kernel diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 87bd76554..1caab3ce5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -435,9 +435,9 @@ class vm_area_struct(objects.StructType): return self.vm_pgoff << constants.linux.PAGE_SHIFT - def get_name(self, task): + def get_name(self, context, task): if self.vm_file != 0: - fname = linux.LinuxUtilities.path_for_file(task, self.vm_file) + fname = linux.LinuxUtilities.path_for_file(context, task, self.vm_file) elif self.vm_start <= task.mm.start_brk and self.vm_end >= task.mm.brk: fname = "[heap]" elif self.vm_start <= task.mm.start_stack <= self.vm_end: @@ -890,7 +890,7 @@ class vfsmount(objects.StructType): Returns: mount: the struct 'mount' containing this 'vfsmount'. """ - vmlinux = linux.LinuxUtilities.get_vmlinux_from_volobj(self) + vmlinux = linux.LinuxUtilities.get_vmlinux_from_volobj(self._context, self) return linux.LinuxUtilities.container_of( self.vol.offset, "mount", "mnt", vmlinux ) From 859482445907fd9c2ae579d37e0a898b10857690 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 9 May 2023 06:47:07 +0100 Subject: [PATCH 059/101] Linux: Ensure that objects made when parsing maple tree use the native_layer_name --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index a8bf1b6aa..931003b60 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -349,7 +349,7 @@ class maple_tree(objects.StructType): symbol_table_name = self.get_symbol_table_name() node_parent_mte = self._context.object( symbol_table_name + constants.BANG + "pointer", - layer_name=self.vol.layer_name, + layer_name=self.vol.native_layer_name, offset=pointer, ) @@ -424,7 +424,7 @@ class mm_struct(objects.StructType): # convert pointer to vm_area_struct and yield vma = self._context.object( symbol_table_name + constants.BANG + "vm_area_struct", - layer_name=self.vol.layer_name, + layer_name=self.vol.native_layer_name, offset=vma_pointer ) yield vma From a2906ad270b3ec09dd38db3bc1d24b58de0c7054 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 9 May 2023 07:07:39 +0100 Subject: [PATCH 060/101] Linux: Update comments and var names for maple tree depth warnings --- .../symbols/linux/extensions/__init__.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 931003b60..67a4cd2d0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -314,15 +314,15 @@ class maple_tree(objects.StructType): def get_slot_iter(self): """Parse the Maple Tree and return every non zero slot.""" maple_tree_offset = self.vol.offset & ~(self.MAPLE_NODE_POINTER_MASK) - maple_tree_depth = ( + expected_maple_tree_depth = ( self.ma_flags & self.MT_FLAGS_HEIGHT_MASK ) >> self.MT_FLAGS_HEIGHT_OFFSET yield from self._parse_maple_tree_node( - self.ma_root, maple_tree_offset, maple_tree_depth + self.ma_root, maple_tree_offset, expected_maple_tree_depth ) def _parse_maple_tree_node( - self, maple_tree_entry, parent, maple_tree_depth, seen=set(), depth=1 + self, maple_tree_entry, parent, expected_maple_tree_depth, seen=set(), current_depth=1 ): """Recursively parse Maple Tree Nodes and yield all non empty slots""" @@ -334,11 +334,16 @@ class maple_tree(objects.StructType): return else: seen.add(maple_tree_entry) - if maple_tree_depth < depth: + + # check if we have exceeded the expected depth of this maple tree. + # e.g. when current_depth is larger than expected_maple_tree_depth there may be an issue. + # it is normal that expected_maple_tree_depth is equal to current_depth. + if expected_maple_tree_depth < current_depth: vollog.warning( - f"The depth for the maple tree at {hex(self.vol.offset)} is {maple_tree_depth}, however when parsing the nodes " - f"a depth of {depth} was reached. This is unexpected and may lead to incorrect results." + f"The depth for the maple tree at {hex(self.vol.offset)} is {expected_maple_tree_depth}, however when parsing the nodes " + f"a depth of {current_depth} was reached. This is unexpected and may lead to incorrect results." ) + # parse the mte to extract the pointer value, node type, and leaf status pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) node_type = ( @@ -379,13 +384,13 @@ class maple_tree(objects.StructType): for slot in node.mr64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( - slot, pointer, maple_tree_depth, seen, depth + 1 + slot, pointer, expected_maple_tree_depth, seen, current_depth + 1 ) elif node_type == self.MAPLE_ARANGE_64: for slot in node.ma64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( - slot, pointer, maple_tree_depth, seen, depth + 1 + slot, pointer, expected_maple_tree_depth, seen, current_depth + 1 ) else: # unkown maple node type From d367b973a5cc305aad8f3a5f0b92e4b525122d11 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 9 May 2023 09:05:37 +0200 Subject: [PATCH 061/101] Renamed get_vmlinux_from_volobj() to get_module_from_volobj_type() --- volatility3/framework/symbols/linux/__init__.py | 4 ++-- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3f51d3c0c..dbdf1b777 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -152,7 +152,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): """ dentry = filp.get_dentry() - kernel_module = cls.get_vmlinux_from_volobj(context, dentry) + kernel_module = cls.get_module_from_volobj_type(context, dentry) sym_addr = dentry.d_op.d_dname symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) @@ -380,7 +380,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ) @classmethod - def get_vmlinux_from_volobj( + def get_module_from_volobj_type( cls, context: interfaces.context.ContextInterface, volobj: interfaces.objects.ObjectInterface, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1caab3ce5..060da4d7d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -890,7 +890,7 @@ class vfsmount(objects.StructType): Returns: mount: the struct 'mount' containing this 'vfsmount'. """ - vmlinux = linux.LinuxUtilities.get_vmlinux_from_volobj(self._context, self) + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) return linux.LinuxUtilities.container_of( self.vol.offset, "mount", "mnt", vmlinux ) From 6c5db21ae76519ffff538eba822dccfb2c63f4c2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 9 May 2023 09:22:02 +0200 Subject: [PATCH 062/101] Undo mnt_ns_id cast to str --- volatility3/framework/plugins/linux/mountinfo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 0606884ff..87ba4f9c1 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -160,7 +160,7 @@ class MountInfo(plugins.PluginInterface): mnt_namespace = task.nsproxy.mnt_ns try: - mnt_ns_id = str(mnt_namespace.get_inode()) + mnt_ns_id = mnt_namespace.get_inode() except AttributeError: mnt_ns_id = renderers.NotAvailableValue() @@ -256,7 +256,7 @@ class MountInfo(plugins.PluginInterface): self.context, self.config["kernel"], filter_func=pid_filter ) - columns = [("MNT_NS_ID", str)] + columns = [("MNT_NS_ID", int)] # The PID column does not make sense when a PID filter is not specified. In that case, the default behavior is # to displays the mountpoints per namespace. if pids: From 89ed65cc56bdce8b11c3480f5dcc93d0e1021961 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 9 May 2023 09:59:21 +0200 Subject: [PATCH 063/101] Improve filter warning implementation --- .../framework/plugins/linux/mountinfo.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 87ba4f9c1..e4081dc83 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -184,19 +184,12 @@ class MountInfo(plugins.PluginInterface): mount_format: bool, filtered_by_pids: bool, ) -> Iterable[Tuple[int, Tuple]]: - warning_shown = False + show_filter_warning = False for task, mnt, mnt_ns_id in self._get_tasks_mountpoints( tasks, filtered_by_pids ): - if ( - not warning_shown - and mnt_ns_ids - and isinstance(mnt_ns_id, renderers.NotAvailableValue) - ): - vollog.warning( - "Cannot filter by namespace id, it is not available in this kernel." - ) - warning_shown = True + if mnt_ns_ids and isinstance(mnt_ns_id, renderers.NotAvailableValue): + show_filter_warning = True if ( not isinstance(mnt_ns_id, renderers.NotAvailableValue) @@ -246,6 +239,11 @@ class MountInfo(plugins.PluginInterface): yield (0, fields_values) + if show_filter_warning: + vollog.warning( + "Could not filter by mount namespace id. This field is not available in this kernel." + ) + def run(self): pids = self.config.get("pids") mount_ns_ids = self.config.get("mntns") From 186b1ed1c2087a5e28edbf972f603c8c03ca26bf Mon Sep 17 00:00:00 2001 From: cpuu Date: Wed, 10 May 2023 09:49:11 +0900 Subject: [PATCH 064/101] Add more process information to mac.pslist plugin Enhance the PsList plugin by including additional process information such as offset, UID, GID, and start time in the output. Update the TreeGrid columns to display the new information. --- volatility3/framework/plugins/mac/pslist.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 1d97216bf..f88715c9c 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime import logging from typing import Callable, Iterable, List, Dict @@ -9,6 +10,7 @@ from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.symbols import mac +from volatility3.framework.renderers import format_hints vollog = logging.getLogger(__name__) @@ -105,10 +107,19 @@ class PsList(interfaces.plugins.PluginInterface): self.config["kernel"], filter_func=self.create_pid_filter(self.config.get("pid", None)), ): - pid = task.p_pid - ppid = task.p_ppid + offset = format_hints.Hex(task.vol.offset) name = utility.array_to_string(task.p_comm) - yield (0, (pid, ppid, name)) + pid = task.p_pid + uid = task.p_uid + gid = task.p_gid + start_time_seconds = task.p_start.tv_sec + start_time_microseconds = task.p_start.tv_usec + start_time = datetime.datetime.fromtimestamp(start_time_seconds + start_time_microseconds / 1e6) + + + ppid = task.p_ppid + + yield (0, (offset, name, pid, uid, gid, start_time, ppid)) @classmethod def list_tasks_allproc( @@ -310,5 +321,5 @@ class PsList(interfaces.plugins.PluginInterface): def run(self): return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str)], self._generator() + [("OFFSET", format_hints.Hex), ("NAME", str), ("PID", int), ("UID", int), ("GID", int), ("Start Time", datetime.datetime), ("PPID", int)], self._generator() ) From 396daa528c1028e559ac626571f8894e35450f3e Mon Sep 17 00:00:00 2001 From: cpuu Date: Wed, 10 May 2023 10:07:09 +0900 Subject: [PATCH 065/101] Apply Black Python linter to mac.pslist plugin Applied the Black Python linter to the mac PsList plugin, resulting in more readable and consistent code formatting. --- volatility3/framework/plugins/mac/pslist.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index f88715c9c..dbed09818 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -114,11 +114,12 @@ class PsList(interfaces.plugins.PluginInterface): gid = task.p_gid start_time_seconds = task.p_start.tv_sec start_time_microseconds = task.p_start.tv_usec - start_time = datetime.datetime.fromtimestamp(start_time_seconds + start_time_microseconds / 1e6) - + start_time = datetime.datetime.fromtimestamp( + start_time_seconds + start_time_microseconds / 1e6 + ) ppid = task.p_ppid - + yield (0, (offset, name, pid, uid, gid, start_time, ppid)) @classmethod @@ -321,5 +322,14 @@ class PsList(interfaces.plugins.PluginInterface): def run(self): return renderers.TreeGrid( - [("OFFSET", format_hints.Hex), ("NAME", str), ("PID", int), ("UID", int), ("GID", int), ("Start Time", datetime.datetime), ("PPID", int)], self._generator() + [ + ("OFFSET", format_hints.Hex), + ("NAME", str), + ("PID", int), + ("UID", int), + ("GID", int), + ("Start Time", datetime.datetime), + ("PPID", int), + ], + self._generator(), ) From 7f9afccf6f44fb69cd1039ab478854317bad5d5b Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 10 May 2023 08:49:46 +0100 Subject: [PATCH 066/101] Linux: apply black formating to maple tree parsing --- .../symbols/linux/extensions/__init__.py | 46 +++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 67a4cd2d0..7398a9ecd 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -294,6 +294,7 @@ class fs_struct(objects.StructType): raise AttributeError("Unable to find the root mount") + class maple_tree(objects.StructType): # include/linux/maple_tree.h # Mask for Maple Tree Flags @@ -322,7 +323,12 @@ class maple_tree(objects.StructType): ) def _parse_maple_tree_node( - self, maple_tree_entry, parent, expected_maple_tree_depth, seen=set(), current_depth=1 + self, + maple_tree_entry, + parent, + expected_maple_tree_depth, + seen=set(), + current_depth=1, ): """Recursively parse Maple Tree Nodes and yield all non empty slots""" @@ -384,13 +390,21 @@ class maple_tree(objects.StructType): for slot in node.mr64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( - slot, pointer, expected_maple_tree_depth, seen, current_depth + 1 + slot, + pointer, + expected_maple_tree_depth, + seen, + current_depth + 1, ) elif node_type == self.MAPLE_ARANGE_64: for slot in node.ma64.slot: if (slot & ~(self.MAPLE_NODE_TYPE_MASK)) != 0: yield from self._parse_maple_tree_node( - slot, pointer, expected_maple_tree_depth, seen, current_depth + 1 + slot, + pointer, + expected_maple_tree_depth, + seen, + current_depth + 1, ) else: # unkown maple node type @@ -398,13 +412,16 @@ class maple_tree(objects.StructType): f"Unkown Maple Tree node type {node_type} at offset {hex(pointer)}." ) + class mm_struct(objects.StructType): def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct.""" - if not self.has_member('mmap'): - raise AttributeError("get_mmap_iter called on mm_struct where no mmap member exists.") - + if not self.has_member("mmap"): + raise AttributeError( + "get_mmap_iter called on mm_struct where no mmap member exists." + ) + if not self.mmap: return @@ -420,9 +437,11 @@ class mm_struct(objects.StructType): def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mm_mt member of an mm_struct.""" - - if not self.has_member('mm_mt'): - raise AttributeError("get_maple_tree_iter called on mm_struct where no mm_mt member exists.") + + if not self.has_member("mm_mt"): + raise AttributeError( + "get_maple_tree_iter called on mm_struct where no mm_mt member exists." + ) symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): @@ -430,20 +449,21 @@ class mm_struct(objects.StructType): vma = self._context.object( symbol_table_name + constants.BANG + "vm_area_struct", layer_name=self.vol.native_layer_name, - offset=vma_pointer + offset=vma_pointer, ) yield vma def get_vma_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" - if self.has_member('mmap'): + if self.has_member("mmap"): yield from self.get_mmap_iter() - elif self.has_member('mm_mt'): + elif self.has_member("mm_mt"): yield from self.get_maple_tree_iter() else: raise AttributeError("Unable to find mmap or mm_mt in mm_struct") - + + class super_block(objects.StructType): # include/linux/kdev_t.h MINORBITS = 20 From 834372a6f89dc4c49c0a2fb563c1dd1cea1392ae Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 10 May 2023 09:12:03 +0100 Subject: [PATCH 067/101] Linux: apply black v23.3.0 formating to volatility3/framework/symbols/linux/extensions/__init__.py --- .../symbols/linux/extensions/__init__.py | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7398a9ecd..eecb200f9 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -34,10 +34,8 @@ class module(generic.GenericIntelProcess): def get_init_size(self): if self.has_member("init_layout"): return self.init_layout.size - elif self.has_member("init_size"): return self.init_size - raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module" ) @@ -45,10 +43,8 @@ class module(generic.GenericIntelProcess): def get_core_size(self): if self.has_member("core_layout"): return self.core_layout.size - elif self.has_member("core_size"): return self.core_size - raise AttributeError( "module -> get_core_size: Unable to determine core size of module" ) @@ -58,7 +54,6 @@ class module(generic.GenericIntelProcess): return self.core_layout.base elif self.has_member("module_core"): return self.module_core - raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): @@ -66,7 +61,6 @@ class module(generic.GenericIntelProcess): return self.init_layout.base elif self.has_member("module_init"): return self.module_init - raise AttributeError("module -> get_module_core: Unable to get module init") def get_name(self): @@ -88,7 +82,6 @@ class module(generic.GenericIntelProcess): idx = 0 while arr[idx]: idx = idx + 1 - return idx def get_sections(self): @@ -97,7 +90,6 @@ class module(generic.GenericIntelProcess): num_sects = self.sect_attrs.nsections else: num_sects = self._get_sect_count(self.sect_attrs.grp) - arr = self._context.object( self.get_symbol_table().name + constants.BANG + "array", layer_name=self.vol.layer_name, @@ -116,7 +108,6 @@ class module(generic.GenericIntelProcess): prefix = "Elf64_" else: prefix = "Elf32_" - elf_table_name = intermed.IntermediateSymbolTable.create( self.context, self.config_path, @@ -155,7 +146,6 @@ class module(generic.GenericIntelProcess): return self.kallsyms.symtab elif self.has_member("symtab"): return self.symtab - raise AttributeError("module -> symtab: Unable to get symtab") @property @@ -164,7 +154,6 @@ class module(generic.GenericIntelProcess): return int(self.kallsyms.num_symtab) elif self.has_member("num_symtab"): return int(self.num_symtab) - raise AttributeError( "module -> num_symtab: Unable to determine number of symbols" ) @@ -177,7 +166,6 @@ class module(generic.GenericIntelProcess): # Older kernels elif self.has_member("strtab"): return self.strtab - raise AttributeError("module -> strtab: Unable to get strtab") @@ -195,19 +183,15 @@ class task_struct(generic.GenericIntelProcess): pgd = self.mm.pgd except exceptions.InvalidAddressException: return None - if not isinstance(parent_layer, linear.LinearlyMappedLayer): raise TypeError( "Parent layer is not a translation layer, unable to construct process layer" ) - dtb, layer_name = parent_layer.translate(pgd) if not dtb: return None - if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.pid}" - # Add the constructed layer and return the name return self._add_process_layer( self._context, dtb, config_prefix, preferred_name @@ -229,7 +213,6 @@ class task_struct(generic.GenericIntelProcess): vollog.info( f"adding vma: {start:x} {self.mm.brk:x} | {end:x} {self.mm.start_brk:x}" ) - yield (start, end - start) @property @@ -282,7 +265,6 @@ class fs_struct(objects.StructType): return self.root elif self.root.has_member("dentry"): return self.root.dentry - raise AttributeError("Unable to find the root dentry") def get_root_mnt(self): @@ -291,7 +273,6 @@ class fs_struct(objects.StructType): return self.rootmnt elif self.root.has_member("mnt"): return self.root.mnt - raise AttributeError("Unable to find the root mount") @@ -340,7 +321,6 @@ class maple_tree(objects.StructType): return else: seen.add(maple_tree_entry) - # check if we have exceeded the expected depth of this maple tree. # e.g. when current_depth is larger than expected_maple_tree_depth there may be an issue. # it is normal that expected_maple_tree_depth is equal to current_depth. @@ -349,7 +329,6 @@ class maple_tree(objects.StructType): f"The depth for the maple tree at {hex(self.vol.offset)} is {expected_maple_tree_depth}, however when parsing the nodes " f"a depth of {current_depth} was reached. This is unexpected and may lead to incorrect results." ) - # parse the mte to extract the pointer value, node type, and leaf status pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) node_type = ( @@ -421,10 +400,8 @@ class mm_struct(objects.StructType): raise AttributeError( "get_mmap_iter called on mm_struct where no mmap member exists." ) - if not self.mmap: return - yield self.mmap seen = {self.mmap.vol.offset} @@ -442,7 +419,6 @@ class mm_struct(objects.StructType): raise AttributeError( "get_maple_tree_iter called on mm_struct where no mm_mt member exists." ) - symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): # convert pointer to vm_area_struct and yield @@ -569,7 +545,6 @@ class vm_area_struct(objects.StructType): retval = retval + char else: retval = retval + "-" - return retval # only parse the rwx bits @@ -583,7 +558,6 @@ class vm_area_struct(objects.StructType): def get_page_offset(self) -> int: if self.vm_file == 0: return 0 - return self.vm_pgoff << constants.linux.PAGE_SHIFT def get_name(self, context, task): @@ -600,7 +574,6 @@ class vm_area_struct(objects.StructType): fname = "[vdso]" else: fname = "Anonymous Mapping" - return fname # used by malfind @@ -611,10 +584,8 @@ class vm_area_struct(objects.StructType): if flags_str == "rwx": ret = True - elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True - return ret @@ -624,12 +595,10 @@ class qstr(objects.StructType): str_length = self.len + 1 # Maximum length should include null terminator else: str_length = 255 - try: ret = objects.utility.pointer_to_string(self.name, str_length) except (exceptions.InvalidAddressException, ValueError): ret = "" - return ret @@ -660,7 +629,6 @@ class dentry(objects.StructType): """ if self.vol.offset == old_dentry: return True - return self.d_ancestor(old_dentry) def d_ancestor(self, ancestor_dentry): @@ -678,10 +646,8 @@ class dentry(objects.StructType): ): if current_dentry.d_parent == ancestor_dentry.vol.offset: return current_dentry - dentry_seen.add(current_dentry.vol.offset) current_dentry = current_dentry.d_parent - return None @@ -738,12 +704,10 @@ class list_head(objects.StructType, collections.abc.Iterable): link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: return - if not sentinel: yield self._context.object( symbol_type, layer, offset=self.vol.offset - relative_offset ) - seen = {self.vol.offset} while link.vol.offset not in seen: obj = self._context.object( @@ -869,7 +833,6 @@ class mount(objects.StructType): peer = current_mnt.get_peer_under_root(self.mnt_ns, root) if peer and peer.vol.offset != 0: return peer.mnt_group_id - mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.mnt_master return 0 @@ -885,12 +848,10 @@ class mount(objects.StructType): current_mnt.mnt.mnt_root, root ): return current_mnt - mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.next_peer() if current_mnt.vol.offset == self.vol.offset: break - return None def is_path_reachable(self, current_dentry, root): @@ -907,7 +868,6 @@ class mount(objects.StructType): current_dentry = current_mnt.mnt_mountpoint mnt_seen.add(current_mnt.vol.offset) current_mnt = current_mnt.mnt_parent - return current_mnt.mnt.vol.offset == root.mnt and current_dentry.is_subdir( root.dentry ) @@ -968,7 +928,6 @@ class kobject(objects.StructType): ret = refcnt.counter else: ret = refcnt.refs.counter - return ret @@ -987,7 +946,6 @@ class mnt_namespace(objects.StructType): if not self._context.symbol_space.has_type(mnt_type): # Old kernels ~ 2.6 mnt_type = table_name + constants.BANG + "vfsmount" - for mount in self.list.to_list(mnt_type, "mnt_list"): yield mount @@ -1012,7 +970,6 @@ class socket(objects.StructType): ) if not module_names: raise ValueError(f"No module using the symbol table {symbol_table}") - kernel_module_name = module_names[0] kernel = self._context.modules[kernel_module_name] return kernel @@ -1022,7 +979,6 @@ class socket(objects.StructType): kernel = self._get_vol_kernel() except ValueError: return 0 - socket_alloc = linux.LinuxUtilities.container_of( self.vol.offset, "socket_alloc", "socket", kernel ) @@ -1048,7 +1004,6 @@ class sock(objects.StructType): def get_inode(self): if not self.sk_socket: return 0 - return self.sk_socket.get_inode() def get_protocol(self): @@ -1058,7 +1013,6 @@ class sock(objects.StructType): # Return the generic socket state if self.has_member("sk"): return self.sk.sk_socket.get_state() - return self.sk_socket.get_state() @@ -1066,7 +1020,6 @@ class unix_sock(objects.StructType): def get_name(self): if not self.addr: return - sockaddr_un = self.addr.name.cast("sockaddr_un") saddr = str(utility.array_to_string(sockaddr_un.sun_path)) return saddr @@ -1102,7 +1055,6 @@ class inet_sock(objects.StructType): protocol = IP_PROTOCOLS.get(self.sk.sk_protocol) if self.get_family() == "AF_INET6": protocol = IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) - return protocol def get_state(self): @@ -1133,7 +1085,6 @@ class inet_sock(objects.StructType): dport_le = sk_common.skc_dport else: return - return socket_module.htons(dport_le) def get_src_addr(self): @@ -1152,7 +1103,6 @@ class inet_sock(objects.StructType): saddr = self.pinet6.saddr else: return - parent_layer = self._context.layers[self.vol.layer_name] try: addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) @@ -1161,7 +1111,6 @@ class inet_sock(objects.StructType): f"Unable to read socket src address from {saddr.vol.offset:#x}" ) return - return socket_module.inet_ntop(family, addr_bytes) def get_dst_addr(self): @@ -1183,7 +1132,6 @@ class inet_sock(objects.StructType): addr_size = 16 else: return - parent_layer = self._context.layers[self.vol.layer_name] try: addr_bytes = parent_layer.read(daddr.vol.offset, addr_size) @@ -1192,7 +1140,6 @@ class inet_sock(objects.StructType): f"Unable to read socket dst address from {daddr.vol.offset:#x}" ) return - return socket_module.inet_ntop(family, addr_bytes) From b3c348820f341ea43a5bb215a5841349f46fb16c Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 10 May 2023 09:18:05 +0100 Subject: [PATCH 068/101] Linux: apply black v23.3.0 formating to volatility3/framework/symbols/linux/__init__.py --- volatility3/framework/symbols/linux/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 7a22241a5..c012f9cc7 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -53,6 +53,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): # Only found in 6.1+ kernels self.optional_set_type_class("maple_tree", extensions.maple_tree) + class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" From 7516346b649d7678c2996d1f4e02c48637fb050f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 10 May 2023 10:55:19 +0200 Subject: [PATCH 069/101] Adjust framework versioning --- volatility3/framework/plugins/linux/mountinfo.py | 5 ++++- volatility3/framework/symbols/linux/__init__.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index e4081dc83..da743bb60 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -50,6 +50,9 @@ class MountInfo(plugins.PluginInterface): requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) + ), requirements.ListRequirement( name="pids", description="Filter on specific process IDs.", @@ -86,7 +89,7 @@ class MountInfo(plugins.PluginInterface): if not mnt_root: return None - path_root = linux.LinuxUtilities._get_path_mnt(task, mnt) + path_root = linux.LinuxUtilities.get_path_mnt(task, mnt) if not path_root: return None diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index dbdf1b777..9ae8479b6 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -55,7 +55,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 0, 0) + _version = (2, 1, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -79,7 +79,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt) @classmethod - def _get_path_mnt(cls, task, mnt) -> str: + def get_path_mnt(cls, task, mnt) -> str: """Returns the mount point pathname relative to the task's root directory. Args: From e28b42731e663baf09051978a669d47e72506aaf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 10 May 2023 21:55:16 +0100 Subject: [PATCH 070/101] Core: Change readthedocs python version Attempting to fix #953 --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 4d21d9b40..3f10db145 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -14,6 +14,6 @@ formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: 3.7 + version: 3.11 install: - requirements: doc/requirements.txt From 3f1797f7e5b47a9bfd03f521604cd014012a0c69 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 10 May 2023 21:57:50 +0100 Subject: [PATCH 071/101] Core: Change readthedocs build platform Fixes #953 --- .readthedocs.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 3f10db145..e7c2b25d5 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,8 +12,12 @@ sphinx: # Optionally build your docs in additional formats such as PDF and ePub formats: all +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # Optionally set the version of Python and requirements required to build your docs python: - version: 3.11 install: - requirements: doc/requirements.txt From 8e56cb39d13c22575ba5133ffbc6b83344bed0ba Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 10 May 2023 22:07:38 +0100 Subject: [PATCH 072/101] Docs: Fix the documentation dependencies --- doc/requirements.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 93d6ea70a..b715e59f5 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,8 @@ # These packages are required for building the documentation. -sphinx>=4.0.0 +sphinx>=4.0.0,<7 sphinx_autodoc_typehints>=1.4.0 sphinx-rtd-theme>=0.4.3 + +yara-python +pycryptodome +pefile From 3d9efc3b3466a91dcbaf5aacfbc302b816bba316 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 17 May 2023 14:14:33 +0100 Subject: [PATCH 073/101] Added linux capabilities plugin --- .../framework/constants/linux/__init__.py | 45 ++++ .../framework/plugins/linux/capabilities.py | 218 ++++++++++++++++++ .../framework/symbols/linux/__init__.py | 2 + .../symbols/linux/extensions/__init__.py | 108 ++++++++- 4 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 volatility3/framework/plugins/linux/capabilities.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 1b133eb42..a802e0ada 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -234,3 +234,48 @@ BLUETOOTH_PROTOCOLS = ( "HIDP", "AVDTP", ) + +# Ref: include/uapi/linux/capability.h +CAPABILITIES = ( + "chown", + "dac_override", + "dac_read_search", + "fowner", + "fsetid", + "kill", + "setgid", + "setuid", + "setpcap", + "linux_immutable", + "net_bind_service", + "net_broadcast", + "net_admin", + "net_raw", + "ipc_lock", + "ipc_owner", + "sys_module", + "sys_rawio", + "sys_chroot", + "sys_ptrace", + "sys_pacct", + "sys_admin", + "sys_boot", + "sys_nice", + "sys_resource", + "sys_time", + "sys_tty_config", + "mknod", + "lease", + "audit_write", + "audit_control", + "setfcap", + "mac_override", + "mac_admin", + "syslog", + "wake_alarm", + "block_suspend", + "audit_read", + "perfmon", + "bpf", + "checkpoint_restore", +) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py new file mode 100644 index 000000000..8bcd80eef --- /dev/null +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -0,0 +1,218 @@ +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import Iterable, List, Tuple, Dict + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols.linux import extensions +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class Capabilities(plugins.PluginInterface): + """Lists process capabilities""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pids", + description="Filter on specific process IDs.", + element_type=int, + optional=True, + ), + requirements.BooleanRequirement( + name="inheritable", + description="Show only inheritable capabilities in human-readable strings.", + optional=True, + ), + requirements.BooleanRequirement( + name="permitted", + description="Show only permitted capabilities in human-readable strings.", + optional=True, + ), + requirements.BooleanRequirement( + name="effective", + description="Show only effective capabilities in human-readable strings.", + optional=True, + ), + requirements.BooleanRequirement( + name="bounding", + description="Show only bounding capabilities in human-readable strings.", + optional=True, + ), + requirements.BooleanRequirement( + name="ambient", + description="Show only ambient capabilities in human-readable strings.", + optional=True, + ), + ] + + def _check_capabilities_support(self): + """Checks that the framework supports at least as much capabilities as + the kernel being analysed. Otherwise, it shows a warning for the + developers. + """ + vmlinux = self.context.modules[self.config["kernel"]] + + kernel_cap_last_cap = vmlinux.object(object_type="int", offset=kernel_cap_last_cap) + vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() + if kernel_cap_last_cap > vol2_last_cap: + vollog.warning("Developers: The supported Linux capabilities of this plugin are outdated for this kernel") + + @staticmethod + def _decode_cap(cap: interfaces.objects.ObjectInterface) -> str: + """Returns a textual representation of the capability set. + The format is a comma-separated list of capabilitites. In order to + summarize the output and if all the capabilities are enabled, instead of + the individual capabilities, the special name "all" will be shown. + + Args: + cap: Kernel capability object. Usually a 'kernel_cap_struct' struct + + Returns: + str: A string with a comma separated list of decoded capabilities + """ + if isinstance(cap, renderers.NotAvailableValue): + return cap + + cap_value = cap.get_capabilities() + if cap_value == 0: + return "-" + + CAP_FULL = 0xffffffff + if cap_value == CAP_FULL: + return "all" + + return ", ".join(cap.enumerate_capabilities()) + + @classmethod + def get_task_capabilities(cls, task: interfaces.objects.ObjectInterface) -> Dict: + """Returns a dict with the task basic information along with its capabilities + + Args: + task: A task object from where to get the fields. + + Returns: + dict: A dict with the task basic information along with its capabilities + """ + task_cred = task.real_cred + fields = { + "common": [ + utility.array_to_string(task.comm), + int(task.pid), + int(task.tgid), + int(task.parent.pid), + int(task.cred.euid), + ], + "capabilities": [ + task_cred.cap_inheritable, + task_cred.cap_permitted, + task_cred.cap_effective, + task_cred.cap_bset, + ] + } + + # Ambient capabilities were added in kernels 4.3.6 + if task_cred.has_member("cap_ambient"): + fields["capabilities"].append(task_cred.cap_ambient) + else: + fields["capabilities"].append(renderers.NotAvailableValue()) + + return fields + + def get_tasks_capabilities(self, tasks: List[interfaces.objects.ObjectInterface]) -> Iterable[Dict]: + """Yields a dict for each task containing the task's basic information along with its capabilities + + Args: + tasks: An iterable with the tasks to process. + + Yields: + Iterable[Dict]: A dict for each task containing the task's basic information along with its capabilities + """ + for task in tasks: + if task.is_kernel_thread: + continue + + yield self.get_task_capabilities(task) + + def _generator(self, tasks: Iterable[interfaces.objects.ObjectInterface]) -> Iterable[Tuple[int, Tuple]]: + for fields in self.get_tasks_capabilities(tasks): + selected_fields = fields["common"] + cap_inh, cap_prm, cap_eff, cap_bnd, cap_amb = fields["capabilities"] + + if self.config.get("inheritable"): + selected_fields.append(self._decode_cap(cap_inh)) + elif self.config.get("permitted"): + selected_fields.append(self._decode_cap(cap_prm)) + elif self.config.get("effective"): + selected_fields.append(self._decode_cap(cap_eff)) + elif self.config.get("bounding"): + selected_fields.append(self._decode_cap(cap_bnd)) + elif self.config.get("ambient"): + selected_fields.append(self._decode_cap(cap_amb)) + else: + # Raw values + selected_fields.append(format_hints.Hex(cap_inh.get_capabilities())) + selected_fields.append(format_hints.Hex(cap_prm.get_capabilities())) + selected_fields.append(format_hints.Hex(cap_eff.get_capabilities())) + selected_fields.append(format_hints.Hex(cap_bnd.get_capabilities())) + + # Ambient capabilities were added in kernels 4.3.6 + if isinstance(cap_amb, renderers.NotAvailableValue): + selected_fields.append(cap_amb) + else: + selected_fields.append(format_hints.Hex(cap_amb.get_capabilities())) + + yield 0, selected_fields + + def run(self): + pids = self.config.get("pids") + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks(self.context, self.config["kernel"], filter_func=pid_filter) + + columns = [ + ("Name", str), + ("Tid", int), + ("Pid", int), + ("PPid", int), + ("EUID", int), + ] + + if self.config.get("inheritable"): + columns.append(("cap_inheritable", str)) + elif self.config.get("permitted"): + columns.append(("cap_permitted", str)) + elif self.config.get("effective"): + columns.append(("cap_effective", str)) + elif self.config.get("bounding"): + columns.append(("cap_bounding", str)) + elif self.config.get("ambient"): + columns.append(("cap_ambient", str)) + else: + columns.append(("cap_inheritable", format_hints.Hex)) + columns.append(("cap_permitted", format_hints.Hex)) + columns.append(("cap_effective", format_hints.Hex)) + columns.append(("cap_bounding", format_hints.Hex)) + columns.append(("cap_ambient", format_hints.Hex)) + + return renderers.TreeGrid(columns, self._generator(tasks)) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 36314b2c6..0bab9dedf 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -28,6 +28,8 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("fs_struct", extensions.fs_struct) self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) + self.set_type_class("cred", extensions.cred) + self.set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9ac98b5da..8a8785bd0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -5,7 +5,7 @@ import collections.abc import logging import socket as socket_module -from typing import Generator, Iterable, Iterator, Optional, Tuple +from typing import Generator, Iterable, Iterator, Optional, Tuple, List from volatility3.framework import constants from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY @@ -1428,3 +1428,109 @@ class bpf_prog(objects.StructType): # kernel < 3.18.140 raise AttributeError("Unable to find the BPF type") + +class cred(objects.StructType): + # struct cred was added in kernels 2.6.29 + def _get_cred_int_value(self, member: str) -> int: + """Helper to obtain the right cred member value for the current kernel. + + Args: + member (str): The requested cred member name to obtain its value + + Raises: + AttributeError: When the requested cred member doesn't exist + AttributeError: When the cred implementation is not supported. + + Returns: + int: The cred member value + """ + if not self.has_member(member): + raise AttributeError(f"struct cred doesn't have a '{member}' member") + + cred_val = self.member(member) + if hasattr(cred_val, "val"): + # From kernels 3.5.7 on it is a 'kuid_t' type + value = cred_val.val + elif isinstance(cred_val, objects.Integer): + # From at least 2.6.30 and until 3.5.7 it was a 'uid_t' type which was an 'unsigned int' + value = cred_val + else: + raise AttributeError("Kernel struct cred is not supported") + + return int(value) + + @property + def euid(self): + """Returns the effective user ID + + Returns: + int: the effective user ID value + """ + return self._get_cred_int_value("euid") + + +class kernel_cap_struct(objects.StructType): + # struct kernel_cap_struct was added in kernels 2.5.0 + @classmethod + def get_last_cap_value(cls) -> int: + """Returns the latest capability ID supported by the framework. + + Returns: + int: The latest supported capability ID supported by the framework. + """ + return len(constants.CAPABILITIES) - 1 + + @classmethod + def capabilities_to_string(cls, capabilities_bitfield: int) -> List[str]: + """Translates a capability bitfield to a list of capability strings. + + Args: + capabilities_bitfield (int): The capability bitfield value. + + Returns: + List[str]: A list of capability strings. + """ + + capabilities = [] + for bit, name in enumerate(constants.CAPABILITIES): + if capabilities_bitfield & (1 << bit) != 0: + capabilities.append(name) + + return capabilities + + def get_capabilities(self) -> int: + """Returns the capability bitfield value + + Returns: + int: The capability bitfield value. + """ + # In kernels 2.6.25.20 the kernel_cap_struct::cap became and array + cap_value = self.cap[0] if isinstance(self.cap, objects.Array) else self.cap + return int(cap_value & 0xffffffff) + + def enumerate_capabilities(self) -> List[str]: + """Returns the list of capability strings. + + Returns: + List[str]: The list of capability strings. + """ + capabilities_value = self.get_capabilities() + return self.capabilities_to_string(capabilities_value) + + def has_capability(self, capability: str) -> bool: + """Checks if the given capability string is enabled. + + Args: + capability (str): A string representing the capability i.e. dac_read_search + + Raises: + AttributeError: If the fiven capability is unknown to the framework. + + Returns: + bool: "True" if the given capability is enabled. + """ + if capability not in constants.CAPABILITIES: + raise AttributeError(f"Unknown capability with name '{capability}'") + + cap_value = 1 << constants.CAPABILITIES.index(capability) + return cap_value & self.get_capabilities() != 0 From d5d318d3cfd43f4d33fdb9252b120b5ed8666a12 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 17 May 2023 15:26:24 +0100 Subject: [PATCH 074/101] Fix wrong constants module --- .../framework/symbols/linux/extensions/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 8a8785bd0..f626db434 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,6 +13,7 @@ from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES +from volatility3.framework.constants.linux import CAPABILITIES from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility @@ -1478,7 +1479,7 @@ class kernel_cap_struct(objects.StructType): Returns: int: The latest supported capability ID supported by the framework. """ - return len(constants.CAPABILITIES) - 1 + return len(CAPABILITIES) - 1 @classmethod def capabilities_to_string(cls, capabilities_bitfield: int) -> List[str]: @@ -1492,7 +1493,7 @@ class kernel_cap_struct(objects.StructType): """ capabilities = [] - for bit, name in enumerate(constants.CAPABILITIES): + for bit, name in enumerate(CAPABILITIES): if capabilities_bitfield & (1 << bit) != 0: capabilities.append(name) @@ -1529,8 +1530,8 @@ class kernel_cap_struct(objects.StructType): Returns: bool: "True" if the given capability is enabled. """ - if capability not in constants.CAPABILITIES: + if capability not in CAPABILITIES: raise AttributeError(f"Unknown capability with name '{capability}'") - cap_value = 1 << constants.CAPABILITIES.index(capability) + cap_value = 1 << CAPABILITIES.index(capability) return cap_value & self.get_capabilities() != 0 From 4f5ca6116c73c456a9e55c56a41882b5bcccf7ac Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 17 May 2023 16:07:56 +0100 Subject: [PATCH 075/101] Add black recommendations --- .../framework/plugins/linux/capabilities.py | 24 +++++++++++++------ .../symbols/linux/extensions/__init__.py | 3 ++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 8bcd80eef..ed08c4c05 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -74,10 +74,14 @@ class Capabilities(plugins.PluginInterface): """ vmlinux = self.context.modules[self.config["kernel"]] - kernel_cap_last_cap = vmlinux.object(object_type="int", offset=kernel_cap_last_cap) + kernel_cap_last_cap = vmlinux.object( + object_type="int", offset=kernel_cap_last_cap + ) vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() if kernel_cap_last_cap > vol2_last_cap: - vollog.warning("Developers: The supported Linux capabilities of this plugin are outdated for this kernel") + vollog.warning( + "Developers: The supported Linux capabilities of this plugin are outdated for this kernel" + ) @staticmethod def _decode_cap(cap: interfaces.objects.ObjectInterface) -> str: @@ -99,7 +103,7 @@ class Capabilities(plugins.PluginInterface): if cap_value == 0: return "-" - CAP_FULL = 0xffffffff + CAP_FULL = 0xFFFFFFFF if cap_value == CAP_FULL: return "all" @@ -129,7 +133,7 @@ class Capabilities(plugins.PluginInterface): task_cred.cap_permitted, task_cred.cap_effective, task_cred.cap_bset, - ] + ], } # Ambient capabilities were added in kernels 4.3.6 @@ -140,7 +144,9 @@ class Capabilities(plugins.PluginInterface): return fields - def get_tasks_capabilities(self, tasks: List[interfaces.objects.ObjectInterface]) -> Iterable[Dict]: + def get_tasks_capabilities( + self, tasks: List[interfaces.objects.ObjectInterface] + ) -> Iterable[Dict]: """Yields a dict for each task containing the task's basic information along with its capabilities Args: @@ -155,7 +161,9 @@ class Capabilities(plugins.PluginInterface): yield self.get_task_capabilities(task) - def _generator(self, tasks: Iterable[interfaces.objects.ObjectInterface]) -> Iterable[Tuple[int, Tuple]]: + def _generator( + self, tasks: Iterable[interfaces.objects.ObjectInterface] + ) -> Iterable[Tuple[int, Tuple]]: for fields in self.get_tasks_capabilities(tasks): selected_fields = fields["common"] cap_inh, cap_prm, cap_eff, cap_bnd, cap_amb = fields["capabilities"] @@ -188,7 +196,9 @@ class Capabilities(plugins.PluginInterface): def run(self): pids = self.config.get("pids") pid_filter = pslist.PsList.create_pid_filter(pids) - tasks = pslist.PsList.list_tasks(self.context, self.config["kernel"], filter_func=pid_filter) + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=pid_filter + ) columns = [ ("Name", str), diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f626db434..0051cdc2f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1430,6 +1430,7 @@ class bpf_prog(objects.StructType): # kernel < 3.18.140 raise AttributeError("Unable to find the BPF type") + class cred(objects.StructType): # struct cred was added in kernels 2.6.29 def _get_cred_int_value(self, member: str) -> int: @@ -1507,7 +1508,7 @@ class kernel_cap_struct(objects.StructType): """ # In kernels 2.6.25.20 the kernel_cap_struct::cap became and array cap_value = self.cap[0] if isinstance(self.cap, objects.Array) else self.cap - return int(cap_value & 0xffffffff) + return int(cap_value & 0xFFFFFFFF) def enumerate_capabilities(self) -> List[str]: """Returns the list of capability strings. From f0bd9787160c8bca29e52090dc484dea213004e7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 20 May 2023 12:17:09 +0200 Subject: [PATCH 076/101] Fix typo --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0051cdc2f..4b1c53683 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1526,7 +1526,7 @@ class kernel_cap_struct(objects.StructType): capability (str): A string representing the capability i.e. dac_read_search Raises: - AttributeError: If the fiven capability is unknown to the framework. + AttributeError: If the given capability is unknown to the framework. Returns: bool: "True" if the given capability is enabled. From bea981180d5044aa979d39e6f2e06e13909513a1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 22 May 2023 01:32:00 +0100 Subject: [PATCH 077/101] Prevent potential dentry pointer memory smear --- volatility3/framework/symbols/linux/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 36314b2c6..3ddffb49a 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -200,8 +200,11 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): Returns: str: A file (or sock pipe) pathname relative to the task's root directory. """ + + # Memory smear protection: Check that both the file and dentry pointers are valids. try: dentry = filp.get_dentry() + dentry.is_root() except exceptions.InvalidAddressException: return "" From 0f42f0eaf1bbc47b86a180289a5d393cbcafc6d7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 22 May 2023 01:39:14 +0100 Subject: [PATCH 078/101] fix typo --- volatility3/framework/symbols/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3ddffb49a..339f6417f 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -201,7 +201,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): str: A file (or sock pipe) pathname relative to the task's root directory. """ - # Memory smear protection: Check that both the file and dentry pointers are valids. + # Memory smear protection: Check that both the file and dentry pointers are valid. try: dentry = filp.get_dentry() dentry.is_root() From b9e5cfb393c35d005235996804be6c6853a34b73 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 22 May 2023 15:05:39 +0100 Subject: [PATCH 079/101] Core: Protect from clearing a non-existant cache Issue kindly raised by @garanews, thanks! 5:D --- volatility3/framework/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index c7b23a9c3..9c17846a8 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -224,4 +224,7 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: def clear_cache(complete=False): - os.unlink(os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)) + try: + os.unlink(os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)) + except FileNotFoundError: + vollog.log(constants.LOGLEVEL_VVVV, "Attempting to clear a non-existant cache") From aa04b8ca3d6ba9db3d7658436927a2430cc6371e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 16 Jun 2023 18:03:07 +0100 Subject: [PATCH 080/101] Windows: Fix VAD offset canonicalization #969 --- volatility3/framework/plugins/windows/vadinfo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 812affe86..abc6142fe 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -198,6 +198,7 @@ class VadInfo(interfaces.plugins.PluginInterface): def _generator(self, procs): kernel = self.context.modules[self.config["kernel"]] + kernel_layer = self.context.layers[kernel.layer_name] def passthrough(_: interfaces.objects.ObjectInterface) -> bool: return False @@ -229,7 +230,7 @@ class VadInfo(interfaces.plugins.PluginInterface): ( proc.UniqueProcessId, process_name, - format_hints.Hex(vad.vol.offset), + format_hints.Hex(kernel_layer.canonicalize(vad.vol.offset)), format_hints.Hex(vad.get_start()), format_hints.Hex(vad.get_end()), vad.get_tag(), From a9698e4e731841932a17153847b264f0ceeb70a5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 16:31:48 +0100 Subject: [PATCH 081/101] Documentation: minor fixes/updates --- doc/source/conf.py | 144 ++++++++++++------ doc/source/getting-started-mac-tutorial.rst | 8 +- .../framework/plugins/linux/sockstat.py | 5 +- .../symbols/linux/extensions/__init__.py | 32 ++-- 4 files changed, 121 insertions(+), 68 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 895219b25..8b467ec1d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -21,57 +21,72 @@ import sphinx.ext.apidoc def setup(app): - volatility_directory = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'volatility3')) + volatility_directory = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "volatility3") + ) source_dir = os.path.abspath(os.path.dirname(__file__)) - sphinx.ext.apidoc.main(argv = ['-e', '-M', '-f', '-T', '-o', source_dir, volatility_directory]) + sphinx.ext.apidoc.main( + argv=["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory] + ) # Go through the volatility3.framework.plugins files and change them to volatility3.plugins for dir, _, files in os.walk(os.path.dirname(__file__)): for filename in files: - if filename.startswith('volatility3.framework.plugins') and filename != 'volatility3.framework.plugins.rst': + if ( + filename.startswith("volatility3.framework.plugins") + and filename != "volatility3.framework.plugins.rst" + ): # Change all volatility3.framework.plugins to volatility3.plugins in the file # Rename the file - new_filename = filename.replace('volatility3.framework.plugins', 'volatility3.plugins') + new_filename = filename.replace( + "volatility3.framework.plugins", "volatility3.plugins" + ) replace_string = b"Submodules\n----------\n\n.. toctree::\n\n" submodules = replace_string # If file already exists, read out the subpackages entries from it add them to the new list if os.path.exists(os.path.join(dir, new_filename)): - with open(os.path.join(dir, new_filename), 'rb') as newfile: + with open(os.path.join(dir, new_filename), "rb") as newfile: data = newfile.read() index = data.find(replace_string) if index > -1: submodules = data[index:] - with open(os.path.join(dir, new_filename), 'wb') as newfile: + with open(os.path.join(dir, new_filename), "wb") as newfile: with open(os.path.join(dir, filename), "rb") as oldfile: line = oldfile.read() - correct_plugins = line.replace(b'volatility3.framework.plugins', b'volatility3.plugins') - correct_submodules = correct_plugins.replace(replace_string, submodules) + correct_plugins = line.replace( + b"volatility3.framework.plugins", b"volatility3.plugins" + ) + correct_submodules = correct_plugins.replace( + replace_string, submodules + ) newfile.write(correct_submodules) os.remove(os.path.join(dir, filename)) - elif filename == 'volatility3.framework.rst': + elif filename == "volatility3.framework.rst": with open(os.path.join(dir, filename), "rb") as contents: lines = contents.readlines() plugins_seen = False with open(os.path.join(dir, filename), "wb") as contents: for line in lines: - if b'volatility3.framework.plugins' in line: + if b"volatility3.framework.plugins" in line: plugins_seen = True - if plugins_seen and line == b'': - contents.write(b' volatility3.plugins') + if plugins_seen and line == b"": + contents.write(b" volatility3.plugins") contents.write(line) - elif filename == 'volatility3.plugins.rst': + elif filename == "volatility3.plugins.rst": with open(os.path.join(dir, filename), "rb") as contents: lines = contents.readlines() - with open(os.path.join(dir, 'volatility3.framework.plugins.rst'), "rb") as contents: + with open( + os.path.join(dir, "volatility3.framework.plugins.rst"), "rb" + ) as contents: real_lines = contents.readlines() # Process real_lines for line_index in range(len(real_lines)): - if b'Submodules' in real_lines[line_index]: + if b"Submodules" in real_lines[line_index]: break else: line_index = len(real_lines) @@ -82,36 +97,52 @@ def setup(app): for line in lines: contents.write(line) for line in submodule_lines: - contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins')) + contents.write( + line.replace( + b"volatility3.framework.plugins", b"volatility3.plugins" + ) + ) # Clear up the framework.plugins page - with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "rb") as contents: + with open( + os.path.join(os.path.dirname(__file__), "volatility3.framework.plugins.rst"), + "rb", + ) as contents: real_lines = contents.readlines() - with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "wb") as contents: + with open( + os.path.join(os.path.dirname(__file__), "volatility3.framework.plugins.rst"), + "wb", + ) as contents: for line in real_lines: - if b'volatility3.framework.plugins.' not in line: + if b"volatility3.framework.plugins." not in line: contents.write(line) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('../..')) +sys.path.insert(0, os.path.abspath("../..")) from volatility3.framework import constants # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '2.0' +needs_sphinx = "2.0" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.autosectionlabel' + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", + "sphinx.ext.autosectionlabel", ] autosectionlabel_prefix_document = True @@ -119,7 +150,7 @@ autosectionlabel_prefix_document = True try: import sphinx_autodoc_typehints - extensions.append('sphinx_autodoc_typehints') + extensions.append("sphinx_autodoc_typehints") except ImportError: # If the autodoc typehints extension isn't available, carry on regardless pass @@ -128,17 +159,17 @@ except ImportError: # templates_path = ['tools/templates'] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'Volatility 3' -copyright = '2012-2022, Volatility Foundation' +project = "Volatility 3" +copyright = "2012-2022, Volatility Foundation" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -147,7 +178,7 @@ copyright = '2012-2022, Volatility Foundation' # The full version, including alpha/beta/rc tags. release = constants.PACKAGE_VERSION # The short X.Y version. -version = ".".join(release.split('.')[0:2]) +version = ".".join(release.split(".")[0:2]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -180,7 +211,7 @@ add_module_names = False # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -196,8 +227,8 @@ pygments_style = 'sphinx' # html_theme = 'pydoctheme' # html_theme_options = {'collapsiblesidebar': True} # html_theme_path = ['tools'] -html_theme = 'sphinx_rtd_theme' -html_theme_options = {'logo_only': True} +html_theme = "sphinx_rtd_theme" +html_theme_options = {"logo_only": True} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -216,17 +247,17 @@ html_theme_options = {'logo_only': True} # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = '_static/vol.png' +html_logo = "_static/vol.png" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -html_favicon = '_static/favicon.ico' +html_favicon = "_static/favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied @@ -275,17 +306,15 @@ html_static_path = ['_static'] # html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'Volatilitydoc' +htmlhelp_basename = "Volatilitydoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # 'preamble': '', } @@ -294,7 +323,13 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'Volatility.tex', 'Volatility 3 Documentation', 'Volatility Foundation', 'manual'), + ( + "index", + "Volatility.tex", + "Volatility 3 Documentation", + "Volatility Foundation", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of @@ -321,7 +356,15 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [('vol-cli', 'volatility', 'Volatility 3 Documentation', ['Volatility Foundation'], 1)] +man_pages = [ + ( + "vol-cli", + "volatility", + "Volatility 3 Documentation", + ["Volatility Foundation"], + 1, + ) +] # If true, show URL addresses after external links. # man_show_urls = False @@ -332,8 +375,15 @@ man_pages = [('vol-cli', 'volatility', 'Volatility 3 Documentation', ['Volatilit # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Volatility', 'Volatility 3 Documentation', 'Volatility Foundation', 'Volatility', - 'Memory forensics framework.', 'Miscellaneous'), + ( + "index", + "Volatility", + "Volatility 3 Documentation", + "Volatility Foundation", + "Volatility", + "Memory forensics framework.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. @@ -349,10 +399,14 @@ texinfo_documents = [ # texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {"python": ("http://docs.python.org/", None)} # -- Autodoc options ------------------------------------------------------- # autodoc_member_order = 'groupwise' -autodoc_default_options = {'members': True, 'inherited-members': True, 'show-inheritance': True} -autoclass_content = 'both' +autodoc_default_options = { + "members": True, + "inherited-members": True, + "show-inheritance": True, +} +autoclass_content = "both" diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index cfb0afa9a..42e58c0d5 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -78,10 +78,10 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-macos-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-mac-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. mac.pslist -~~~~~~~~~~~~ +~~~~~~~~~~ .. code-block:: shell-session @@ -107,7 +107,7 @@ mac.pslist ``mac.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. mac.pstree -~~~~~~~~~~~~ +~~~~~~~~~~ .. code-block:: shell-session @@ -128,7 +128,7 @@ mac.pstree ``mac.pstree`` helps us to display the parent child relationships between processes. mac.ifconfig -~~~~~~~~~~ +~~~~~~~~~~~~ .. code-block:: shell-session diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index f06b3ad8e..fa67122ba 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -83,7 +83,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock: Kernel generic `sock` object Returns a tuple with: - sock: The respective kernel's *_sock object for that socket family + sock: The respective kernel's \*_sock object for that socket family sock_stat: A tuple with the source and destination (address and port) along with its state string socket_filter: A dictionary with information about the socket filter """ @@ -501,8 +501,7 @@ class Sockstat(plugins.PluginInterface): family: Socket family string (AF_UNIX, AF_INET, etc) sock_type: Socket type string (STREAM, DGRAM, etc) protocol: Protocol string (UDP, TCP, etc) - sock_fields: A tuple with the *_sock object, the sock stats and the - extended info dictionary + sock_fields: A tuple with the \*_sock object, the sock stats and the extended info dictionary """ vmlinux = context.modules[symbol_table] diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9ac98b5da..64c038f5a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -794,7 +794,7 @@ class mount(objects.StructType): """Gets the fs where we are mounted on Returns: - A 'mount *' + A mount pointer """ return self.mnt_parent @@ -802,7 +802,7 @@ class mount(objects.StructType): """Gets the dentry of the mountpoint Returns: - A 'dentry *' + A dentry pointer """ return self.mnt_mountpoint @@ -839,7 +839,7 @@ class mount(objects.StructType): """Returns the root of the mounted tree Returns: - A 'dentry *' + A dentry pointer """ vfsmnt = self.get_vfsmnt_current() dentry = vfsmnt.mnt_root @@ -850,7 +850,7 @@ class mount(objects.StructType): """Returns the parent root of the mounted tree Returns: - A 'dentry *' + A dentry pointer """ return self.get_mnt_parent().get_dentry_current() @@ -970,17 +970,17 @@ class vfsmount(objects.StructType): """Helper to make sure it is comparing two pointers to 'vfsmount'. Depending on the kernel version, the calling object (self) could be - a 'vfsmount *' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust + a 'vfsmount \*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust in the framework "auto" dereferencing ability to assure that when we reach this point 'self' will be a 'vfsmount' already and self.vol.offset - a 'vfsmount *' and not a 'vfsmount **'. The argument must be a 'vfsmount *'. + a 'vfsmount \*' and not a 'vfsmount \*\*'. The argument must be a 'vfsmount \*'. Typically, it's called from do_get_path(). Args: - vfsmount_ptr (vfsmount *): A pointer to a 'vfsmount' + vfsmount_ptr (vfsmount \*): A pointer to a 'vfsmount' Raises: - exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount *' + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \*' Returns: bool: 'True' if the given argument points to the the same 'vfsmount' @@ -1010,7 +1010,7 @@ class vfsmount(objects.StructType): """Returns the current fs where we are mounted on Returns: - A 'vfsmount *' + A vfsmount pointer """ return self.get_mnt_parent() @@ -1018,8 +1018,8 @@ class vfsmount(objects.StructType): """Gets the parent fs (vfsmount) to where it's mounted on Returns: - For kernels < 3.3.8: A 'vfsmount *' - For kernels >= 3.3.8: A 'vfsmount' + For kernels < 3.3.8: A vfsmount pointer + For kernels >= 3.3.8: A vfsmount object """ if self._is_kernel_prior_to_struct_mount(): return self.get_mnt_parent() @@ -1030,7 +1030,7 @@ class vfsmount(objects.StructType): """Returns the root of the mounted tree Returns: - A 'dentry *' + A dentry pointer """ if self._is_kernel_prior_to_struct_mount(): return self.get_mnt_mountpoint() @@ -1041,7 +1041,7 @@ class vfsmount(objects.StructType): """Returns the parent root of the mounted tree Returns: - A 'dentry *' + A dentry pointer """ if self._is_kernel_prior_to_struct_mount(): return self.get_mnt_mountpoint() @@ -1052,8 +1052,8 @@ class vfsmount(objects.StructType): """Gets the mnt_parent member. Returns: - For kernels < 3.3.8: A 'vfsmount *' - For kernels >= 3.3.8: A 'mount *' + For kernels < 3.3.8: A vfsmount pointer + For kernels >= 3.3.8: A mount pointer """ if self._is_kernel_prior_to_struct_mount(): return self.mnt_parent @@ -1064,7 +1064,7 @@ class vfsmount(objects.StructType): """Gets the dentry of the mountpoint Returns: - A 'dentry *' + A dentry pointer """ if self.has_member("mnt_mountpoint"): return self.mnt_mountpoint From 1ff2d80b44cc919474f502d37840c4bf5fe0e6f1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 17:01:57 +0100 Subject: [PATCH 082/101] Documentation: Update basics - memory layer information --- doc/source/basics.rst | 73 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/doc/source/basics.rst b/doc/source/basics.rst index d493c61b3..1b8e64780 100644 --- a/doc/source/basics.rst +++ b/doc/source/basics.rst @@ -1,7 +1,7 @@ Volatility 3 Basics =================== -Volatility splits memory analysis down to several components: +Volatility splits memory analysis down to several components. The main ones are: * Memory layers * Templates and Objects @@ -13,22 +13,65 @@ which acts as a container for all the various layers and tables necessary to con Memory layers ------------- -A memory layer is a body of data that can be accessed by requesting data at a specific address. Memory is seen as -sequential when accessed through sequential addresses, however, there is no obligation for the data to be stored -sequentially, and modern processors tend to store the memory in a paged format. Moreover, there is no need for the data -to be stored in an easily accessible format, it could be encoded or encrypted or more, it could be the combination of -two other sources. These are typically handled by programs that process file formats, or the memory manager of the -processor, but these are all translations (either in the geometric or linguistic sense) of the original data. +A memory layer is a body of data that can be accessed by requesting data at a specific address. At its lowest level +this data is stored on a phyiscal medium (RAM) and very early computers addresses locations in memory directly. However, +as the size of memory increased and it became more difficult to manage memory most architectures moved to a "paged" model +of memory, where the available memory is cut into specific fixed-sized pages. To help further, programs can ask for any address +and the processor will look up their (virtual) address in a map, to find out where the (physical) address that it lives at is, +in the actual memory of the system. -In Volatility 3 this is represented by a directed graph, whose end nodes are -:py:class:`DataLayers ` and whose internal nodes are -specifically called a :py:class:`TranslationLayer `. -In this way, a raw memory image in the LiME file format and a page file can be -combined to form a single Intel virtual memory layer. When requesting addresses from the Intel layer, it will use the -Intel memory mapping algorithm, along with the address of the directory table base or page table map, to translate that +Volatility can work with these layers as long as it knows the map (so, for example that virtual address `1` looks up at physical +address `9`). The automagic that runs at the start of every volatility session often locates the kernel's memory map, and creates +a kernel virtual layer, which allows for kernel addresses to be looked up and the correct data returned. There can, however, be +several maps, and in general there is a different map for each process (although a portion of the operating system's memory is +usually mapped to the same location across all processes). The maps may take the same address but point to a different part of +physical memory. It also means that two processes could theoretically share memory, but having an virtual address mapped to the +same physical address as another process. See the worked example below for more information. + +To translate an address on a layer, call :py:meth:`layer.mapping(offset, length, ignore_errors) ` and it will return a list of chunks without overlap, in order, +for the requested range. If a portion cannot be mapped, an exception will be thrown unless `ignore_errors` is true. Each +chunk will contain the original offset of the chunk, the translated offset, the original size and the translated size of +the chunk, as well as the lower layer the chunk lives within. + +Worked example +^^^^^^^^^^^^^^ + +The operating system and two programs may all appear to have access to all of physical memory, but actually the maps they each have +mean they each see something different: + +.. code-block:: + :caption: Memory mapping example + + Operating system map Physical Memory + 1 -> 9 1 - Free + 2 -> 3 2 - OS.4, Process 1.4, Process 2.4 + 3 -> 7 3 - OS.2 + 4 -> 2 4 - Free + 5 - Free + Process 1 map 6 - Process 1.2, Process 2.3 + 1 -> 12 7 - OS.3 + 2 -> 6 8 - Process1.3 + 3 -> 8 9 - OS.1 + 4 -> 2 10 - Process2.1 + 11 - Free + Process 2 map 12 - Process1.1 + 1 -> 10 13 - Free + 2 -> 15 14 - Free + 3 -> 6 15 - Process2.2 + 4 -> 2 16 - Free + +In this example, part of the operating system is visible across all processes (although not all processes can write to the memory, there +is a permissions model for intel addressing which is not discussed further here).) + +In Volatility 3 mappings are represented by a directed graph of layers, whose end nodes are +:py:class:`DataLayers ` and whose internal nodes are :py:class:`TranslationLayers `. +In this way, a raw memory image in the LiME file format and a page file can be combined to form a single Intel virtual +memory layer. When requesting addresses from the Intel layer, it will use the Intel memory mapping algorithm, along +with the address of the directory table base or page table map, to translate that address into a physical address, which will then either be directed towards the swap layer or the LiME layer. Should it -be directed towards the LiME layer, the LiME file format algorithm will be translated to determine where within the file -the data is stored and that will be returned. +be directed towards the LiME layer, the LiME file format algorithm will be translate the new address to determine where +within the file the data is stored. When the :py:meth:`layer.read() ` +method is called, the translation is done automatically and the correct data gathered and combined. .. note:: Volatility 2 had a similar concept, called address spaces, but these could only stack linearly one on top of another. From 50b5d2232b7e1519156292875fb9ebc625cde4ac Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 19:00:03 +0100 Subject: [PATCH 083/101] Core: Add type parameter to object_from_symbol --- API_CHANGES.md | 4 ++++ volatility3/framework/constants/__init__.py | 4 ++-- volatility3/framework/contexts/__init__.py | 15 +++++++++++---- volatility3/framework/interfaces/context.py | 2 ++ 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/API_CHANGES.md b/API_CHANGES.md index 98a08f09d..61d8781fb 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,10 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.5.0 +===== +Add in support for specifying a type override for object_from_symbol + 2.4.0 ===== Add a `get_size()` method to Windows VAD structures and fix several off-by-one issues when calculating VAD sizes. diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3a6b24ea8..de1674885 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 4 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_MINOR = 5 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index ecce5041c..81b516765 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -272,8 +272,9 @@ class Module(interfaces.context.ModuleInterface): symbol_name: str, native_layer_name: Optional[str] = None, absolute: bool = False, + object_type: Optional[Union[str, interfaces.objects.ObjectInterface]] = None, **kwargs, - ) -> "interfaces.objects.ObjectInterface": + ) -> interfaces.objects.ObjectInterface: """Returns an object based on a specific symbol (containing type and offset information) and the layer_name of the Module. This will throw a ValueError if the symbol does not contain an associated type, or if @@ -284,6 +285,7 @@ class Module(interfaces.context.ModuleInterface): symbol_name: Name of the symbol (within the module) to construct native_layer_name: Name of the layer in which constructed objects are made (for pointers) absolute: whether the symbol's address is absolute or relative to the module + object_type: Override for the type from the symobl to use (or if the symbol type is missing) """ if constants.BANG not in symbol_name: symbol_name = self.symbol_table_name + constants.BANG + symbol_name @@ -299,8 +301,13 @@ class Module(interfaces.context.ModuleInterface): if not absolute: offset += self._offset - if symbol_val.type is None: - raise TypeError(f"Symbol {symbol_val.name} has no associated type") + if object_type is None: + if symbol_val.type is None: + raise TypeError( + f"Symbol {symbol_val.name} has no associated type and no object_type specified" + ) + else: + object_type = symbol_val.type # Ensure we don't use a layer_name other than the module's, why would anyone do that? if "layer_name" in kwargs: @@ -308,7 +315,7 @@ class Module(interfaces.context.ModuleInterface): # Since type may be a template, we don't just call our own module method return self._context.object( - object_type=symbol_val.type, + object_type=object_type, layer_name=self._layer_name, offset=offset, native_layer_name=native_layer_name or self._native_layer_name, diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 7e385746d..03f2d9f1b 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -253,6 +253,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol_name: str, native_layer_name: Optional[str] = None, absolute: bool = False, + object_type: Optional[Union[str, interfaces.objects.ObjectInterface]] = None, **kwargs, ) -> "interfaces.objects.ObjectInterface": """Returns an object created using the symbol_table_name and layer_name @@ -262,6 +263,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol_name: The name of a symbol (that must be present in the module's symbol table). The symbol's associated type will be used to construct an object at the symbol's offset. native_layer_name: The native layer for objects that reference a different layer (if not the default provided during module construction) absolute: A boolean specifying whether the offset is absolute within the layer, or relative to the start of the module + object_type: Override for the type from the symobl to use (or if the symbol type is missing) Returns: The constructed object From c8e53ff16a0feab2f9d036fc3e173fe1969d8621 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 19:02:51 +0100 Subject: [PATCH 084/101] Core: Fix small typing issue in previous patch --- volatility3/framework/contexts/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 81b516765..73868a58f 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -272,9 +272,9 @@ class Module(interfaces.context.ModuleInterface): symbol_name: str, native_layer_name: Optional[str] = None, absolute: bool = False, - object_type: Optional[Union[str, interfaces.objects.ObjectInterface]] = None, + object_type: Optional[Union[str, "interfaces.objects.ObjectInterface"]] = None, **kwargs, - ) -> interfaces.objects.ObjectInterface: + ) -> "interfaces.objects.ObjectInterface": """Returns an object based on a specific symbol (containing type and offset information) and the layer_name of the Module. This will throw a ValueError if the symbol does not contain an associated type, or if From 66598f5b631a959d7cd73b8e929e95d122ca9a58 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 19:05:22 +0100 Subject: [PATCH 085/101] Core: Second fix is the charm... --- volatility3/framework/interfaces/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 03f2d9f1b..29cb41379 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -253,7 +253,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol_name: str, native_layer_name: Optional[str] = None, absolute: bool = False, - object_type: Optional[Union[str, interfaces.objects.ObjectInterface]] = None, + object_type: Optional[Union[str, "interfaces.objects.ObjectInterface"]] = None, **kwargs, ) -> "interfaces.objects.ObjectInterface": """Returns an object created using the symbol_table_name and layer_name From 6c0ce1d130092ac496c85eec294580a689c6ce2d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 1 Jul 2023 20:58:36 +0100 Subject: [PATCH 086/101] Volshell: Typo Fixes #958 --- volatility3/cli/volshell/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index df369c53e..ea9e65d9b 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -480,7 +480,7 @@ class Volshell(interfaces.plugins.PluginInterface): accessor = resources.ResourceAccessor() with accessor.open(url=location) as fp: self.__console.runsource( - io.TextIOWrapper(fp.read(), encoding="utf-8"), symbol="exec" + io.TextIOWrapper(fp, encoding="utf-8").read(), symbol="exec" ) print("\nCode complete") From 241cf832f9b3676fbcf4e724c646dbfe46c24eb2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jul 2023 00:07:40 +0200 Subject: [PATCH 087/101] Plugin parameters removed. Instead, it now shows each capability set in their textual representation. The dictionary was replaced by a dataclass. Last but not leas, CAP_FULL moved to constants. --- .../framework/constants/linux/__init__.py | 2 + .../framework/plugins/linux/capabilities.py | 179 ++++++++---------- .../symbols/linux/extensions/__init__.py | 4 +- 3 files changed, 83 insertions(+), 102 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index a802e0ada..e57fa30d4 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -279,3 +279,5 @@ CAPABILITIES = ( "bpf", "checkpoint_restore", ) + +CAP_FULL = 0xFFFFFFFF diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index ed08c4c05..9a59f31c1 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -3,19 +3,50 @@ # import logging +from dataclasses import dataclass, astuple, fields from typing import Iterable, List, Tuple, Dict -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions +from volatility3.framework.constants.linux import CAP_FULL from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility -from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.linux import extensions from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) +@dataclass +class TaskData: + """Stores basic information about a task""" + + comm: str + pid: int + tgid: int + ppid: int + euid: int + + +@dataclass +class CapabilitiesData: + """Stores each set of capabilties for a task""" + + cap_inheritable: interfaces.objects.ObjectInterface + cap_permitted: interfaces.objects.ObjectInterface + cap_effective: interfaces.objects.ObjectInterface + cap_bset: interfaces.objects.ObjectInterface + cap_ambient: interfaces.objects.ObjectInterface + + def astuple(self) -> Tuple: + """Returns a shallow copy of the capability sets in a tuple. + + Otherwise, when dataclasses.astuple() performs a deep-copy recursion on + ObjectInterface will take a substantial amount of time. + """ + return tuple(getattr(self, field.name) for field in fields(self)) + + class Capabilities(plugins.PluginInterface): """Lists process capabilities""" @@ -40,43 +71,26 @@ class Capabilities(plugins.PluginInterface): element_type=int, optional=True, ), - requirements.BooleanRequirement( - name="inheritable", - description="Show only inheritable capabilities in human-readable strings.", - optional=True, - ), - requirements.BooleanRequirement( - name="permitted", - description="Show only permitted capabilities in human-readable strings.", - optional=True, - ), - requirements.BooleanRequirement( - name="effective", - description="Show only effective capabilities in human-readable strings.", - optional=True, - ), - requirements.BooleanRequirement( - name="bounding", - description="Show only bounding capabilities in human-readable strings.", - optional=True, - ), - requirements.BooleanRequirement( - name="ambient", - description="Show only ambient capabilities in human-readable strings.", - optional=True, - ), ] - def _check_capabilities_support(self): + def _check_capabilities_support( + self, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ): """Checks that the framework supports at least as much capabilities as the kernel being analysed. Otherwise, it shows a warning for the developers. """ - vmlinux = self.context.modules[self.config["kernel"]] - kernel_cap_last_cap = vmlinux.object( - object_type="int", offset=kernel_cap_last_cap - ) + vmlinux = context.modules[vmlinux_module_name] + + try: + kernel_cap_last_cap = vmlinux.object_from_symbol(symbol_name="cap_last_cap") + except exceptions.SymbolError: + # It should be a kernel < 3.2 + return + vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() if kernel_cap_last_cap > vol2_last_cap: vollog.warning( @@ -103,7 +117,6 @@ class Capabilities(plugins.PluginInterface): if cap_value == 0: return "-" - CAP_FULL = 0xFFFFFFFF if cap_value == CAP_FULL: return "all" @@ -119,33 +132,32 @@ class Capabilities(plugins.PluginInterface): Returns: dict: A dict with the task basic information along with its capabilities """ + task_data = TaskData( + comm=utility.array_to_string(task.comm), + pid=int(task.pid), + tgid=int(task.tgid), + ppid=int(task.parent.pid), + euid=int(task.cred.euid), + ) + task_cred = task.real_cred - fields = { - "common": [ - utility.array_to_string(task.comm), - int(task.pid), - int(task.tgid), - int(task.parent.pid), - int(task.cred.euid), - ], - "capabilities": [ - task_cred.cap_inheritable, - task_cred.cap_permitted, - task_cred.cap_effective, - task_cred.cap_bset, - ], - } + capabilities_data = CapabilitiesData( + cap_inheritable=task_cred.cap_inheritable, + cap_permitted=task_cred.cap_permitted, + cap_effective=task_cred.cap_effective, + cap_bset=task_cred.cap_bset, + cap_ambient=renderers.NotAvailableValue(), + ) # Ambient capabilities were added in kernels 4.3.6 if task_cred.has_member("cap_ambient"): - fields["capabilities"].append(task_cred.cap_ambient) - else: - fields["capabilities"].append(renderers.NotAvailableValue()) + capabilities_data.cap_ambient = task_cred.cap_ambient - return fields + return task_data, capabilities_data + @classmethod def get_tasks_capabilities( - self, tasks: List[interfaces.objects.ObjectInterface] + cls, tasks: List[interfaces.objects.ObjectInterface] ) -> Iterable[Dict]: """Yields a dict for each task containing the task's basic information along with its capabilities @@ -156,44 +168,23 @@ class Capabilities(plugins.PluginInterface): Iterable[Dict]: A dict for each task containing the task's basic information along with its capabilities """ for task in tasks: - if task.is_kernel_thread: - continue - - yield self.get_task_capabilities(task) + yield cls.get_task_capabilities(task) def _generator( self, tasks: Iterable[interfaces.objects.ObjectInterface] ) -> Iterable[Tuple[int, Tuple]]: - for fields in self.get_tasks_capabilities(tasks): - selected_fields = fields["common"] - cap_inh, cap_prm, cap_eff, cap_bnd, cap_amb = fields["capabilities"] + for task_fields, capabilities_fields in self.get_tasks_capabilities(tasks): + task_fields = astuple(task_fields) - if self.config.get("inheritable"): - selected_fields.append(self._decode_cap(cap_inh)) - elif self.config.get("permitted"): - selected_fields.append(self._decode_cap(cap_prm)) - elif self.config.get("effective"): - selected_fields.append(self._decode_cap(cap_eff)) - elif self.config.get("bounding"): - selected_fields.append(self._decode_cap(cap_bnd)) - elif self.config.get("ambient"): - selected_fields.append(self._decode_cap(cap_amb)) - else: - # Raw values - selected_fields.append(format_hints.Hex(cap_inh.get_capabilities())) - selected_fields.append(format_hints.Hex(cap_prm.get_capabilities())) - selected_fields.append(format_hints.Hex(cap_eff.get_capabilities())) - selected_fields.append(format_hints.Hex(cap_bnd.get_capabilities())) + capabilities_text = tuple( + self._decode_cap(cap) for cap in capabilities_fields.astuple() + ) - # Ambient capabilities were added in kernels 4.3.6 - if isinstance(cap_amb, renderers.NotAvailableValue): - selected_fields.append(cap_amb) - else: - selected_fields.append(format_hints.Hex(cap_amb.get_capabilities())) - - yield 0, selected_fields + yield 0, task_fields + capabilities_text def run(self): + self._check_capabilities_support(self.context, self.config["kernel"]) + pids = self.config.get("pids") pid_filter = pslist.PsList.create_pid_filter(pids) tasks = pslist.PsList.list_tasks( @@ -206,23 +197,11 @@ class Capabilities(plugins.PluginInterface): ("Pid", int), ("PPid", int), ("EUID", int), + ("cap_inheritable", str), + ("cap_permitted", str), + ("cap_effective", str), + ("cap_bounding", str), + ("cap_ambient", str), ] - if self.config.get("inheritable"): - columns.append(("cap_inheritable", str)) - elif self.config.get("permitted"): - columns.append(("cap_permitted", str)) - elif self.config.get("effective"): - columns.append(("cap_effective", str)) - elif self.config.get("bounding"): - columns.append(("cap_bounding", str)) - elif self.config.get("ambient"): - columns.append(("cap_ambient", str)) - else: - columns.append(("cap_inheritable", format_hints.Hex)) - columns.append(("cap_permitted", format_hints.Hex)) - columns.append(("cap_effective", format_hints.Hex)) - columns.append(("cap_bounding", format_hints.Hex)) - columns.append(("cap_ambient", format_hints.Hex)) - return renderers.TreeGrid(columns, self._generator(tasks)) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 4b1c53683..641f77728 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,7 +13,7 @@ from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES -from volatility3.framework.constants.linux import CAPABILITIES +from volatility3.framework.constants.linux import CAPABILITIES, CAP_FULL from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility @@ -1508,7 +1508,7 @@ class kernel_cap_struct(objects.StructType): """ # In kernels 2.6.25.20 the kernel_cap_struct::cap became and array cap_value = self.cap[0] if isinstance(self.cap, objects.Array) else self.cap - return int(cap_value & 0xFFFFFFFF) + return cap_value & CAP_FULL def enumerate_capabilities(self) -> List[str]: """Returns the list of capability strings. From 239e164ce6b2237b1b5569bba9e540b5a5694a45 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 9 Jul 2023 23:50:13 +0100 Subject: [PATCH 088/101] Windows: Fix strings plugin missing kernel pages @eve-mem spotted that we were only recording the first kernel page in any contiguous set of kernel pages, thus missing some results. This should now be fixed. --- volatility3/framework/plugins/windows/strings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 32f3df4c8..0eaa65884 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -149,9 +149,9 @@ class Strings(interfaces.plugins.PluginInterface): for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors=True): offset, _, mapped_offset, mapped_size, maplayer = mapval for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): - cur_set = reverse_map.get(mapped_offset >> 12, set()) + cur_set = reverse_map.get(val >> 12, set()) cur_set.add(("kernel", offset)) - reverse_map[mapped_offset >> 12] = cur_set + reverse_map[val >> 12] = cur_set if progress_callback: progress_callback( (offset * 100) / layer.maximum_address, From 63c7719763d8d5c5080840bad740d7fa6be14816 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jul 2023 01:01:30 +0200 Subject: [PATCH 089/101] Fixed returned types. An empty string is now returned when no capability is enabled. --- .../framework/plugins/linux/capabilities.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 9a59f31c1..aa3edfe4b 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -114,8 +114,8 @@ class Capabilities(plugins.PluginInterface): return cap cap_value = cap.get_capabilities() - if cap_value == 0: - return "-" + if not cap_value: + return "" if cap_value == CAP_FULL: return "all" @@ -123,14 +123,16 @@ class Capabilities(plugins.PluginInterface): return ", ".join(cap.enumerate_capabilities()) @classmethod - def get_task_capabilities(cls, task: interfaces.objects.ObjectInterface) -> Dict: + def get_task_capabilities( + cls, task: interfaces.objects.ObjectInterface + ) -> Tuple[TaskData, CapabilitiesData]: """Returns a dict with the task basic information along with its capabilities Args: task: A task object from where to get the fields. Returns: - dict: A dict with the task basic information along with its capabilities + A tuple with the task basic information and its capabilities """ task_data = TaskData( comm=utility.array_to_string(task.comm), @@ -158,14 +160,14 @@ class Capabilities(plugins.PluginInterface): @classmethod def get_tasks_capabilities( cls, tasks: List[interfaces.objects.ObjectInterface] - ) -> Iterable[Dict]: + ) -> Iterable[Tuple[TaskData, CapabilitiesData]]: """Yields a dict for each task containing the task's basic information along with its capabilities Args: tasks: An iterable with the tasks to process. Yields: - Iterable[Dict]: A dict for each task containing the task's basic information along with its capabilities + A tuple for each task containing the task's basic information and its capabilities """ for task in tasks: yield cls.get_task_capabilities(task) From 38e9b6baa87dbfb9ad8795621aa97a5fd1c61b30 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jul 2023 01:08:46 +0200 Subject: [PATCH 090/101] Removed Dict type from the imports --- volatility3/framework/plugins/linux/capabilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index aa3edfe4b..0eb9e3b7a 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -4,7 +4,7 @@ import logging from dataclasses import dataclass, astuple, fields -from typing import Iterable, List, Tuple, Dict +from typing import Iterable, List, Tuple from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.constants.linux import CAP_FULL From 42fe37ed52cf62b5eebe073420e9a6245307eb60 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jul 2023 01:12:20 +0200 Subject: [PATCH 091/101] Adjust docstring to the new retuned type. --- volatility3/framework/plugins/linux/capabilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 0eb9e3b7a..84d1d543f 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -126,7 +126,7 @@ class Capabilities(plugins.PluginInterface): def get_task_capabilities( cls, task: interfaces.objects.ObjectInterface ) -> Tuple[TaskData, CapabilitiesData]: - """Returns a dict with the task basic information along with its capabilities + """Returns a tuple with the task basic information along with its capabilities Args: task: A task object from where to get the fields. @@ -161,7 +161,7 @@ class Capabilities(plugins.PluginInterface): def get_tasks_capabilities( cls, tasks: List[interfaces.objects.ObjectInterface] ) -> Iterable[Tuple[TaskData, CapabilitiesData]]: - """Yields a dict for each task containing the task's basic information along with its capabilities + """Yields a tuple for each task containing the task's basic information along with its capabilities Args: tasks: An iterable with the tasks to process. From f8fc5d5e58495d9f45d900cd9041ba37d0d078dc Mon Sep 17 00:00:00 2001 From: cpuu Date: Wed, 12 Jul 2023 14:43:38 +0900 Subject: [PATCH 092/101] Update pslist.py list modules in code in alphabetical order --- volatility3/framework/plugins/mac/pslist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index dbed09818..88045a277 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -4,13 +4,13 @@ import datetime import logging -from typing import Callable, Iterable, List, Dict +from typing import Callable, Dict, Iterable, List -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility -from volatility3.framework.symbols import mac from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import mac vollog = logging.getLogger(__name__) From 9ba3d9ba4e5ed4e239c81a59c50de0ca5db08ef1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 19 Jul 2023 22:48:06 +0200 Subject: [PATCH 093/101] Several fixes: * The capabilities array was to have a 64bit bitwise. * Capabilities set has to be tested with the kernel maximum. We can't use the plugin capabilities set, otherwise we can have wrong interpretations when we tried to compress the list of capabilities to "all" * Supports kernels >= 6.3. They changed the kernel_cap_struct::cap type again to a u64 type. --- .../framework/constants/linux/__init__.py | 2 -- .../framework/plugins/linux/capabilities.py | 3 +- .../symbols/linux/extensions/__init__.py | 31 ++++++++++++++++--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index e57fa30d4..a802e0ada 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -279,5 +279,3 @@ CAPABILITIES = ( "bpf", "checkpoint_restore", ) - -CAP_FULL = 0xFFFFFFFF diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 84d1d543f..518f52603 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -7,7 +7,6 @@ from dataclasses import dataclass, astuple, fields from typing import Iterable, List, Tuple from volatility3.framework import interfaces, renderers, exceptions -from volatility3.framework.constants.linux import CAP_FULL from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -117,7 +116,7 @@ class Capabilities(plugins.PluginInterface): if not cap_value: return "" - if cap_value == CAP_FULL: + if cap_value == cap.get_kernel_cap_full(): return "all" return ", ".join(cap.enumerate_capabilities()) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0137b2cc4..bc2c6e27b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,7 +13,7 @@ from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES -from volatility3.framework.constants.linux import CAPABILITIES, CAP_FULL +from volatility3.framework.constants.linux import CAPABILITIES from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility @@ -1482,6 +1482,21 @@ class kernel_cap_struct(objects.StructType): """ return len(CAPABILITIES) - 1 + def get_kernel_cap_full(self) -> int: + """Return the maximum value allowed for this kernel for a capability + + Returns: + int: _description_ + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + try: + cap_last_cap = vmlinux.object_from_symbol(symbol_name="cap_last_cap") + except exceptions.SymbolError: + # It should be a kernel < 3.2, let's use our list of capabilities + cap_last_cap = self.get_last_cap_value() + + return (1 << cap_last_cap + 1) - 1 + @classmethod def capabilities_to_string(cls, capabilities_bitfield: int) -> List[str]: """Translates a capability bitfield to a list of capability strings. @@ -1506,9 +1521,17 @@ class kernel_cap_struct(objects.StructType): Returns: int: The capability bitfield value. """ - # In kernels 2.6.25.20 the kernel_cap_struct::cap became and array - cap_value = self.cap[0] if isinstance(self.cap, objects.Array) else self.cap - return cap_value & CAP_FULL + + if isinstance(self.cap, objects.Array): + # In 2.6.25.x <= kernels < 6.3 kernel_cap_struct::cap is an array + # to become a 64bit bitfield + cap_value = (self.cap[1] << 32) | self.cap[0] + else: + # In kernels < 2.6.25.x kernel_cap_struct::cap was a u32 + # In kernels >= 6.3 kernel_cap_struct::cap is a u64 + cap_value = self.cap + + return cap_value & self.get_kernel_cap_full() def enumerate_capabilities(self) -> List[str]: """Returns the list of capability strings. From 90da6298e0f69cbc42ef2a6e1da5863aae4d5031 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 20 Jul 2023 00:29:52 +0200 Subject: [PATCH 094/101] Added further details to the kernel_cap_struct::cap comments --- .../framework/symbols/linux/extensions/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index bc2c6e27b..527785a69 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1523,11 +1523,15 @@ class kernel_cap_struct(objects.StructType): """ if isinstance(self.cap, objects.Array): - # In 2.6.25.x <= kernels < 6.3 kernel_cap_struct::cap is an array - # to become a 64bit bitfield + # In 2.6.25.x <= kernels < 6.3 kernel_cap_struct::cap is a two + # elements __u32 array that constitutes a 64bit bitfield. + # Technically, it can also be an array of 1 element if + # _KERNEL_CAPABILITY_U32S = _LINUX_CAPABILITY_U32S_1 + # However, in the source code, that never happens. + # From 2.6.24 to 2.6.25 cap became an array of 2 elements. cap_value = (self.cap[1] << 32) | self.cap[0] else: - # In kernels < 2.6.25.x kernel_cap_struct::cap was a u32 + # In kernels < 2.6.25.x kernel_cap_struct::cap was a __u32 # In kernels >= 6.3 kernel_cap_struct::cap is a u64 cap_value = self.cap From d872abaf302291cbffed1695bccbeb2805aa4c1e Mon Sep 17 00:00:00 2001 From: xabrouck Date: Thu, 27 Jul 2023 09:28:53 +0200 Subject: [PATCH 095/101] fix bug in snappy lib loading and make it work on macOS. --- volatility3/framework/layers/avml.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index c9c682ac4..1f4a7e053 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -19,11 +19,17 @@ vollog = logging.getLogger(__name__) try: # TODO: Find library for windows if needed try: - # Linux/Mac + # Linux lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.so.1") except OSError: lib_snappy = None + try: + # macOS + lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.1.dylib") + except OSError: + lib_snappy = None + try: if not lib_snappy: # Windows 64 @@ -31,7 +37,7 @@ try: except OSError: lib_snappy = None - if lib_snappy: + if not lib_snappy: # Windows 32 lib_snappy = ctypes.cdll.LoadLibrary("snappy32") From a226b90b4de512f3777210a361297b7a81324d53 Mon Sep 17 00:00:00 2001 From: xabrouck Date: Thu, 27 Jul 2023 09:37:57 +0200 Subject: [PATCH 096/101] previous commit would break linux snappy support --- volatility3/framework/layers/avml.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index 1f4a7e053..c825464cc 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -25,8 +25,9 @@ try: lib_snappy = None try: - # macOS - lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.1.dylib") + if not lib_snappy: + # macOS + lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.1.dylib") except OSError: lib_snappy = None From 5d4c70d5174ff354176288fcd17ca2d1acaf2f56 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 3 Aug 2023 15:16:36 +0100 Subject: [PATCH 097/101] Documentation: Fix requirements -> get_requirements #993 --- doc/source/using-as-a-library.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index c63adcfc3..fb012f1ae 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -67,9 +67,10 @@ return a dictionary of plugin names and the plugin classes. Determine what configuration options a plugin requires ------------------------------------------------------ -For each plugin class, we can call the classmethod `requirements` on it, which will return a list of objects that -adhere to the :py:class:`~volatility3.framework.interfaces.configuration.RequirementInterface` method. The various -types of Requirement are split roughly in two, +For each plugin class, we can call the classmethod +:py:func:`~volatility3.framework.interfaces.configuration.ConfigurableInterface.get_requirements` on it, which will +return a list of objects that adhere to the :py:class:`~volatility3.framework.interfaces.configuration.RequirementInterface` +method. The various types of Requirement are split roughly in two, :py:class:`~volatility3.framework.interfaces.configuration.SimpleTypeRequirement` (such as integers, booleans, floats and strings) and more complex requirements (such as lists, choices, multiple requirements, translation layer requirements or symbol table requirements). A requirement just specifies a type of data and a name, and must be From 7c82da4f5044a4bf35028c01924e659ccac5e828 Mon Sep 17 00:00:00 2001 From: xabrouck Date: Mon, 14 Aug 2023 11:53:14 +0200 Subject: [PATCH 098/101] check IoC of dirty bit in PTEs from executable VMAs. this can for example detect code injected using ptrace(). this can also detect injected code that was reset to the original code (malware uninstalled before memory dump happened). --- volatility3/framework/layers/intel.py | 9 +++++++++ volatility3/framework/plugins/linux/malfind.py | 2 +- .../framework/symbols/linux/extensions/__init__.py | 11 ++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 478eb168f..e2d89540d 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -110,6 +110,11 @@ class Intel(linear.LinearlyMappedLayer): def _page_is_valid(entry: int) -> bool: """Returns whether a particular page is valid based on its entry.""" return bool(entry & 1) + + @staticmethod + def _page_is_dirty(entry: int) -> bool: + """Returns whether a particular page is dirty based on its entry.""" + return bool(entry & (1<<6)) def canonicalize(self, addr: int) -> int: """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" @@ -259,6 +264,10 @@ class Intel(linear.LinearlyMappedLayer): except exceptions.InvalidAddressException: return False + def is_dirty(self, offset: int) -> bool: + """Returns whether the page at offset is marked dirty""" + return self._page_is_dirty(self._translate_entry(offset)[0]) + def mapping( self, offset: int, length: int, ignore_errors: bool = False ) -> Iterable[Tuple[int, int, int, int, str]]: diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 1fd005de8..332d5ede1 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -47,7 +47,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.mm.get_vma_iter(): - if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]": + if vma.is_suspicious(proc_layer) and vma.get_name(self.context, task) != "[vdso]": data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 527785a69..d2e6197ef 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -578,7 +578,7 @@ class vm_area_struct(objects.StructType): return fname # used by malfind - def is_suspicious(self): + def is_suspicious(self, proclayer): ret = False flags_str = self.get_protection() @@ -587,6 +587,15 @@ class vm_area_struct(objects.StructType): ret = True elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True + elif "x" in flags_str: + for i in range(self.vm_start,self.vm_end,constants.linux.PAGE_SHIFT): + try: + if proclayer.is_dirty(i): + vollog.warning(f"Found malicious (dirty+exec) page at {hex(i)} !") + ret = True + break + except (exceptions.PagedInvalidAddressException, exceptions.InvalidAddressException): + pass return ret From 6f7f1284adbbcfacd759ec0d931859e0789cfc8a Mon Sep 17 00:00:00 2001 From: xabrouck Date: Fri, 18 Aug 2023 11:37:04 +0200 Subject: [PATCH 099/101] Fix bug with PAGE_SHIFT that wasn't shifted, also greatly increases performance Use black Better logging --- volatility3/framework/layers/intel.py | 6 +++--- .../framework/plugins/linux/malfind.py | 13 ++++++++++-- .../symbols/linux/extensions/__init__.py | 21 +++++++++++++------ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index e2d89540d..046203fa6 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -110,11 +110,11 @@ class Intel(linear.LinearlyMappedLayer): def _page_is_valid(entry: int) -> bool: """Returns whether a particular page is valid based on its entry.""" return bool(entry & 1) - + @staticmethod def _page_is_dirty(entry: int) -> bool: """Returns whether a particular page is dirty based on its entry.""" - return bool(entry & (1<<6)) + return bool(entry & (1 << 6)) def canonicalize(self, addr: int) -> int: """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" @@ -267,7 +267,7 @@ class Intel(linear.LinearlyMappedLayer): def is_dirty(self, offset: int) -> bool: """Returns whether the page at offset is marked dirty""" return self._page_is_dirty(self._translate_entry(offset)[0]) - + def mapping( self, offset: int, length: int, ignore_errors: bool = False ) -> Iterable[Tuple[int, int, int, int, str]]: diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 332d5ede1..8a21afc03 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -3,7 +3,7 @@ # from typing import List - +import logging from volatility3.framework import constants, interfaces from volatility3.framework import renderers from volatility3.framework.configuration import requirements @@ -11,6 +11,8 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) + class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" @@ -47,7 +49,14 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.mm.get_vma_iter(): - if vma.is_suspicious(proc_layer) and vma.get_name(self.context, task) != "[vdso]": + vma_name = vma.get_name(self.context, task) + vollog.debug( + f"Injections : processing PID {task.pid} : VMA {vma_name} : {hex(vma.vm_start)}-{hex(vma.vm_end)}" + ) + if ( + vma.is_suspicious(proc_layer) + and vma.get_name(self.context, task) != "[vdso]" + ): data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d2e6197ef..616e54e70 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -578,7 +578,7 @@ class vm_area_struct(objects.StructType): return fname # used by malfind - def is_suspicious(self, proclayer): + def is_suspicious(self, proclayer=None): ret = False flags_str = self.get_protection() @@ -587,15 +587,24 @@ class vm_area_struct(objects.StructType): ret = True elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True - elif "x" in flags_str: - for i in range(self.vm_start,self.vm_end,constants.linux.PAGE_SHIFT): + elif proclayer and "x" in flags_str: + for i in range(self.vm_start, self.vm_end, 1 << constants.linux.PAGE_SHIFT): try: if proclayer.is_dirty(i): - vollog.warning(f"Found malicious (dirty+exec) page at {hex(i)} !") + vollog.warning( + f"Found malicious (dirty+exec) page at {hex(i)} !" + ) + # We do not attempt to find other dirty+exec pages once we have found one ret = True break - except (exceptions.PagedInvalidAddressException, exceptions.InvalidAddressException): - pass + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ) as excp: + vollog.debug(f"Unable to translate address {hex(i)} : {excp}") + # Abort as it is likely that other addresses in the same range will also fail + ret = False + break return ret From 43bef641dc9b79f6ba3f5c0e345e9d76ffcd1318 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Sep 2023 20:37:38 +0100 Subject: [PATCH 100/101] Revert "Linux: Fix slight issue in envvars renaming" This reverts commit 60292c2da1efa886d0f46bd720af537a6069a623. --- volatility3/framework/plugins/linux/envars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 758943312..c4e3ed3c9 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -1,4 +1,4 @@ -from volatility3.plugins.linux import envvars +from volatility3.plugins import envvars import logging vollog = logging.getLogger(__name__) From 56cb5ea93ba74819ba4fe3a9bec39220d6d60d43 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Sep 2023 20:37:51 +0100 Subject: [PATCH 101/101] Revert "Linux: Rename linux.envars to linux.envvars" This reverts commit d9a365d96fcd990c7faba32ab7aa63523203e9f8. --- volatility3/framework/plugins/linux/envars.py | 121 +++++++++++++++++- .../framework/plugins/linux/envvars.py | 121 ------------------ 2 files changed, 115 insertions(+), 127 deletions(-) delete mode 100644 volatility3/framework/plugins/linux/envvars.py diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index c4e3ed3c9..5cbf0f502 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -1,12 +1,121 @@ -from volatility3.plugins import envvars +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import logging +from volatility3.framework import exceptions, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist + vollog = logging.getLogger(__name__) -class Envars(envvars.Envvars): - def run(self, *args, **kwargs): - vollog.warning( - "The linux.envars plugin has been renamed to linux.envvars and will only be accessible through the new name in a future release" +class Envars(plugins.PluginInterface): + """Lists processes with their environment variables""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + ] + + def _generator(self, tasks): + """Generates a listing of processes along with environment variables""" + + # walk the process list and return the envars + for task in tasks: + pid = task.pid + + # get process name as string + name = utility.array_to_string(task.comm) + + # try and get task parent + try: + ppid = task.parent.pid + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." + ) + ppid = 0 + + # kernel threads never have an mm as they do not have userland mappings + try: + mm = task.mm + except exceptions.InvalidAddressException: + # no mm so cannot get envars + vollog.debug( + f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." + ) + mm = None + continue + + # if mm exists attempt to get envars + if mm: + # get process layer to read envars from + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + vollog.debug( + f"Unable to construct process layer for task {pid} {name}, will not extract any envars." + ) + continue + proc_layer = self.context.layers[proc_layer_name] + + # get the size of the envars with sanity checking + envars_size = task.mm.env_end - task.mm.env_start + if not (0 < envars_size <= 8192): + vollog.debug( + f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." + ) + continue + + # attempt to read all envars data + try: + envar_data = proc_layer.read(task.mm.env_start, envars_size) + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." + ) + continue + + # parse envar data, envars are null terminated, keys and values are separated by '=' + envar_data = envar_data.rstrip(b"\x00") + for envar_pair in envar_data.split(b"\x00"): + try: + key, value = envar_pair.decode().split("=", 1) + except ValueError: + vollog.debug( + f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" + ) + continue + yield (0, (pid, ppid, name, key, value)) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + return renderers.TreeGrid( + [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), ) - return super().run(*args, **kwargs) diff --git a/volatility3/framework/plugins/linux/envvars.py b/volatility3/framework/plugins/linux/envvars.py deleted file mode 100644 index 1d6c8b784..000000000 --- a/volatility3/framework/plugins/linux/envvars.py +++ /dev/null @@ -1,121 +0,0 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -import logging - -from volatility3.framework import exceptions, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins -from volatility3.framework.objects import utility -from volatility3.plugins.linux import pslist - -vollog = logging.getLogger(__name__) - - -class Envvars(plugins.PluginInterface): - """Lists processes with their environment variables""" - - _required_framework_version = (2, 0, 0) - - @classmethod - def get_requirements(cls): - # Since we're calling the plugin, make sure we have the plugin's requirements - return [ - requirements.ModuleRequirement( - name="kernel", - description="Linux kernel", - architectures=["Intel32", "Intel64"], - ), - requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) - ), - requirements.ListRequirement( - name="pid", - description="Filter on specific process IDs", - element_type=int, - optional=True, - ), - ] - - def _generator(self, tasks): - """Generates a listing of processes along with environment variables""" - - # walk the process list and return the envars - for task in tasks: - pid = task.pid - - # get process name as string - name = utility.array_to_string(task.comm) - - # try and get task parent - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." - ) - ppid = 0 - - # kernel threads never have an mm as they do not have userland mappings - try: - mm = task.mm - except exceptions.InvalidAddressException: - # no mm so cannot get envars - vollog.debug( - f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." - ) - mm = None - continue - - # if mm exists attempt to get envars - if mm: - # get process layer to read envars from - proc_layer_name = task.add_process_layer() - if proc_layer_name is None: - vollog.debug( - f"Unable to construct process layer for task {pid} {name}, will not extract any envars." - ) - continue - proc_layer = self.context.layers[proc_layer_name] - - # get the size of the envars with sanity checking - envars_size = task.mm.env_end - task.mm.env_start - if not (0 < envars_size <= 8192): - vollog.debug( - f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." - ) - continue - - # attempt to read all envars data - try: - envar_data = proc_layer.read(task.mm.env_start, envars_size) - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." - ) - continue - - # parse envar data, envars are null terminated, keys and values are separated by '=' - envar_data = envar_data.rstrip(b"\x00") - for envar_pair in envar_data.split(b"\x00"): - try: - key, value = envar_pair.decode().split("=", 1) - except ValueError: - vollog.debug( - f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" - ) - continue - yield (0, (pid, ppid, name, key, value)) - - def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - - return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], - self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ), - )