From 867c8e3f1bdeb0fa57dd3e02056f32b526a81fb4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 15:33:19 +1000 Subject: [PATCH] Linux: Add support for threads in both lsof and sockstat plugins. - lsof plugin: source code refactored - lsof plugin: Added the 'device' column to complete the inode information. An inode number is specific to the filesystem/device it belongs to. - lsof/sockstat plugins: Add threads support. Threads may or may not share the file descriptor table with the thread group leader, depending on whether the CLONE_FILES flag is included in the clone() syscall. Also, once started, a thread can unshare the fd table with its parent. Refer to the unshare() libc syscall wrapper man page, unshare(2). Additionally, note that the Linux lsof command in user space includes thread listings by default as well. Now, there are two columns to identify the thread group ID (PID) and the task/thread ID (TID). - Added inode getters from both, the dentry and file structs. From kernels +3.9 the file struct cached the inode pointer. So, when possible, we get this value. - Improve smear protection in these plugins and various APIs (https://github.com/volatilityfoundation/volatility3/pull/1243) --- volatility3/framework/plugins/linux/lsof.py | 229 +++++++++++------- .../framework/plugins/linux/sockstat.py | 18 +- .../framework/symbols/linux/__init__.py | 4 +- .../symbols/linux/extensions/__init__.py | 42 +++- 4 files changed, 185 insertions(+), 108 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 360f89749..0325ebd4c 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -1,12 +1,12 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" -import logging, datetime +import logging +from datetime import datetime +from dataclasses import dataclass, astuple, field from typing import List, Callable -from volatility3.framework import renderers, interfaces, constants, exceptions +from volatility3.framework import renderers, interfaces, constants from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -17,11 +17,94 @@ from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) +@dataclass +class FDUser: + """FD user representation, featuring augmented information and formatted fields. + This is the data the plugin will eventually display. + """ + + task_tgid: int + task_tid: int + task_comm: str + fd_num: int + full_path: str + device: str = field(default=renderers.NotAvailableValue()) + inode_num: int = field(default=renderers.NotAvailableValue()) + inode_type: str = field(default=renderers.NotAvailableValue()) + file_mode: str = field(default=renderers.NotAvailableValue()) + change_time: datetime = field(default=renderers.NotAvailableValue()) + modification_time: datetime = field(default=renderers.NotAvailableValue()) + access_time: datetime = field(default=renderers.NotAvailableValue()) + inode_size: int = field(default=renderers.NotAvailableValue()) + + +@dataclass +class FDInternal: + """FD internal representation containing only the core objects + + Fields: + task: 'task_truct' object + fd_fields: FD fields as obtained from LinuxUtilities.files_descriptors_for_process() + """ + + task: interfaces.objects.ObjectInterface + fd_fields: tuple[int, int, str] + + def to_user(self) -> FDUser: + """Augment the FD information to be presented to the user + + Returns: + An InodeUser dataclass + """ + # Ensure all types are atomic immutable. Otherwise, astuple() will take a long + # time doing a deepcopy of the Volatility objects. + task_tgid = int(self.task.tgid) + task_tid = int(self.task.pid) + task_comm = utility.array_to_string(self.task.comm) + fd_num, filp, full_path = self.fd_fields + fd_num = int(fd_num) + full_path = str(full_path) + inode = filp.get_inode() + if inode: + superblock_ptr = inode.i_sb + if superblock_ptr and superblock_ptr.is_readable(): + device = f"{superblock_ptr.major}:{superblock_ptr.minor}" + else: + device = renderers.NotAvailableValue() + + fd_user = FDUser( + task_tgid=task_tgid, + task_tid=task_tid, + task_comm=task_comm, + fd_num=fd_num, + full_path=full_path, + device=device, + inode_num=int(inode.i_ino), + inode_type=inode.get_inode_type() or renderers.UnparsableValue(), + file_mode=inode.get_file_mode(), + change_time=inode.get_change_time(), + modification_time=inode.get_modification_time(), + access_time=inode.get_access_time(), + inode_size=int(inode.i_size), + ) + else: + # We use the dataclasses' default values + fd_user = FDUser( + task_tgid=task_tgid, + task_tid=task_tid, + task_comm=task_comm, + fd_num=fd_num, + full_path=full_path, + ) + + return fd_user + + class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 2, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -45,126 +128,86 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] - @classmethod - def get_inode_metadata(cls, filp: interfaces.objects.ObjectInterface): - try: - dentry = filp.get_dentry() - if dentry: - inode_object = dentry.d_inode - if inode_object and inode_object.is_valid(): - itype = ( - inode_object.get_inode_type() or renderers.NotAvailableValue() - ) - return ( - inode_object.i_ino, - itype, - inode_object.i_size, - inode_object.get_file_mode(), - inode_object.get_change_time(), - inode_object.get_modification_time(), - inode_object.get_access_time(), - ) - except (exceptions.InvalidAddressException, AttributeError) as e: - vollog.warning(f"Can't get inode metadata: {e}") - return None - @classmethod def list_fds( cls, context: interfaces.context.ContextInterface, - symbol_table: str, + vmlinux_module_name: str, filter_func: Callable[[int], bool] = lambda _: False, - ): + ) -> FDInternal: + """Enumerates open file descriptors in tasks + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + filter_func: A function which takes a process object and returns True if the process + should be ignored/filtered + + Yields: + A FDInternal object + """ linuxutils_symbol_table = None - for task in pslist.PsList.list_tasks(context, symbol_table, filter_func): + for task in pslist.PsList.list_tasks( + context, vmlinux_module_name, filter_func, include_threads=True + ): if linuxutils_symbol_table is None: if constants.BANG not in task.vol.type_name: raise ValueError("Task is not part of a symbol table") linuxutils_symbol_table = task.vol.type_name.split(constants.BANG)[0] - task_comm = utility.array_to_string(task.comm) - pid = int(task.pid) - fd_generator = linux.LinuxUtilities.files_descriptors_for_process( context, linuxutils_symbol_table, task ) for fd_fields in fd_generator: - yield pid, task_comm, task, fd_fields + yield FDInternal(task=task, fd_fields=fd_fields) - @classmethod - def list_fds_and_inodes( - cls, - context: interfaces.context.ContextInterface, - symbol_table: str, - filter_func: Callable[[int], bool] = lambda _: False, - ): - for pid, task_comm, task, (fd_num, filp, full_path) in cls.list_fds( - context, symbol_table, filter_func - ): - inode_metadata = cls.get_inode_metadata(filp) - if inode_metadata is None: - inode_metadata = tuple( - interfaces.renderers.BaseAbsentValue() for _ in range(7) - ) - yield pid, task_comm, task, fd_num, filp, full_path, inode_metadata - - def _generator(self, pids, symbol_table): + def _generator(self, pids, vmlinux_module_name): filter_func = pslist.PsList.create_pid_filter(pids) - fds_generator = self.list_fds_and_inodes( - self.context, symbol_table, filter_func=filter_func - ) - - for ( - pid, - task_comm, - task, - fd_num, - filp, - full_path, - inode_metadata, - ) in fds_generator: - inode_num, itype, file_size, imode, ctime, mtime, atime = inode_metadata - fields = ( - pid, - task_comm, - fd_num, - full_path, - inode_num, - itype, - imode, - ctime, - mtime, - atime, - file_size, - ) - yield (0, fields) + for fd_internal in self.list_fds( + self.context, vmlinux_module_name, filter_func=filter_func + ): + fd_user = fd_internal.to_user() + yield (0, astuple(fd_user)) def run(self): pids = self.config.get("pid", None) - symbol_table = self.config["kernel"] + vmlinux_module_name = self.config["kernel"] tree_grid_args = [ ("PID", int), + ("TID", int), ("Process", str), ("FD", int), ("Path", str), + ("Device", str), ("Inode", int), ("Type", str), ("Mode", str), - ("Changed", datetime.datetime), - ("Modified", datetime.datetime), - ("Accessed", datetime.datetime), + ("Changed", datetime), + ("Modified", datetime), + ("Accessed", datetime), ("Size", int), ] - return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) + return renderers.TreeGrid( + tree_grid_args, self._generator(pids, vmlinux_module_name) + ) def generate_timeline(self): pids = self.config.get("pid", None) - symbol_table = self.config["kernel"] - for row in self._generator(pids, symbol_table): - _depth, row_data = row - description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' - yield description, timeliner.TimeLinerType.CHANGED, row_data[7] - yield description, timeliner.TimeLinerType.MODIFIED, row_data[8] - yield description, timeliner.TimeLinerType.ACCESSED, row_data[9] + vmlinux_module_name = self.config["kernel"] + + filter_func = pslist.PsList.create_pid_filter(pids) + for fd_internal in self.list_fds( + self.context, vmlinux_module_name, filter_func=filter_func + ): + fd_user = fd_internal.to_user() + + description = ( + f"Process {fd_user.task_comm} ({fd_user.task_tgid}/{fd_user.task_tid}) " + f"Open '{fd_user.full_path}'" + ) + + yield description, timeliner.TimeLinerType.CHANGED, fd_user.change_time + yield description, timeliner.TimeLinerType.MODIFIED, fd_user.modification_time + yield description, timeliner.TimeLinerType.ACCESSED, fd_user.access_time diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index b0503b105..c76e0a6b4 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -22,9 +22,10 @@ class SockHandlers(interfaces.configuration.VersionableInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) - def __init__(self, vmlinux, task): + def __init__(self, vmlinux, task, *args, **kwargs): + super().__init__(*args, **kwargs) self._vmlinux = vmlinux self._task = task @@ -438,7 +439,7 @@ class Sockstat(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -452,7 +453,7 @@ class Sockstat(plugins.PluginInterface): name="SockHandlers", component=SockHandlers, version=(1, 0, 0) ), requirements.PluginRequirement( - name="lsof", plugin=lsof.Lsof, version=(1, 1, 0) + name="lsof", plugin=lsof.Lsof, version=(2, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) @@ -507,8 +508,9 @@ class Sockstat(plugins.PluginInterface): dfop_addr = vmlinux.object_from_symbol("sockfs_dentry_operations").vol.offset fd_generator = lsof.Lsof.list_fds(context, vmlinux.name, filter_func) - for _pid, _task_comm, task, fd_fields in fd_generator: - fd_num, filp, _full_path = fd_fields + for fd_internal in fd_generator: + fd_num, filp, _full_path = fd_internal.fd_fields + task = fd_internal.task if filp.f_op not in (sfop_addr, dfop_addr): continue @@ -617,6 +619,7 @@ class Sockstat(plugins.PluginInterface): fields = ( netns_id, + task.tgid, task.pid, fd_num, format_hints.Hex(sock.vol.offset), @@ -636,7 +639,8 @@ class Sockstat(plugins.PluginInterface): tree_grid_args = [ ("NetNS", int), - ("Pid", int), + ("PID", int), + ("TID", int), ("FD", int), ("Sock Offset", format_hints.Hex), ("Family", str), diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 91abf7db4..6e9297517 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -256,7 +256,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): task: interfaces.objects.ObjectInterface, ): # task.files can be null - if not task.files: + if not (task.files and task.files.is_readable()): return None fd_table = task.files.get_fds() @@ -276,7 +276,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ) for fd_num, filp in enumerate(fds): - if filp != 0: + if filp and filp.is_readable(): full_path = LinuxUtilities.path_for_file(context, task, filp) yield fd_num, filp, full_path diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fdd34403a..fd5ddcffa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -843,25 +843,55 @@ class dentry(objects.StructType): dentry_type_name = self.get_symbol_table_name() + constants.BANG + "dentry" yield from list_head_member.to_list(dentry_type_name, walk_member) + def get_inode(self) -> interfaces.objects.ObjectInterface: + """Returns the inode associated with this dentry""" + + inode_ptr = self.d_inode + if not (inode_ptr and inode_ptr.is_valid()): + return None + + return inode_ptr.dereference() + class struct_file(objects.StructType): def get_dentry(self) -> interfaces.objects.ObjectInterface: - if self.has_member("f_dentry"): - return self.f_dentry - elif self.has_member("f_path"): + """Returns a pointer to the dentry associated with this file""" + if self.has_member("f_path"): return self.f_path.dentry + elif self.has_member("f_dentry"): + return self.f_dentry else: raise AttributeError("Unable to find file -> dentry") def get_vfsmnt(self) -> interfaces.objects.ObjectInterface: """Returns the fs (vfsmount) where this file is mounted""" - if self.has_member("f_vfsmnt"): - return self.f_vfsmnt - elif self.has_member("f_path"): + if self.has_member("f_path"): return self.f_path.mnt + elif self.has_member("f_vfsmnt"): + return self.f_vfsmnt else: raise AttributeError("Unable to find file -> vfs mount") + def get_inode(self) -> interfaces.objects.ObjectInterface: + """Returns an inode associated with this file""" + + inode_ptr = None + if self.has_member("f_inode") and self.f_inode and self.f_inode.is_readable(): + # Try first the cached value, kernels +3.9 + inode_ptr = self.f_inode + + if not (inode_ptr and inode_ptr.is_valid()): + dentry_ptr = self.get_dentry() + if not (dentry_ptr and dentry_ptr.is_readable()): + return None + + inode_ptr = dentry_ptr.d_inode + + if not (inode_ptr and inode_ptr.is_valid()): + return None + + return inode_ptr.dereference() + class list_head(objects.StructType, collections.abc.Iterable): def to_list(