From 558b31cbdc9002dd8b0dacc2af8c5296ee3bcba5 Mon Sep 17 00:00:00 2001 From: cstation Date: Fri, 13 Jan 2023 11:58:11 +0100 Subject: [PATCH 01/22] Dump ELFs to file --- volatility3/framework/plugins/linux/elfs.py | 113 +++++++++++++++++- volatility3/framework/plugins/linux/pslist.py | 101 +++++++++++----- 2 files changed, 180 insertions(+), 34 deletions(-) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 822a69dd6..f56438f4f 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -4,20 +4,26 @@ """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" -from typing import List +import logging +from typing import List, Optional, Type -from volatility3.framework import renderers, interfaces +from volatility3.framework import constants, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.linux.extensions import elf from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) + class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -36,9 +42,95 @@ class Elfs(plugins.PluginInterface): element_type=int, optional=True, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed processes", + default=False, + optional=True, + ), ] + @classmethod + def elf_dump( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + elf_table_name: str, + vma: interfaces.objects.ObjectInterface, + task: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + ) -> Optional[interfaces.plugins.FileHandlerInterface]: + """Extracts an ELF as a FileHandlerInterface + Args: + context: the context to operate upon + layer_name: The name of the layer on which to operate + elf_table_name: the name for the symbol table containing the symbols for ELF-files + vma: virtual memory allocation of ELF + task: the task object whose memory should be output + open_method: class to provide context manager for opening the file + Returns: + An open FileHandlerInterface object containing the complete data for the task or None in the case of failure + """ + + proc_layer = context.layers[layer_name] + file_handle = None + + try: + elf_object = context.object( + elf_table_name + constants.BANG + "Elf", + offset=vma.vm_start, + layer_name=layer_name, + ) + + if not elf_object.is_valid(): + return None + + sections = {} + # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? + for phdr in elf_object.get_program_headers(): + if phdr.p_type != 1: # PT_LOAD = 1 + continue + + start = phdr.p_vaddr + size = phdr.p_memsz + end = start + size + + # Use complete memory pages for dumping + # If start isn't a multiple of 4096, stick to the highest multiple < start + # If end isn't a multiple of 4096, stick to the lowest multiple > end + if start % 4096: + start = start & ~0xFFF + + if end % 4096: + end = (end & ~0xFFF) + 4096 + + real_size = end - start + + if real_size < 0 or real_size > 100000000: + continue + + sections[start] = real_size + + elf_data = b"" + for section_start in sorted(sections.keys()): + read_size = sections[section_start] + + buf = proc_layer.read(vma.vm_start + section_start, read_size, pad=True) + elf_data = elf_data + buf + + file_handle = open_method( + f"pid.{task.pid}.{utility.array_to_string(task.comm)}.{vma.vm_start:#x}.dmp" + ) + file_handle.write(elf_data) + except Exception as e: + vollog.debug(f"Unable to dump ELF with pid {task.pid}: {e}") + + return file_handle + def _generator(self, tasks): + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "linux", "elf", class_types=elf.class_types + ) for task in tasks: proc_layer_name = task.add_process_layer() if not proc_layer_name: @@ -60,6 +152,21 @@ class Elfs(plugins.PluginInterface): path = vma.get_name(self.context, task) + file_output = "Disabled" + if self.config["dump"]: + file_handle = self.elf_dump( + self.context, + proc_layer_name, + elf_table_name, + vma, + task, + self.open, + ) + file_output = "Error outputting file" + if file_handle: + file_handle.close() + file_output = str(file_handle.preferred_filename) + yield ( 0, ( @@ -68,6 +175,7 @@ class Elfs(plugins.PluginInterface): format_hints.Hex(vma.vm_start), format_hints.Hex(vma.vm_end), path, + file_output, ), ) @@ -81,6 +189,7 @@ class Elfs(plugins.PluginInterface): ("Start", format_hints.Hex), ("End", format_hints.Hex), ("File Path", str), + ("File Output", str), ], self._generator( pslist.PsList.list_tasks( diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index af260a772..16e370b6e 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -1,12 +1,15 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Callable, Iterable, List, Any, Tuple +from typing import Any, Callable, Iterable, List -from volatility3.framework import renderers, interfaces +from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.linux.extensions import elf +from volatility3.plugins.linux import elfs class PsList(interfaces.plugins.PluginInterface): @@ -24,6 +27,9 @@ class PsList(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.PluginRequirement( + name="elfs", plugin=elfs.Elfs, version=(2, 0, 0) + ), requirements.ListRequirement( name="pid", description="Filter on specific process IDs", @@ -42,6 +48,12 @@ class PsList(interfaces.plugins.PluginInterface): optional=True, default=False, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed processes", + optional=True, + default=False, + ), ] @classmethod @@ -66,38 +78,12 @@ class PsList(interfaces.plugins.PluginInterface): else: return lambda _: False - def _get_task_fields( - self, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False - ) -> Tuple[int, int, int, str]: - """Extract the fields needed for the final output - - Args: - task: A task object from where to get the fields. - decorate_comm: If True, it decorates the comm string of - - User threads: in curly brackets, - - Kernel threads: in square brackets - Defaults to False. - Returns: - A tuple with the fields to show in the plugin output. - """ - pid = task.tgid - tid = task.pid - ppid = task.parent.tgid if task.parent else 0 - name = utility.array_to_string(task.comm) - if decorate_comm: - if task.is_kernel_thread: - name = f"[{name}]" - elif task.is_user_thread: - name = f"{{{name}}}" - - task_fields = (format_hints.Hex(task.vol.offset), pid, tid, ppid, name) - return task_fields - def _generator( self, pid_filter: Callable[[Any], bool], include_threads: bool = False, decorate_comm: bool = False, + dump: bool = False, ): """Generates the tasks list. @@ -110,14 +96,63 @@ class PsList(interfaces.plugins.PluginInterface): - User threads: in curly brackets, - Kernel threads: in square brackets Defaults to False. + dump: If True, the main executable of the process is written to a file + Defaults to False. Yields: Each rows """ for task in self.list_tasks( self.context, self.config["kernel"], pid_filter, include_threads ): - row = self._get_task_fields(task, decorate_comm) - yield (0, row) + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "linux", + "elf", + class_types=elf.class_types, + ) + file_output = "Disabled" + if dump: + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + continue + + # Find the vma that belongs to the main ELF of the process + file_output = "Error outputting file" + + for v in task.mm.get_mmap_iter(): + if v.vm_start == task.mm.start_code: + file_handle = elfs.Elfs.elf_dump( + self.context, + proc_layer_name, + elf_table_name, + v, + task, + self.open, + ) + if file_handle: + file_output = str(file_handle.preferred_filename) + file_handle.close() + break + + pid = task.tgid + tid = task.pid + ppid = task.parent.tgid if task.parent else 0 + name = utility.array_to_string(task.comm) + if decorate_comm: + if task.is_kernel_thread: + name = f"[{name}]" + elif task.is_user_thread: + name = f"{{{name}}}" + + yield 0, ( + format_hints.Hex(task.vol.offset), + pid, + tid, + ppid, + name, + file_output, + ) @classmethod def list_tasks( @@ -155,6 +190,7 @@ class PsList(interfaces.plugins.PluginInterface): pids = self.config.get("pid") include_threads = self.config.get("threads") decorate_comm = self.config.get("decorate_comm") + dump = self.config.get("dump") filter_func = self.create_pid_filter(pids) columns = [ @@ -163,7 +199,8 @@ class PsList(interfaces.plugins.PluginInterface): ("TID", int), ("PPID", int), ("COMM", str), + ("File output", str), ] return renderers.TreeGrid( - columns, self._generator(filter_func, include_threads, decorate_comm) + columns, self._generator(filter_func, include_threads, decorate_comm, dump) ) From 3386a4fdc0e0c75d3ba9fb03b50a8d99b0cbc57d Mon Sep 17 00:00:00 2001 From: cstation Date: Tue, 14 Mar 2023 22:22:51 +0100 Subject: [PATCH 02/22] Remove broad try-except clause --- volatility3/framework/plugins/linux/elfs.py | 73 ++++++++++----------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index f56438f4f..23bdf1c3a 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -75,55 +75,52 @@ class Elfs(plugins.PluginInterface): proc_layer = context.layers[layer_name] file_handle = None - try: - elf_object = context.object( - elf_table_name + constants.BANG + "Elf", - offset=vma.vm_start, - layer_name=layer_name, - ) + elf_object = context.object( + elf_table_name + constants.BANG + "Elf", + offset=vma.vm_start, + layer_name=layer_name, + ) - if not elf_object.is_valid(): - return None + if not elf_object.is_valid(): + return None - sections = {} - # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? - for phdr in elf_object.get_program_headers(): - if phdr.p_type != 1: # PT_LOAD = 1 - continue + sections = {} + # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? + for phdr in elf_object.get_program_headers(): + if phdr.p_type != 1: # PT_LOAD = 1 + continue - start = phdr.p_vaddr - size = phdr.p_memsz - end = start + size + start = phdr.p_vaddr + size = phdr.p_memsz + end = start + size - # Use complete memory pages for dumping - # If start isn't a multiple of 4096, stick to the highest multiple < start - # If end isn't a multiple of 4096, stick to the lowest multiple > end - if start % 4096: - start = start & ~0xFFF + # Use complete memory pages for dumping + # If start isn't a multiple of 4096, stick to the highest multiple < start + # If end isn't a multiple of 4096, stick to the lowest multiple > end + if start % 4096: + start = start & ~0xFFF - if end % 4096: - end = (end & ~0xFFF) + 4096 + if end % 4096: + end = (end & ~0xFFF) + 4096 - real_size = end - start + real_size = end - start - if real_size < 0 or real_size > 100000000: - continue + if real_size < 0 or real_size > 100000000: + continue - sections[start] = real_size + sections[start] = real_size - elf_data = b"" - for section_start in sorted(sections.keys()): - read_size = sections[section_start] + elf_data = b"" + for section_start in sorted(sections.keys()): + read_size = sections[section_start] - buf = proc_layer.read(vma.vm_start + section_start, read_size, pad=True) - elf_data = elf_data + buf + buf = proc_layer.read(vma.vm_start + section_start, read_size, pad=True) + elf_data = elf_data + buf - file_handle = open_method( - f"pid.{task.pid}.{utility.array_to_string(task.comm)}.{vma.vm_start:#x}.dmp" - ) - file_handle.write(elf_data) - except Exception as e: - vollog.debug(f"Unable to dump ELF with pid {task.pid}: {e}") + file_handle = open_method( + f"pid.{task.pid}.{utility.array_to_string(task.comm)}.{vma.vm_start:#x}.dmp" + ) + file_handle.write(elf_data) return file_handle From 8373b5ed5ac8fe73a67c2d0c8b4f69367d60e9d0 Mon Sep 17 00:00:00 2001 From: cstation Date: Sat, 22 Jul 2023 18:52:39 +0200 Subject: [PATCH 03/22] Push ELF export limit to a constant --- volatility3/framework/constants/linux/__init__.py | 2 ++ volatility3/framework/plugins/linux/elfs.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 1b133eb42..ba7181db8 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -234,3 +234,5 @@ BLUETOOTH_PROTOCOLS = ( "HIDP", "AVDTP", ) + +ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 23bdf1c3a..c6334c977 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -105,8 +105,9 @@ class Elfs(plugins.PluginInterface): real_size = end - start - if real_size < 0 or real_size > 100000000: - continue + # Check if ELF has a legitimate size + if real_size < 0 or real_size > constants.linux.ELF_MAX_EXTRACTION_SIZE: + raise ValueError(f"The claimed size of the ELF is invalid: {real_size}") sections[start] = real_size From d53714fac84ce3c31fef6cd2b5b5dc6201ec5315 Mon Sep 17 00:00:00 2001 From: cstation Date: Sat, 22 Jul 2023 17:03:57 +0000 Subject: [PATCH 04/22] Fix linting --- volatility3/framework/constants/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index e5132f6ce..6e8883f19 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -280,4 +280,4 @@ CAPABILITIES = ( "checkpoint_restore", ) -ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 \ No newline at end of file +ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 From 7c82da4f5044a4bf35028c01924e659ccac5e828 Mon Sep 17 00:00:00 2001 From: xabrouck Date: Mon, 14 Aug 2023 11:53:14 +0200 Subject: [PATCH 05/22] check IoC of dirty bit in PTEs from executable VMAs. this can for example detect code injected using ptrace(). this can also detect injected code that was reset to the original code (malware uninstalled before memory dump happened). --- volatility3/framework/layers/intel.py | 9 +++++++++ volatility3/framework/plugins/linux/malfind.py | 2 +- .../framework/symbols/linux/extensions/__init__.py | 11 ++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 478eb168f..e2d89540d 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -110,6 +110,11 @@ class Intel(linear.LinearlyMappedLayer): def _page_is_valid(entry: int) -> bool: """Returns whether a particular page is valid based on its entry.""" return bool(entry & 1) + + @staticmethod + def _page_is_dirty(entry: int) -> bool: + """Returns whether a particular page is dirty based on its entry.""" + return bool(entry & (1<<6)) def canonicalize(self, addr: int) -> int: """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" @@ -259,6 +264,10 @@ class Intel(linear.LinearlyMappedLayer): except exceptions.InvalidAddressException: return False + def is_dirty(self, offset: int) -> bool: + """Returns whether the page at offset is marked dirty""" + return self._page_is_dirty(self._translate_entry(offset)[0]) + def mapping( self, offset: int, length: int, ignore_errors: bool = False ) -> Iterable[Tuple[int, int, int, int, str]]: diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 1fd005de8..332d5ede1 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -47,7 +47,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.mm.get_vma_iter(): - if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]": + if vma.is_suspicious(proc_layer) and vma.get_name(self.context, task) != "[vdso]": data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 527785a69..d2e6197ef 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -578,7 +578,7 @@ class vm_area_struct(objects.StructType): return fname # used by malfind - def is_suspicious(self): + def is_suspicious(self, proclayer): ret = False flags_str = self.get_protection() @@ -587,6 +587,15 @@ class vm_area_struct(objects.StructType): ret = True elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True + elif "x" in flags_str: + for i in range(self.vm_start,self.vm_end,constants.linux.PAGE_SHIFT): + try: + if proclayer.is_dirty(i): + vollog.warning(f"Found malicious (dirty+exec) page at {hex(i)} !") + ret = True + break + except (exceptions.PagedInvalidAddressException, exceptions.InvalidAddressException): + pass return ret From 804d68d94d507747d42f018baa3255dfe8635b4d Mon Sep 17 00:00:00 2001 From: Eve <120014766+eve-mem@users.noreply.github.com> Date: Thu, 17 Aug 2023 13:38:23 +0100 Subject: [PATCH 06/22] Linux: fix bug where get_process_memory_sections fails with 6.1+ kernels --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 527785a69..a40a87d5c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -203,7 +203,7 @@ class task_struct(generic.GenericIntelProcess): ) -> Generator[Tuple[int, int], None, None]: """Returns a list of sections based on the memory manager's view of this task's virtual memory.""" - for vma in self.mm.get_mmap_iter(): + for vma in self.mm.get_vma_iter(): start = int(vma.vm_start) end = int(vma.vm_end) From 6f7f1284adbbcfacd759ec0d931859e0789cfc8a Mon Sep 17 00:00:00 2001 From: xabrouck Date: Fri, 18 Aug 2023 11:37:04 +0200 Subject: [PATCH 07/22] Fix bug with PAGE_SHIFT that wasn't shifted, also greatly increases performance Use black Better logging --- volatility3/framework/layers/intel.py | 6 +++--- .../framework/plugins/linux/malfind.py | 13 ++++++++++-- .../symbols/linux/extensions/__init__.py | 21 +++++++++++++------ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index e2d89540d..046203fa6 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -110,11 +110,11 @@ class Intel(linear.LinearlyMappedLayer): def _page_is_valid(entry: int) -> bool: """Returns whether a particular page is valid based on its entry.""" return bool(entry & 1) - + @staticmethod def _page_is_dirty(entry: int) -> bool: """Returns whether a particular page is dirty based on its entry.""" - return bool(entry & (1<<6)) + return bool(entry & (1 << 6)) def canonicalize(self, addr: int) -> int: """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" @@ -267,7 +267,7 @@ class Intel(linear.LinearlyMappedLayer): def is_dirty(self, offset: int) -> bool: """Returns whether the page at offset is marked dirty""" return self._page_is_dirty(self._translate_entry(offset)[0]) - + def mapping( self, offset: int, length: int, ignore_errors: bool = False ) -> Iterable[Tuple[int, int, int, int, str]]: diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 332d5ede1..8a21afc03 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -3,7 +3,7 @@ # from typing import List - +import logging from volatility3.framework import constants, interfaces from volatility3.framework import renderers from volatility3.framework.configuration import requirements @@ -11,6 +11,8 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) + class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" @@ -47,7 +49,14 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] for vma in task.mm.get_vma_iter(): - if vma.is_suspicious(proc_layer) and vma.get_name(self.context, task) != "[vdso]": + vma_name = vma.get_name(self.context, task) + vollog.debug( + f"Injections : processing PID {task.pid} : VMA {vma_name} : {hex(vma.vm_start)}-{hex(vma.vm_end)}" + ) + if ( + vma.is_suspicious(proc_layer) + and vma.get_name(self.context, task) != "[vdso]" + ): data = proc_layer.read(vma.vm_start, 64, pad=True) yield vma, data diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d2e6197ef..616e54e70 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -578,7 +578,7 @@ class vm_area_struct(objects.StructType): return fname # used by malfind - def is_suspicious(self, proclayer): + def is_suspicious(self, proclayer=None): ret = False flags_str = self.get_protection() @@ -587,15 +587,24 @@ class vm_area_struct(objects.StructType): ret = True elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True - elif "x" in flags_str: - for i in range(self.vm_start,self.vm_end,constants.linux.PAGE_SHIFT): + elif proclayer and "x" in flags_str: + for i in range(self.vm_start, self.vm_end, 1 << constants.linux.PAGE_SHIFT): try: if proclayer.is_dirty(i): - vollog.warning(f"Found malicious (dirty+exec) page at {hex(i)} !") + vollog.warning( + f"Found malicious (dirty+exec) page at {hex(i)} !" + ) + # We do not attempt to find other dirty+exec pages once we have found one ret = True break - except (exceptions.PagedInvalidAddressException, exceptions.InvalidAddressException): - pass + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ) as excp: + vollog.debug(f"Unable to translate address {hex(i)} : {excp}") + # Abort as it is likely that other addresses in the same range will also fail + ret = False + break return ret From b4c6b661f01fc3dde54362a4f55be4d89e4cc6e5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Sep 2023 21:11:36 +0100 Subject: [PATCH 08/22] Core: Include only volatility3 in distributions packages Fixes #951 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 936a12af2..cfcda3d5c 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setuptools.setup( include_package_data=True, exclude_package_data={"": ["development", "development.*"], "development": ["*"]}, packages=setuptools.find_namespace_packages( - exclude=["development", "development.*"] + include=["volatility3"] ), entry_points={ "console_scripts": [ From da203f7d6828fdeca1fd8f4d85361e65813804eb Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 3 Sep 2023 21:33:32 +0100 Subject: [PATCH 09/22] Documentation: Improve library documentation Fixes #993. --- doc/source/conf.py | 2 +- doc/source/using-as-a-library.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 8b467ec1d..d601c1eee 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -27,7 +27,7 @@ def setup(app): source_dir = os.path.abspath(os.path.dirname(__file__)) sphinx.ext.apidoc.main( - argv=["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory] + ["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory] ) # Go through the volatility3.framework.plugins files and change them to volatility3.plugins diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index fb012f1ae..4acf35f98 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -54,6 +54,12 @@ also be included, which can be found in `volatility3.constants.PLUGINS_PATH`. volatility3.plugins.__path__ = + constants.PLUGINS_PATH failures = framework.import_files(volatility3.plugins, True) +.. note:: + + Volatility uses the `volatility3.plugins` namespace for all plugins (including those in `volatility3.framework.plugins`). + Please ensure you only use `volatility3.plugins` and only ever import plugins from this namespace. + This ensures the ability of users to override core plugins without needing write access to the framework directory. + Once the plugins have been imported, we can interrogate which plugins are available. The :py:func:`~volatility3.framework.list_plugins` call will return a dictionary of plugin names and the plugin classes. From 455487dbb65644ac8c633fdcda6351bbc3011019 Mon Sep 17 00:00:00 2001 From: xabrouck Date: Tue, 5 Sep 2023 16:20:46 +0200 Subject: [PATCH 10/22] Making cred optional as it didn't exist in old 2.6 kernels --- volatility3/framework/symbols/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 5c42a436d..f96302684 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -28,7 +28,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("fs_struct", extensions.fs_struct) self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) - self.set_type_class("cred", extensions.cred) + self.optional_set_type_class("cred", extensions.cred) self.set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) From 05df365936a5965171632c7b0b0dbd1bee6c08a9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 5 Sep 2023 18:23:48 +0100 Subject: [PATCH 11/22] Core: Fix missing packages in setup.py Fixes #1002. --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index cfcda3d5c..44ece8484 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ with open("README.md", "r", encoding="utf-8") as fh: def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -20,6 +20,7 @@ def get_install_requires(): requirements.append(stripped_line) return requirements + setuptools.setup( name="volatility3", description="Memory forensics framework", @@ -39,9 +40,8 @@ setuptools.setup( python_requires=">=3.7.0", include_package_data=True, exclude_package_data={"": ["development", "development.*"], "development": ["*"]}, - packages=setuptools.find_namespace_packages( - include=["volatility3"] - ), + packages=setuptools.find_namespace_packages(where="volatility3"), + package_dir={"": "volatility3"}, entry_points={ "console_scripts": [ "vol = volatility3.cli:main", From 627e2fbab92b30c389be91688fed01f4620c30a0 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 5 Sep 2023 22:27:53 +0200 Subject: [PATCH 12/22] Adding install.yml workflow --- .github/workflows/install.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/install.yml diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml new file mode 100644 index 000000000..e7f9936b6 --- /dev/null +++ b/.github/workflows/install.yml @@ -0,0 +1,29 @@ +name: Test install Volatility3 +on: [push, pull_request] +jobs: + + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.7"] + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup python-pip + run: python -m pip install --upgrade pip + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Install volatility3 + run: pip install . + + - name: Run volatility3 + run: vol --help \ No newline at end of file From 2a67aa639a63b35b2203da07b64be4603d8d8ead Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 5 Sep 2023 22:29:25 +0200 Subject: [PATCH 13/22] Removing Python version --- .github/workflows/install.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index e7f9936b6..51768cc56 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -1,19 +1,14 @@ -name: Test install Volatility3 +name: Install Volatility3 test on: [push, pull_request] jobs: - build: + install: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.7"] steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - name: Setup python-pip run: python -m pip install --upgrade pip From 9982a44131f76987cbd55513a16d3582d27bb505 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 5 Sep 2023 22:36:40 +0200 Subject: [PATCH 14/22] Adding matrix strategy for hosts and python version --- .github/workflows/install.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 51768cc56..9ac14e6c6 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -2,13 +2,20 @@ name: Install Volatility3 test on: [push, pull_request] jobs: - install: - runs-on: ubuntu-latest + install_test: + runs-on: ${{ matrix.host }} + strategy: + matrix: + fail-fast: false + host: [ ubuntu-latest, macOS-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v3 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} - name: Setup python-pip run: python -m pip install --upgrade pip From ef341d54d6edb665b629799a76207f0f4a113f7b Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 5 Sep 2023 22:37:45 +0200 Subject: [PATCH 15/22] Fixing fail-fast strategy --- .github/workflows/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 9ac14e6c6..0a5fec25e 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -5,8 +5,8 @@ jobs: install_test: runs-on: ${{ matrix.host }} strategy: + fail-fast: false matrix: - fail-fast: false host: [ ubuntu-latest, macOS-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: From 9d2fd4051731ac696718cf40bb6e6543c6a1f40f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 5 Sep 2023 23:07:57 +0100 Subject: [PATCH 16/22] Revert "Core: Include only volatility3 in distributions packages" This reverts commit b4c6b661f01fc3dde54362a4f55be4d89e4cc6e5. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cfcda3d5c..936a12af2 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setuptools.setup( include_package_data=True, exclude_package_data={"": ["development", "development.*"], "development": ["*"]}, packages=setuptools.find_namespace_packages( - include=["volatility3"] + exclude=["development", "development.*"] ), entry_points={ "console_scripts": [ From 803c56e3c4c6495b2725b77cc7d045e39c98a9bd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 5 Sep 2023 23:51:17 +0100 Subject: [PATCH 17/22] Core: include the volatility3 package and all volatility3 subpackages --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 44ece8484..c2c55067d 100644 --- a/setup.py +++ b/setup.py @@ -37,11 +37,12 @@ setuptools.setup( "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, + packages=setuptools.find_namespace_packages( + include=["volatility3", "volatility3.*"] + ), + package_dir={"volatility3": "volatility3"}, python_requires=">=3.7.0", include_package_data=True, - exclude_package_data={"": ["development", "development.*"], "development": ["*"]}, - packages=setuptools.find_namespace_packages(where="volatility3"), - package_dir={"": "volatility3"}, entry_points={ "console_scripts": [ "vol = volatility3.cli:main", From 47ddf5d0e5142d6deeb071225ebb2d8bc366d381 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 6 Sep 2023 20:34:13 +0100 Subject: [PATCH 18/22] Core: Bump the version after 2.5.0 release branch --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index de1674885..c3ebaca27 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 370b6774ea3de8fb98bf625ed30b4347d68e2b54 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Sun, 17 Sep 2023 23:21:56 +0200 Subject: [PATCH 19/22] Remove macOS from matrix hosts --- .github/workflows/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 0a5fec25e..cc2a7fd3e 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - host: [ ubuntu-latest, macOS-latest, windows-latest ] + host: [ ubuntu-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v3 From 93b297283292ff7080f539cc483e48c28271dfe8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Oct 2023 22:19:17 +0100 Subject: [PATCH 20/22] Renderers: Allow nodes to be turned into dictionaries --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/renderers/__init__.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index c3ebaca27..09dded076 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 534686022..43bb59a21 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -10,7 +10,7 @@ import collections import collections.abc import datetime import logging -from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar, Union +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, TypeVar, Union from volatility3.framework import interfaces from volatility3.framework.interfaces import renderers @@ -96,6 +96,10 @@ class TreeNode(interfaces.renderers.TreeNode): # if isinstance(val, datetime.datetime): # tznaive = val.tzinfo is None or val.tzinfo.utcoffset(val) is None + def asdict(self) -> Dict[str, Any]: + """Returns the contents of the node as a dictionary""" + return self._values._asdict() + @property def values(self) -> List[interfaces.renderers.BaseTypes]: """Returns the list of values from the particular node, based on column From 310b6508db305b46288e5d0f530ca169ce020726 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 11 Oct 2023 09:09:15 +0100 Subject: [PATCH 21/22] vmware: Add warning when no metadata file is found for a vmem file. --- volatility3/framework/layers/vmware.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 0bc1a350b..5467c70c0 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -232,6 +232,9 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): ) if not vmss_success and not vmsn_success: + vollog.warning( + f"No metadata file alongside VMEM file! A VMSS or VMSN file is required to correctly process a VMEM file. These should be placed in the same directory with the same file name, e.g. sample.vmem and sample.vmsn.", + ) return None new_layer_name = context.layers.free_layer_name("VmwareLayer") context.config[ From e45380f4f579e3c54f1475312efc2aff47341b2a Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 13 Oct 2023 09:29:46 +0100 Subject: [PATCH 22/22] Update vmware layer metadata file warning to be less doom and gloom. --- volatility3/framework/layers/vmware.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 5467c70c0..622ff0250 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -4,6 +4,7 @@ import contextlib import logging import struct +import os from typing import Any, Dict, List, Optional from volatility3.framework import constants, exceptions, interfaces @@ -232,8 +233,10 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): ) if not vmss_success and not vmsn_success: + vmem_file_basename = os.path.basename(location) + example_vmss_file_basename = os.path.basename(vmss) vollog.warning( - f"No metadata file alongside VMEM file! A VMSS or VMSN file is required to correctly process a VMEM file. These should be placed in the same directory with the same file name, e.g. sample.vmem and sample.vmsn.", + f"No metadata file found alongside VMEM file. A VMSS or VMSN file may be required to correctly process a VMEM file. These should be placed in the same directory with the same file name, e.g. {vmem_file_basename} and {example_vmss_file_basename}.", ) return None new_layer_name = context.layers.free_layer_name("VmwareLayer")