Merge branch 'volatilityfoundation:develop' into fix-issue-895

.github/workflows/install.yml

@@ -0,0 +1,31 @@
+name: Install Volatility3 test
+on: [push, pull_request]
+jobs:
+
+  install_test:
+    runs-on: ${{ matrix.host }}
+    strategy:
+      fail-fast: false
+      matrix:
+        host: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Setup python-pip
+      run: python -m pip install --upgrade pip
+
+    - name: Install dependencies
+      run: |
+        pip install -r requirements.txt
+
+    - name: Install volatility3
+      run: pip install .
+
+    - name: Run volatility3
+      run: vol --help

doc/source/conf.py

@@ -27,7 +27,7 @@ def setup(app):
 
     source_dir = os.path.abspath(os.path.dirname(__file__))
     sphinx.ext.apidoc.main(
-        argv=["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory]
+        ["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory]
     )
 
     # Go through the volatility3.framework.plugins files and change them to volatility3.plugins

doc/source/using-as-a-library.rst

@@ -54,6 +54,12 @@ also be included, which can be found in `volatility3.constants.PLUGINS_PATH`.
         volatility3.plugins.__path__ = <new_plugin_path> + constants.PLUGINS_PATH
         failures = framework.import_files(volatility3.plugins, True)
 
+.. note::
+
+    Volatility uses the `volatility3.plugins` namespace for all plugins (including those in `volatility3.framework.plugins`).
+    Please ensure you only use `volatility3.plugins` and only ever import plugins from this namespace.
+    This ensures the ability of users to override core plugins without needing write access to the framework directory.
+
 Once the plugins have been imported, we can interrogate which plugins are available.  The
 :py:func:`~volatility3.framework.list_plugins` call will
 return a dictionary of plugin names and the plugin classes.

setup.py

@@ -12,7 +12,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 def get_install_requires():
     requirements = []
-    with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh:
+    with open("requirements-minimal.txt", "r", encoding="utf-8") as fh:
         for line in fh.readlines():
             stripped_line = line.strip()
             if stripped_line == "" or stripped_line.startswith("#"):
@@ -20,6 +20,7 @@ def get_install_requires():
             requirements.append(stripped_line)
     return requirements
 
+
 setuptools.setup(
     name="volatility3",
     description="Memory forensics framework",
@@ -36,12 +37,12 @@ setuptools.setup(
         "Documentation": "https://volatility3.readthedocs.io/",
         "Source Code": "https://github.com/volatilityfoundation/volatility3",
     },
+    packages=setuptools.find_namespace_packages(
+        include=["volatility3", "volatility3.*"]
+    ),
+    package_dir={"volatility3": "volatility3"},
     python_requires=">=3.7.0",
     include_package_data=True,
-    exclude_package_data={"": ["development", "development.*"], "development": ["*"]},
-    packages=setuptools.find_namespace_packages(
-        exclude=["development", "development.*"]
-    ),
     entry_points={
         "console_scripts": [
             "vol = volatility3.cli:main",

volatility3/framework/constants/__init__.py

@@ -45,7 +45,7 @@ BANG = "!"
 # We use the SemVer 2.0.0 versioning scheme
 VERSION_MAJOR = 2  # Number of releases of the library with a breaking change
 VERSION_MINOR = 5  # Number of changes that only add to the interface
-VERSION_PATCH = 0  # Number of changes that do not change the interface
+VERSION_PATCH = 2  # Number of changes that do not change the interface
 VERSION_SUFFIX = ""
 
 # TODO: At version 2.0.0, remove the symbol_shift feature

volatility3/framework/constants/linux/__init__.py

@@ -279,3 +279,5 @@ CAPABILITIES = (
     "bpf",
     "checkpoint_restore",
 )
+
+ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1

volatility3/framework/layers/intel.py

@@ -111,6 +111,11 @@ class Intel(linear.LinearlyMappedLayer):
         """Returns whether a particular page is valid based on its entry."""
         return bool(entry & 1)
 
+    @staticmethod
+    def _page_is_dirty(entry: int) -> bool:
+        """Returns whether a particular page is dirty based on its entry."""
+        return bool(entry & (1 << 6))
+
     def canonicalize(self, addr: int) -> int:
         """Canonicalizes an address by performing an appropiate sign extension on the higher addresses"""
         if self._bits_per_register <= self._maxvirtaddr:
@@ -259,6 +264,10 @@ class Intel(linear.LinearlyMappedLayer):
         except exceptions.InvalidAddressException:
             return False
 
+    def is_dirty(self, offset: int) -> bool:
+        """Returns whether the page at offset is marked dirty"""
+        return self._page_is_dirty(self._translate_entry(offset)[0])
+
     def mapping(
         self, offset: int, length: int, ignore_errors: bool = False
     ) -> Iterable[Tuple[int, int, int, int, str]]:

volatility3/framework/layers/vmware.py

@@ -4,6 +4,7 @@
 import contextlib
 import logging
 import struct
+import os
 from typing import Any, Dict, List, Optional
 
 from volatility3.framework import constants, exceptions, interfaces
@@ -232,6 +233,11 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface):
             )
 
             if not vmss_success and not vmsn_success:
+                vmem_file_basename = os.path.basename(location)
+                example_vmss_file_basename = os.path.basename(vmss)
+                vollog.warning(
+                    f"No metadata file found alongside VMEM file. A VMSS or VMSN file may be required to correctly process a VMEM file. These should be placed in the same directory with the same file name, e.g. {vmem_file_basename} and {example_vmss_file_basename}.",
+                )
                 return None
             new_layer_name = context.layers.free_layer_name("VmwareLayer")
             context.config[

volatility3/framework/plugins/linux/elfs.py

@@ -4,20 +4,26 @@
 """A module containing a collection of plugins that produce data typically
 found in Linux's /proc file system."""
 
-from typing import List
+import logging
+from typing import List, Optional, Type
 
-from volatility3.framework import renderers, interfaces
+from volatility3.framework import constants, interfaces, renderers
 from volatility3.framework.configuration import requirements
 from volatility3.framework.interfaces import plugins
 from volatility3.framework.objects import utility
 from volatility3.framework.renderers import format_hints
+from volatility3.framework.symbols import intermed
+from volatility3.framework.symbols.linux.extensions import elf
 from volatility3.plugins.linux import pslist
 
+vollog = logging.getLogger(__name__)
+
 
 class Elfs(plugins.PluginInterface):
     """Lists all memory mapped ELF files for all processes."""
 
     _required_framework_version = (2, 0, 0)
+    _version = (2, 0, 0)
 
     @classmethod
     def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
@@ -36,9 +42,93 @@ class Elfs(plugins.PluginInterface):
                 element_type=int,
                 optional=True,
             ),
+            requirements.BooleanRequirement(
+                name="dump",
+                description="Extract listed processes",
+                default=False,
+                optional=True,
+            ),
         ]
 
+    @classmethod
+    def elf_dump(
+        cls,
+        context: interfaces.context.ContextInterface,
+        layer_name: str,
+        elf_table_name: str,
+        vma: interfaces.objects.ObjectInterface,
+        task: interfaces.objects.ObjectInterface,
+        open_method: Type[interfaces.plugins.FileHandlerInterface],
+    ) -> Optional[interfaces.plugins.FileHandlerInterface]:
+        """Extracts an ELF as a FileHandlerInterface
+        Args:
+            context: the context to operate upon
+            layer_name: The name of the layer on which to operate
+            elf_table_name: the name for the symbol table containing the symbols for ELF-files
+            vma: virtual memory allocation of ELF
+            task: the task object whose memory should be output
+            open_method: class to provide context manager for opening the file
+        Returns:
+            An open FileHandlerInterface object containing the complete data for the task or None in the case of failure
+        """
+
+        proc_layer = context.layers[layer_name]
+        file_handle = None
+
+        elf_object = context.object(
+            elf_table_name + constants.BANG + "Elf",
+            offset=vma.vm_start,
+            layer_name=layer_name,
+        )
+
+        if not elf_object.is_valid():
+            return None
+
+        sections = {}
+        # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ?
+        for phdr in elf_object.get_program_headers():
+            if phdr.p_type != 1:  # PT_LOAD = 1
+                continue
+
+            start = phdr.p_vaddr
+            size = phdr.p_memsz
+            end = start + size
+
+            # Use complete memory pages for dumping
+            # If start isn't a multiple of 4096, stick to the highest multiple < start
+            # If end isn't a multiple of 4096, stick to the lowest multiple > end
+            if start % 4096:
+                start = start & ~0xFFF
+
+            if end % 4096:
+                end = (end & ~0xFFF) + 4096
+
+            real_size = end - start
+
+            # Check if ELF has a legitimate size
+            if real_size < 0 or real_size > constants.linux.ELF_MAX_EXTRACTION_SIZE:
+                raise ValueError(f"The claimed size of the ELF is invalid: {real_size}")
+
+            sections[start] = real_size
+
+        elf_data = b""
+        for section_start in sorted(sections.keys()):
+            read_size = sections[section_start]
+
+            buf = proc_layer.read(vma.vm_start + section_start, read_size, pad=True)
+            elf_data = elf_data + buf
+
+        file_handle = open_method(
+            f"pid.{task.pid}.{utility.array_to_string(task.comm)}.{vma.vm_start:#x}.dmp"
+        )
+        file_handle.write(elf_data)
+
+        return file_handle
+
     def _generator(self, tasks):
+        elf_table_name = intermed.IntermediateSymbolTable.create(
+            self.context, self.config_path, "linux", "elf", class_types=elf.class_types
+        )
         for task in tasks:
             proc_layer_name = task.add_process_layer()
             if not proc_layer_name:
@@ -60,6 +150,21 @@ class Elfs(plugins.PluginInterface):
 
                 path = vma.get_name(self.context, task)
 
+                file_output = "Disabled"
+                if self.config["dump"]:
+                    file_handle = self.elf_dump(
+                        self.context,
+                        proc_layer_name,
+                        elf_table_name,
+                        vma,
+                        task,
+                        self.open,
+                    )
+                    file_output = "Error outputting file"
+                    if file_handle:
+                        file_handle.close()
+                        file_output = str(file_handle.preferred_filename)
+
                 yield (
                     0,
                     (
@@ -68,6 +173,7 @@ class Elfs(plugins.PluginInterface):
                         format_hints.Hex(vma.vm_start),
                         format_hints.Hex(vma.vm_end),
                         path,
+                        file_output,
                     ),
                 )
 
@@ -81,6 +187,7 @@ class Elfs(plugins.PluginInterface):
                 ("Start", format_hints.Hex),
                 ("End", format_hints.Hex),
                 ("File Path", str),
+                ("File Output", str),
             ],
             self._generator(
                 pslist.PsList.list_tasks(

volatility3/framework/plugins/linux/malfind.py

@@ -3,7 +3,7 @@
 #
 
 from typing import List
-
+import logging
 from volatility3.framework import constants, interfaces
 from volatility3.framework import renderers
 from volatility3.framework.configuration import requirements
@@ -11,6 +11,8 @@ from volatility3.framework.objects import utility
 from volatility3.framework.renderers import format_hints
 from volatility3.plugins.linux import pslist
 
+vollog = logging.getLogger(__name__)
+
 
 class Malfind(interfaces.plugins.PluginInterface):
     """Lists process memory ranges that potentially contain injected code."""
@@ -47,7 +49,14 @@ class Malfind(interfaces.plugins.PluginInterface):
         proc_layer = self.context.layers[proc_layer_name]
 
         for vma in task.mm.get_vma_iter():
-            if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]":
+            vma_name = vma.get_name(self.context, task)
+            vollog.debug(
+                f"Injections : processing PID {task.pid} : VMA {vma_name} : {hex(vma.vm_start)}-{hex(vma.vm_end)}"
+            )
+            if (
+                vma.is_suspicious(proc_layer)
+                and vma.get_name(self.context, task) != "[vdso]"
+            ):
                 data = proc_layer.read(vma.vm_start, 64, pad=True)
                 yield vma, data
 

volatility3/framework/plugins/linux/pslist.py

@@ -1,12 +1,15 @@
 # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0
 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
 #
-from typing import Callable, Iterable, List, Any, Tuple
+from typing import Any, Callable, Iterable, List
 
-from volatility3.framework import renderers, interfaces
+from volatility3.framework import interfaces, renderers
 from volatility3.framework.configuration import requirements
 from volatility3.framework.objects import utility
 from volatility3.framework.renderers import format_hints
+from volatility3.framework.symbols import intermed
+from volatility3.framework.symbols.linux.extensions import elf
+from volatility3.plugins.linux import elfs
 
 
 class PsList(interfaces.plugins.PluginInterface):
@@ -24,6 +27,9 @@ class PsList(interfaces.plugins.PluginInterface):
                 description="Linux kernel",
                 architectures=["Intel32", "Intel64"],
             ),
+            requirements.PluginRequirement(
+                name="elfs", plugin=elfs.Elfs, version=(2, 0, 0)
+            ),
             requirements.ListRequirement(
                 name="pid",
                 description="Filter on specific process IDs",
@@ -42,6 +48,12 @@ class PsList(interfaces.plugins.PluginInterface):
                 optional=True,
                 default=False,
             ),
+            requirements.BooleanRequirement(
+                name="dump",
+                description="Extract listed processes",
+                optional=True,
+                default=False,
+            ),
         ]
 
     @classmethod
@@ -66,38 +78,12 @@ class PsList(interfaces.plugins.PluginInterface):
         else:
             return lambda _: False
 
-    def _get_task_fields(
-        self, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False
-    ) -> Tuple[int, int, int, str]:
-        """Extract the fields needed for the final output
-
-        Args:
-            task: A task object from where to get the fields.
-            decorate_comm: If True, it decorates the comm string of
-                            - User threads: in curly brackets,
-                            - Kernel threads: in square brackets
-                           Defaults to False.
-        Returns:
-            A tuple with the fields to show in the plugin output.
-        """
-        pid = task.tgid
-        tid = task.pid
-        ppid = task.parent.tgid if task.parent else 0
-        name = utility.array_to_string(task.comm)
-        if decorate_comm:
-            if task.is_kernel_thread:
-                name = f"[{name}]"
-            elif task.is_user_thread:
-                name = f"{{{name}}}"
-
-        task_fields = (format_hints.Hex(task.vol.offset), pid, tid, ppid, name)
-        return task_fields
-
     def _generator(
         self,
         pid_filter: Callable[[Any], bool],
         include_threads: bool = False,
         decorate_comm: bool = False,
+        dump: bool = False,
     ):
         """Generates the tasks list.
 
@@ -110,14 +96,63 @@ class PsList(interfaces.plugins.PluginInterface):
                             - User threads: in curly brackets,
                             - Kernel threads: in square brackets
                            Defaults to False.
+            dump: If True, the main executable of the process is written to a file
+                  Defaults to False.
         Yields:
             Each rows
         """
         for task in self.list_tasks(
             self.context, self.config["kernel"], pid_filter, include_threads
         ):
-            row = self._get_task_fields(task, decorate_comm)
-            yield (0, row)
+            elf_table_name = intermed.IntermediateSymbolTable.create(
+                self.context,
+                self.config_path,
+                "linux",
+                "elf",
+                class_types=elf.class_types,
+            )
+            file_output = "Disabled"
+            if dump:
+                proc_layer_name = task.add_process_layer()
+                if not proc_layer_name:
+                    continue
+
+                # Find the vma that belongs to the main ELF of the process
+                file_output = "Error outputting file"
+
+                for v in task.mm.get_mmap_iter():
+                    if v.vm_start == task.mm.start_code:
+                        file_handle = elfs.Elfs.elf_dump(
+                            self.context,
+                            proc_layer_name,
+                            elf_table_name,
+                            v,
+                            task,
+                            self.open,
+                        )
+                        if file_handle:
+                            file_output = str(file_handle.preferred_filename)
+                            file_handle.close()
+                        break
+
+            pid = task.tgid
+            tid = task.pid
+            ppid = task.parent.tgid if task.parent else 0
+            name = utility.array_to_string(task.comm)
+            if decorate_comm:
+                if task.is_kernel_thread:
+                    name = f"[{name}]"
+                elif task.is_user_thread:
+                    name = f"{{{name}}}"
+
+            yield 0, (
+                format_hints.Hex(task.vol.offset),
+                pid,
+                tid,
+                ppid,
+                name,
+                file_output,
+            )
 
     @classmethod
     def list_tasks(
@@ -155,6 +190,7 @@ class PsList(interfaces.plugins.PluginInterface):
         pids = self.config.get("pid")
         include_threads = self.config.get("threads")
         decorate_comm = self.config.get("decorate_comm")
+        dump = self.config.get("dump")
         filter_func = self.create_pid_filter(pids)
 
         columns = [
@@ -163,7 +199,8 @@ class PsList(interfaces.plugins.PluginInterface):
             ("TID", int),
             ("PPID", int),
             ("COMM", str),
+            ("File output", str),
         ]
         return renderers.TreeGrid(
-            columns, self._generator(filter_func, include_threads, decorate_comm)
+            columns, self._generator(filter_func, include_threads, decorate_comm, dump)
         )

volatility3/framework/renderers/__init__.py

@@ -10,7 +10,7 @@ import collections
 import collections.abc
 import datetime
 import logging
-from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar, Union
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, TypeVar, Union
 
 from volatility3.framework import interfaces
 from volatility3.framework.interfaces import renderers
@@ -96,6 +96,10 @@ class TreeNode(interfaces.renderers.TreeNode):
             # if isinstance(val, datetime.datetime):
             #     tznaive = val.tzinfo is None or val.tzinfo.utcoffset(val) is None
 
+    def asdict(self) -> Dict[str, Any]:
+        """Returns the contents of the node as a dictionary"""
+        return self._values._asdict()
+
     @property
     def values(self) -> List[interfaces.renderers.BaseTypes]:
         """Returns the list of values from the particular node, based on column

volatility3/framework/symbols/linux/__init__.py

@@ -28,7 +28,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable):
         self.set_type_class("fs_struct", extensions.fs_struct)
         self.set_type_class("files_struct", extensions.files_struct)
         self.set_type_class("kobject", extensions.kobject)
-        self.set_type_class("cred", extensions.cred)
+        self.optional_set_type_class("cred", extensions.cred)
         self.set_type_class("kernel_cap_struct", extensions.kernel_cap_struct)
         # Might not exist in the current symbols
         self.optional_set_type_class("module", extensions.module)

volatility3/framework/symbols/linux/extensions/__init__.py

@@ -203,7 +203,7 @@ class task_struct(generic.GenericIntelProcess):
     ) -> Generator[Tuple[int, int], None, None]:
         """Returns a list of sections based on the memory manager's view of
         this task's virtual memory."""
-        for vma in self.mm.get_mmap_iter():
+        for vma in self.mm.get_vma_iter():
             start = int(vma.vm_start)
             end = int(vma.vm_end)
 
@@ -578,7 +578,7 @@ class vm_area_struct(objects.StructType):
         return fname
 
     # used by malfind
-    def is_suspicious(self):
+    def is_suspicious(self, proclayer=None):
         ret = False
 
         flags_str = self.get_protection()
@@ -587,6 +587,24 @@ class vm_area_struct(objects.StructType):
             ret = True
         elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0:
             ret = True
+        elif proclayer and "x" in flags_str:
+            for i in range(self.vm_start, self.vm_end, 1 << constants.linux.PAGE_SHIFT):
+                try:
+                    if proclayer.is_dirty(i):
+                        vollog.warning(
+                            f"Found malicious (dirty+exec) page at {hex(i)} !"
+                        )
+                        # We do not attempt to find other dirty+exec pages once we have found one
+                        ret = True
+                        break
+                except (
+                    exceptions.PagedInvalidAddressException,
+                    exceptions.InvalidAddressException,
+                ) as excp:
+                    vollog.debug(f"Unable to translate address {hex(i)} : {excp}")
+                    # Abort as it is likely that other addresses in the same range will also fail
+                    ret = False
+                    break
         return ret