From 9f145ddcf0ddb84cf7ae45585ce7e9da0d204a17 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:04:33 +0100 Subject: [PATCH 01/51] pillow dependency --- requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requirements.txt b/requirements.txt index e0d366391..21c8f9a76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,9 @@ leechcorepyc>=2.4.0; sys_platform != 'darwin' # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage gcsfs>=2023.1.0 s3fs>=2023.1.0 + +# This is required by plugins that manipulate pixels and images. +# https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst +# 10.0.0 dropped support for Python3.7 +# 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 +pillow>=10.0.0,<11.0.0 \ No newline at end of file From 36a18f405b8ba7605ca957ca47d540a5b7c520d7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:05:23 +0100 Subject: [PATCH 02/51] fourcc code converter helper --- volatility3/framework/symbols/linux/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..573bc66d8 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -483,6 +483,22 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return kernel + @classmethod + def convert_fourcc_code(cls, code: int) -> str: + """Convert a fourcc integer back to its fourcc string representation. + + Args: + code: the numerical representation of the fourcc + + Returns: + The fourcc code string. + """ + + code_bytes_length = (code.bit_length() + 7) // 8 + return "".join( + [chr((code >> (i * 8)) & 0xFF) for i in range(code_bytes_length)] + ) + class IDStorage(ABC): """Abstraction to support both XArray and RadixTree""" From a7620cd6f659dfa685a445536a240182225a778c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:09:21 +0100 Subject: [PATCH 03/51] linux fbdev subsystem api plugin --- .../framework/plugins/linux/graphics/fbdev.py | 314 ++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 volatility3/framework/plugins/linux/graphics/fbdev.py diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py new file mode 100644 index 000000000..4bde6f62f --- /dev/null +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -0,0 +1,314 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +import io + +# Image manipulation functions are kept in the plugin, +# to prevent a general exit on missing PIL (pillow) dependency. +from PIL import Image +from dataclasses import dataclass +from typing import Type, List, Dict, Tuple +from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.objects import utility +from volatility3.framework.constants import architectures +from volatility3.framework.symbols import linux + +vollog = logging.getLogger(__name__) + + +@dataclass +class Framebuffer: + """Framebuffer object internal representation. This is useful to unify an framebuffer with precalculated + properties and pass it through functions conveniently.""" + + id: str + xres_virtual: int + yres_virtual: int + line_length: int + bpp: int + """Bits Per Pixel""" + size: int + color_fields: Dict[str, Tuple[int, int, int]] + fb_info: interfaces.objects.ObjectInterface + + +class Fbdev(interfaces.plugins.PluginInterface): + """Extract framebuffers from the fbdev graphics subsystem""" + + _version = (1, 0, 0) + _required_framework_version = (2, 11, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.BooleanRequirement( + name="dump", + description="Dump framebuffers", + default=False, + optional=True, + ), + ] + + @classmethod + def parse_fb_pixel_bitfields( + cls, fb_var_screeninfo: interfaces.objects.ObjectInterface + ) -> Dict[str, Tuple[int, int, int]]: + """Organize a framebuffer pixel format into a dictionary. + This is needed to know the position and bitlength of a color inside + a pixel. + + Args: + fb_var_screeninfo: a fb_var_screeninfo kernel object instance + + Returns: + The color fields mappings + + Documentation: + include/uapi/linux/fb.h: + struct fb_bitfield { + __u32 offset; /* beginning of bitfield */ + __u32 length; /* length of bitfield */ + __u32 msb_right; /* != 0 : Most significant bit is right */ + }; + """ + # Naturally order by RGBA + color_mappings = [ + ("R", fb_var_screeninfo.red), + ("G", fb_var_screeninfo.green), + ("B", fb_var_screeninfo.blue), + ("A", fb_var_screeninfo.transp), + ] + color_fields = {} + for color_code, fb_bitfield in color_mappings: + color_fields[color_code] = ( + int(fb_bitfield.offset), + int(fb_bitfield.length), + int(fb_bitfield.msb_right), + ) + return color_fields + + @classmethod + def convert_fb_raw_buffer_to_image( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + fb: Framebuffer, + ) -> Image.Image: + """Convert raw framebuffer pixels to an image. + + Args: + fb: the relevant Framebuffer object + + Returns: + A PIL Image object + + Documentation: + include/uapi/linux/fb.h: + /* Interpretation of offset for color fields: All offsets are from the right, + * inside a "pixel" value, which is exactly 'bits_per_pixel' wide (means: you + * can use the offset as right argument to <<). A pixel afterwards is a bit + * stream and is written to video memory as that unmodified. + """ + kernel = context.modules[kernel_name] + kernel_layer = context.layers[kernel.layer_name] + + raw_pixels = io.BytesIO(kernel_layer.read(fb.fb_info.screen_base, fb.size)) + bytes_per_pixel = fb.bpp // 8 + image = Image.new("RGBA", (fb.xres_virtual, fb.yres_virtual)) + + # This is not designed to be extremely fast (numpy isn't available), + # but convenient and dynamic for any color field layout. + for y in range(fb.yres_virtual): + for x in range(fb.xres_virtual): + raw_pixel = int.from_bytes(raw_pixels.read(bytes_per_pixel), "little") + pixel = [0, 0, 0, 255] + # The framebuffer is expected to have been correctly constructed, + # especially by parse_fb_pixel_bitfields, to get the needed RGBA mappings. + for i, color_code in enumerate(["R", "G", "B", "A"]): + offset, length, msb_right = fb.color_fields[color_code] + if length == 0: + continue + color_value = (raw_pixel >> offset) & (2**length - 1) + if msb_right: + # Reverse bit order + color_value = int( + "{:0{length}b}".format(color_value, length=length)[::-1], 2 + ) + pixel[i] = color_value + image.putpixel((x, y), tuple(pixel)) + + return image + + @classmethod + def dump_fb( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + fb: Framebuffer, + convert_to_image: bool, + image_format: str = "PNG", + ) -> str: + """Dump a Framebuffer raw buffer to disk. + + Args: + fb: the relevant Framebuffer object + convert_to_image: a boolean specifying if the buffer should be converted to an image + image_format: the target PIL image format (defaults to PNG) + + Returns: + The filename of the dumped buffer. + """ + kernel = context.modules[kernel_name] + kernel_layer = context.layers[kernel.layer_name] + base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" + if convert_to_image: + image = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) + output = io.BytesIO() + image.save(output, image_format) + file_handle = open_method(f"{base_filename}.{image_format.lower()}") + file_handle.write(output.getvalue()) + else: + raw_pixels = kernel_layer.read(fb.fb_info.screen_base, fb.size) + file_handle = open_method(f"{base_filename}.raw") + file_handle.write(raw_pixels) + + file_handle.close() + return file_handle.preferred_filename + + @classmethod + def parse_fb_info( + cls, + fb_info: interfaces.objects.ObjectInterface, + ) -> Framebuffer: + """Parse an fb_info struct + Args: + fb_info: an fb_info kernel object live instance + + Returns: + A Framebuffer object + + Documentation: + https://docs.kernel.org/fb/api.html: + - struct fb_fix_screeninfo stores device independent unchangeable information about the frame buffer device and the current format. + Those information can't be directly modified by applications, but can be changed by the driver when an application modifies the format. + - struct fb_var_screeninfo stores device independent changeable information about a frame buffer device, its current format and video mode, + as well as other miscellaneous parameters. + """ + # NotAvailableValue() messes with the filename output on disk + id = utility.array_to_string(fb_info.fix.id) or "N-A" + color_fields = None + + # 0 = color, 1 = grayscale, >1 = FOURCC + if fb_info.var.grayscale in [0, 1]: + color_fields = cls.parse_fb_pixel_bitfields(fb_info.var) + + # There a lot of tricky pixel formats used by drivers and vendors in include/uapi/linux/videodev2.h. + # As Volatility3 is not a video format converter, it is best to play it safe and let the user parse + # the raw data manually (with ffmpeg for example). + elif fb_info.var.grayscale > 1: + fourcc = linux.LinuxUtilities.convert_fourcc_code(fb_info.var.grayscale) + warn_msg = f"""Framebuffer "{id}" uses a FOURCC pixel format "{fourcc}" that isn't natively supported. +You can try using ffmpeg to decode the raw buffer. Example usage: +"ffmpeg -pix_fmts" to list supported formats, then +"ffmpeg -f rawvideo -video_size {fb_info.var.xres_virtual}x{fb_info.var.yres_virtual} -i .raw -pix_fmt output.png".""" + vollog.warning(warn_msg) + + # Prefer using the virtual resolution, instead of the visible one. + # This prevents missing non-visible data stored in the framebuffer. + fb = Framebuffer( + id, + xres_virtual=fb_info.var.xres_virtual, + yres_virtual=fb_info.var.yres_virtual, + line_length=fb_info.fix.line_length, + bpp=fb_info.var.bits_per_pixel, + size=fb_info.var.yres_virtual * fb_info.fix.line_length, + color_fields=color_fields, + fb_info=fb_info, + ) + + return fb + + def _generator(self): + kernel_name = self.config["kernel"] + kernel = self.context.modules[kernel_name] + + if not kernel.has_symbol("num_registered_fb"): + raise exceptions.SymbolError( + "num_registered_fb", + kernel.symbol_table_name, + "The provided symbol does not exist in the symbol table. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.", + ) + + num_registered_fb = kernel.object_from_symbol("num_registered_fb") + if num_registered_fb < 1: + vollog.info("No registered framebuffer in the fbdev API.") + return None + + registered_fb = kernel.object_from_symbol("registered_fb") + fb_info_list = utility.array_of_pointers( + registered_fb, + num_registered_fb, + kernel.symbol_table_name + constants.BANG + "fb_info", + self.context, + ) + + for fb_info in fb_info_list: + fb = self.parse_fb_info(fb_info) + file_output = "Disabled" + if self.config["dump"]: + try: + file_output = self.dump_fb( + self.context, kernel_name, self.open, fb, bool(fb.color_fields) + ) + except exceptions.InvalidAddressException as excp: + vollog.error( + f'Layer {excp.layer_name} failed to read address {hex(excp.invalid_address)} when dumping framebuffer "{fb.id}".' + ) + file_output = "Error" + + try: + fb_device_name = utility.pointer_to_string( + fb.fb_info.dev.kobj.name, 256 + ) + except exceptions.InvalidAddressException: + fb_device_name = NotAvailableValue() + + yield ( + 0, + ( + format_hints.Hex(fb.fb_info.screen_base), + fb_device_name, + fb.id, + fb.size, + f"{fb.xres_virtual}x{fb.yres_virtual}", + fb.bpp, + "RUNNING" if fb.fb_info.state == 0 else "SUSPENDED", + str(file_output), + ), + ) + + def run(self): + columns = [ + ("Address", format_hints.Hex), + ("Device", str), + ("ID", str), + ("Size", int), + ("Virtual resolution", str), + ("BPP", int), + ("State", str), + ("Filename", str), + ] + + return TreeGrid( + columns, + self._generator(), + ) From f192437a944f56ec3b8eae186d61f3049e691e80 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:11:11 +0100 Subject: [PATCH 04/51] typo --- volatility3/framework/plugins/linux/graphics/fbdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 4bde6f62f..60e00d033 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -21,7 +21,7 @@ vollog = logging.getLogger(__name__) @dataclass class Framebuffer: - """Framebuffer object internal representation. This is useful to unify an framebuffer with precalculated + """Framebuffer object internal representation. This is useful to unify a framebuffer with precalculated properties and pass it through functions conveniently.""" id: str From c45beb3ebe7feaec42567eb8ef3dd665e15db3ae Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:12:19 +0000 Subject: [PATCH 05/51] Automagic: Fixes #1417 --- volatility3/framework/automagic/symbol_cache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index e38771f79..2c9883c7d 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -299,6 +299,13 @@ class SqliteCache(CacheManagerInterface): This also updates remote locations based on a cache timeout. """ + if progress_callback is None: + + def dummy_progress(*args, **kargs) -> None: + return None + + progress_callback = dummy_progress + on_disk_locations = set( [ filename From a11131b3d4767ec15619bab8084c4914ad528f75 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 14:59:58 +0000 Subject: [PATCH 06/51] Update how to write a simple plugin --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 39670a62d..84b921114 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -52,7 +52,7 @@ to be able to run properly. Any that are defined as optional need not necessari version = (2, 0, 0))] -This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how +This is a classmethod, so it can be called before the specific plugin object has been instantiated (in order to know how to instantiate the plugin). At the moment these requirements are fairly straightforward: :: From 675ef6deea00cf4659bca50e0004dea3be43e8c1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:01:09 +0000 Subject: [PATCH 07/51] Generic: Fix up potential issue with isfinfo Fixes #1436 --- volatility3/framework/plugins/isfinfo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 78e78fb9e..1c2ac52e9 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -132,6 +132,7 @@ class IsfInfo(plugins.PluginInterface): valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {entry}") + continue yield ( 0, ( From f6f4c7b986c8c2adf7a73211f7aeb1375b00f255 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:11:42 +0000 Subject: [PATCH 08/51] Layers: Fix MSF page len on a possibly uninitialized variable Fixes #1441 --- volatility3/framework/layers/msf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 03e144e25..2b4fae963 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -194,7 +194,7 @@ class PdbMSFStream(linear.LinearlyMappedLayer): ) -> None: super().__init__(context, config_path, name, metadata) self._base_layer = self.config["base_layer"] - self._pages = self.config.get("pages", None) + self._pages = self.config.get("pages", []) self._pages_len = len(self._pages) if not self._pages: raise PDBFormatException(name, "Invalid/no pages specified") From eaca7b31bba1c9d8f934ad3e607fced26797abfd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:16:17 +0000 Subject: [PATCH 09/51] Layers: Fix vmware layer without a suitable meta Fixes #1442 --- volatility3/framework/layers/vmware.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 622ff0250..39fb21b63 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -57,6 +57,10 @@ class VmwareLayer(segmented.SegmentedLayer): ) meta_layer = self.context.layers.get(self._meta_layer, None) + if meta_layer is None: + raise exceptions.LayerException( + self._meta_layer, "VMware: Meta layer not found" + ) header_size = struct.calcsize(self.header_structure) data = meta_layer.read(0, header_size) magic, unknown, groupCount = struct.unpack(self.header_structure, data) From 88eb2aa88648180439e4d5311c9bf49ef2ae9363 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 21 Dec 2024 13:22:04 +0100 Subject: [PATCH 10/51] switch pillow to pyproject.toml --- pyproject.toml | 78 ++++++++++++++++++++++++++++++++++++++++++++---- requirements.txt | 29 ------------------ 2 files changed, 72 insertions(+), 35 deletions(-) delete mode 100644 requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 2e1636a43..c8b2971e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,13 +8,55 @@ authors = [ ] requires-python = ">=3.8.0" license = { text = "VSL" } -dynamic = ["dependencies", "optional-dependencies", "version"] +dynamic = ["version"] + +dependencies = [ + "pefile>=2024.8.26", +] + +[project.optional-dependencies] +full = [ + "yara-python>=4.5.1,<5", + "capstone>=5.0.3,<6", + "pycryptodome>=3.21.0,<4", + "leechcorepyc>=2.19.2,<3; sys_platform != 'darwin'", + # https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst + # 10.0.0 dropped support for Python3.7 + # 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 + "pillow>=10.0.0,<11.0.0", +] + +cloud = [ + "gcsfs>=2024.10.0", + "s3fs>=2024.10.0", +] + +dev = [ + "volatility3[full,cloud]", + "jsonschema>=4.23.0,<5", + "pyinstaller>=6.11.0,<7", + "pyinstaller-hooks-contrib>=2024.9", +] + +test = [ + "volatility3[dev]", + "pytest>=8.3.3,<9", + "capstone>=5.0.3,<6", + "yara-x>=0.10.0,<1", +] + +docs = [ + "volatility3[dev]", + "sphinx>=8.0.0,<7", + "sphinx-autodoc-typehints>=2.5.0,<3", + "sphinx-rtd-theme>=3.0.1,<4", +] [project.urls] -Homepage = "https://github.com/volatilityfoundation/volatility3/" -"Bug Tracker" = "https://github.com/volatilityfoundation/volatility3/issues" -Documentation = "https://volatility3.readthedocs.io/" -"Source Code" = "https://github.com/volatilityfoundation/volatility3" +homepage = "https://github.com/volatilityfoundation/volatility3/" +documentation = "https://volatility3.readthedocs.io/" +repository = "https://github.com/volatilityfoundation/volatility3" +issues = "https://github.com/volatilityfoundation/volatility3/issues" [project.scripts] vol = "volatility3.cli:main" @@ -22,11 +64,35 @@ volshell = "volatility3.cli.volshell:main" [tool.setuptools.dynamic] version = { attr = "volatility3.framework.constants._version.PACKAGE_VERSION" } -dependencies = { file = "requirements-minimal.txt" } [tool.setuptools.packages.find] include = ["volatility3*"] +[tool.mypy] +mypy_path = "./stubs" +show_traceback = true + +[tool.mypy.overrides] +ignore_missing_imports = true + +[tool.ruff] +line-length = 88 +target-version = "py38" + +[tool.ruff.lint] +select = [ + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade +] + +ignore = [ + "E501", # ignore due to conflict with formatter +] + [build-system] requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 21c8f9a76..000000000 --- a/requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# Include the minimal requirements --r requirements-minimal.txt - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 - -# This is required for several plugins that perform malware analysis and disassemble code. -# It can also improve accuracy of Windows 8 and later memory samples. -# FIXME: Version 6.0.0 is incompatible (#1336) so we'll need an adaptor at some point -capstone>=3.0.5,<6.0.0 - -# This is required by plugins that decrypt passwords, password hashes, etc. -pycryptodome - -# This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0; sys_platform != 'darwin' - -# This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage -gcsfs>=2023.1.0 -s3fs>=2023.1.0 - -# This is required by plugins that manipulate pixels and images. -# https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst -# 10.0.0 dropped support for Python3.7 -# 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 -pillow>=10.0.0,<11.0.0 \ No newline at end of file From f6a54c5c48b6ba47a334956cb7994fecfcf14f0c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 21 Dec 2024 13:31:29 +0100 Subject: [PATCH 11/51] ruff fix --- volatility3/framework/plugins/linux/graphics/fbdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 60e00d033..7144e081a 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -218,7 +218,7 @@ class Fbdev(interfaces.plugins.PluginInterface): fourcc = linux.LinuxUtilities.convert_fourcc_code(fb_info.var.grayscale) warn_msg = f"""Framebuffer "{id}" uses a FOURCC pixel format "{fourcc}" that isn't natively supported. You can try using ffmpeg to decode the raw buffer. Example usage: -"ffmpeg -pix_fmts" to list supported formats, then +"ffmpeg -pix_fmts" to list supported formats, then "ffmpeg -f rawvideo -video_size {fb_info.var.xres_virtual}x{fb_info.var.yres_virtual} -i .raw -pix_fmt output.png".""" vollog.warning(warn_msg) From a84d3611130b1ba42e1e3c317bc6f633adc669ab Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 21 Dec 2024 12:18:23 -0600 Subject: [PATCH 12/51] Add the suspended threads plugin from DEF CON 2024 --- .../plugins/windows/suspended_threads.py | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 volatility3/framework/plugins/windows/suspended_threads.py diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py new file mode 100644 index 000000000..2cc0673d7 --- /dev/null +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -0,0 +1,147 @@ +import logging + +from typing import Dict +from functools import partial + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +import volatility3.plugins.windows.pslist as pslist +import volatility3.plugins.windows.threads as threads +import volatility3.plugins.windows.pe_symbols as pe_symbols + +from volatility3.framework.objects import utility + +vollog = logging.getLogger(__name__) + + +class SuspendedThreads(interfaces.plugins.PluginInterface): + """Enumerates suspended threads.""" + + _required_framework_version = (2, 13, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="threads", component=threads.Threads, version=(1, 0, 0) + ), + ] + + def _generator(self): + """ + The goal of this plugin is to report on threads that are suspended + + Legitimate programs can start threads suspended but then will later resume them + + Subsets of malware techniques, such as EDR evasion and process hollowing, + create suspended threads and do not resume them. These are the threads that this + plugin is designed to catch. + + See the whitepaper from our DEF CON 2024 presentation for more details: + + https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + """ + kernel = self.context.modules[self.config["kernel"]] + + vads_cache: Dict[int, pe_symbols.PESymbols.ranges_type] = {} + + proc_modules = None + + # walk the threads of each process checking for suspended threads + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ): + for thread in threads.Threads.list_threads(kernel, proc): + try: + # we only care if the thread is suspended + if thread.Tcb.SuspendCount == 0: + continue + + # 4 == terminated + if thread.Tcb.State == 4: + continue + + owner_proc = thread.owning_process() + owner_proc_pid = thread.Cid.UniqueProcess + owner_proc_name = utility.array_to_string(owner_proc.ImageFileName) + thread_tid = thread.Cid.UniqueThread + thread_start_addr = thread.StartAddress + thread_win32_addr = thread.Win32StartAddress + except exceptions.InvalidAddressException: + continue + + # Nothing useful to report if a process doesn't have VADs.. Also a sign of smear/terminated + vads = pe_symbols.PESymbols.get_vads_for_process_cache(vads_cache, owner_proc) + if not vads: + continue + + # Only compute this if needed as its expensive and 99.9% of samples + # will not have suspended threads + if not proc_modules: + proc_modules = pe_symbols.PESymbols.get_process_modules( + self.context, kernel.layer_name, kernel.symbol_table_name, None + ) + + path_and_symbol = partial( + pe_symbols.PESymbols.path_and_symbol_for_address, + self.context, + self.config_path, + proc_modules, + ) + + start_file, start_sym = path_and_symbol(vads, thread_start_addr) + win32_file, win32_sym = path_and_symbol(vads, thread_win32_addr) + + # the only false positive found in mass scanning of samples + if start_file and start_file.endswith("\\WorkFoldersShell.dll"): + continue + + if win32_file and win32_file.endswith("\\WorkFoldersShell.dll"): + continue + + yield ( + 0, + ( + owner_proc_name, + owner_proc_pid, + thread_tid, + start_file or renderers.NotAvailableValue(), + start_sym or renderers.NotAvailableValue(), + format_hints.Hex(thread_start_addr), + win32_file or renderers.NotAvailableValue(), + win32_sym or renderers.NotAvailableValue(), + format_hints.Hex(thread_win32_addr), + ), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("TID", int), + ("StartFile", str), + ("StartSymbol", str), + ("StartAddress", format_hints.Hex), + ("Win32StartFile", str), + ("Win32StartSymbol", str), + ("Win32StartAddress", format_hints.Hex), + ], + self._generator(), + ) + From d31ac276edb29721847f248c13953696c4a98a9c Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 21 Dec 2024 12:22:33 -0600 Subject: [PATCH 13/51] Add the suspended threads plugin from DEF CON 2024 --- volatility3/framework/plugins/windows/suspended_threads.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py index 2cc0673d7..6ddfecca7 100644 --- a/volatility3/framework/plugins/windows/suspended_threads.py +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -86,7 +86,9 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): continue # Nothing useful to report if a process doesn't have VADs.. Also a sign of smear/terminated - vads = pe_symbols.PESymbols.get_vads_for_process_cache(vads_cache, owner_proc) + vads = pe_symbols.PESymbols.get_vads_for_process_cache( + vads_cache, owner_proc + ) if not vads: continue @@ -144,4 +146,3 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): ], self._generator(), ) - From 7caf8c572a4629a2a235a974e6dfff112ccabecd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:03:26 +0100 Subject: [PATCH 14/51] PIL import graceful exit --- .../framework/plugins/linux/graphics/fbdev.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 7144e081a..28cae8000 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -4,9 +4,6 @@ import logging import io -# Image manipulation functions are kept in the plugin, -# to prevent a general exit on missing PIL (pillow) dependency. -from PIL import Image from dataclasses import dataclass from typing import Type, List, Dict, Tuple from volatility3.framework import constants, exceptions, interfaces @@ -16,6 +13,15 @@ from volatility3.framework.objects import utility from volatility3.framework.constants import architectures from volatility3.framework.symbols import linux +# Image manipulation functions are kept in the plugin, +# to prevent a general exit on missing PIL (pillow) dependency. +try: + from PIL import Image + + has_pil = True +except ImportError: + has_pil = False + vollog = logging.getLogger(__name__) @@ -101,7 +107,7 @@ class Fbdev(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, kernel_name: str, fb: Framebuffer, - ) -> Image.Image: + ): """Convert raw framebuffer pixels to an image. Args: @@ -238,6 +244,13 @@ You can try using ffmpeg to decode the raw buffer. Example usage: return fb def _generator(self): + + if not has_pil: + vollog.error( + "PIL (pillow) module is required to use this plugin. Please install it manually or through pyproject.toml." + ) + return None + kernel_name = self.config["kernel"] kernel = self.context.modules[kernel_name] From 1fef570022eb0ae13924ed321e5c09a24fe72563 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:15:45 +0100 Subject: [PATCH 15/51] restrict output to PNG, unify file handling --- .../framework/plugins/linux/graphics/fbdev.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 28cae8000..e82827944 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -160,15 +160,13 @@ class Fbdev(interfaces.plugins.PluginInterface): kernel_name: str, open_method: Type[interfaces.plugins.FileHandlerInterface], fb: Framebuffer, - convert_to_image: bool, - image_format: str = "PNG", + convert_to_png_image: bool, ) -> str: - """Dump a Framebuffer raw buffer to disk. + """Dump a Framebuffer buffer to disk. Args: fb: the relevant Framebuffer object convert_to_image: a boolean specifying if the buffer should be converted to an image - image_format: the target PIL image format (defaults to PNG) Returns: The filename of the dumped buffer. @@ -176,19 +174,19 @@ class Fbdev(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_name] kernel_layer = context.layers[kernel.layer_name] base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" - if convert_to_image: - image = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) - output = io.BytesIO() - image.save(output, image_format) - file_handle = open_method(f"{base_filename}.{image_format.lower()}") - file_handle.write(output.getvalue()) + if convert_to_png_image: + image_object = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) + raw_io_output = io.BytesIO() + image_object.save(raw_io_output, "PNG") + final_fb_buffer = raw_io_output.getvalue() + filename = f"{base_filename}.png" else: - raw_pixels = kernel_layer.read(fb.fb_info.screen_base, fb.size) - file_handle = open_method(f"{base_filename}.raw") - file_handle.write(raw_pixels) + final_fb_buffer = kernel_layer.read(fb.fb_info.screen_base, fb.size) + filename = f"{base_filename}.raw" - file_handle.close() - return file_handle.preferred_filename + with open_method(filename) as f: + f.write(final_fb_buffer) + return f.preferred_filename @classmethod def parse_fb_info( From 8d213284e642c545f44502d0fab3f026bc4fa0ff Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:23:04 +0100 Subject: [PATCH 16/51] handle NotAvailableValue in filename --- volatility3/framework/plugins/linux/graphics/fbdev.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index e82827944..7f7deee4e 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -173,7 +173,8 @@ class Fbdev(interfaces.plugins.PluginInterface): """ kernel = context.modules[kernel_name] kernel_layer = context.layers[kernel.layer_name] - base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" + id = "N-A" if isinstance(fb.id, NotAvailableValue) else fb.id + base_filename = f"{id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" if convert_to_png_image: image_object = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) raw_io_output = io.BytesIO() @@ -207,8 +208,7 @@ class Fbdev(interfaces.plugins.PluginInterface): - struct fb_var_screeninfo stores device independent changeable information about a frame buffer device, its current format and video mode, as well as other miscellaneous parameters. """ - # NotAvailableValue() messes with the filename output on disk - id = utility.array_to_string(fb_info.fix.id) or "N-A" + id = utility.array_to_string(fb_info.fix.id) or NotAvailableValue() color_fields = None # 0 = color, 1 = grayscale, >1 = FOURCC From 9d4dd010a7eb6212effc43dd8d57f86e127684f2 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:28:40 +0000 Subject: [PATCH 17/51] Reformat how to write a simple plugin --- doc/source/simple-plugin.rst | 93 ++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 84b921114..aa8ec3a7e 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -41,15 +41,24 @@ to be able to run properly. Any that are defined as optional need not necessari @classmethod def get_requirements(cls): - return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True), - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (2, 0, 0))] + return [ + requirements.ModuleRequirement( + name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"] + ), + requirements.ListRequirement( + name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True + ), + requirements.PluginRequirement( + name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0) + ), + ] This is a classmethod, so it can be called before the specific plugin object has been instantiated (in order to know how @@ -57,8 +66,11 @@ to instantiate the plugin). At the moment these requirements are fairly straigh :: - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"] + ), This requirement specifies the need for a particular submodule. Each module requires a :py:class:`TranslationLayer ` and a @@ -85,9 +97,11 @@ not be requested directly from the user. :: - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.TranslationLayerRequirement( + name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"] + ), This requirement indicates that the plugin will operate on a single :py:class:`TranslationLayer `. The name of the @@ -110,8 +124,10 @@ not be requested directly from the user. :: - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + requirements.SymbolTableRequirement( + name = "nt_symbols", + description = "Windows kernel symbols" + ), This requirement specifies the need for a particular :py:class:`SymbolTable ` @@ -127,10 +143,12 @@ not be requested directly from the user. :: - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True), + requirements.ListRequirement( + name = 'pid', + description = 'Filter on specific process IDs', + element_type = int, + optional = True + ), The next requirement is a List Requirement, populated by integers. The description will be presented to the user to describe what the value represents. The optional flag indicates that the plugin can function without the ``pid`` value @@ -138,9 +156,11 @@ being defined within the configuration tree at all. :: - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (2, 0, 0))] + requirements.PluginRequirement( + name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0) + ) This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements on that plugin. The version is specified in terms of Semantic Versioning meaning that, to be compatible, the major @@ -180,16 +200,21 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) kernel = self.context.modules[self.config['kernel']] - return renderers.TreeGrid([("PID", int), - ("Process", str), - ("Base", format_hints.Hex), - ("Size", format_hints.Hex), - ("Name", str), - ("Path", str)], - self._generator(pslist.PsList.list_processes(self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Base", format_hints.Hex), + ("Size", format_hints.Hex), + ("Name", str), + ("Path", str), + ], + self._generator( + pslist.PsList.list_processes( + self.context, kernel.layer_name, kernel.symbol_table_name, filter_func = filter_func + ) + ) + ) In this instance, the plugin constructs a filter (using the PsList plugin's *classmethod* for creating filters). It checks the plugin's configuration for the ``pid`` value, and passes it in as a list if it finds it, or None if @@ -281,5 +306,3 @@ such as ``!_UNICODE``) and the parameters to that type. Since the cast value must populate a string typed column, it had to be a Python string (such as being cast to the native type string) and could not have been a special Structure such as ``_UNICODE``. For the format hint columns, the format hint type must be used to ensure the error checking does not fail. - - From df37f0a909255410bd5df31cd4d16bdabb1aa9e8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:34:17 +0000 Subject: [PATCH 18/51] Reformat how to write a simple plugin --- doc/source/simple-plugin.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index aa8ec3a7e..07d9e1467 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -211,7 +211,10 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. ], self._generator( pslist.PsList.list_processes( - self.context, kernel.layer_name, kernel.symbol_table_name, filter_func = filter_func + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func = filter_func ) ) ) From 0bb09191aae08b2a1b481fef4fbd565e07a7d91f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 23 Dec 2024 21:28:58 +0100 Subject: [PATCH 19/51] file output failure results in UnreadableValue --- .../framework/plugins/linux/graphics/fbdev.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 7f7deee4e..ab4289cf1 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -8,7 +8,12 @@ from dataclasses import dataclass from typing import Type, List, Dict, Tuple from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.renderers import ( + format_hints, + TreeGrid, + NotAvailableValue, + UnreadableValue, +) from volatility3.framework.objects import utility from volatility3.framework.constants import architectures from volatility3.framework.symbols import linux @@ -280,11 +285,12 @@ You can try using ffmpeg to decode the raw buffer. Example usage: file_output = self.dump_fb( self.context, kernel_name, self.open, fb, bool(fb.color_fields) ) + file_output = str(file_output) except exceptions.InvalidAddressException as excp: vollog.error( f'Layer {excp.layer_name} failed to read address {hex(excp.invalid_address)} when dumping framebuffer "{fb.id}".' ) - file_output = "Error" + file_output = UnreadableValue() try: fb_device_name = utility.pointer_to_string( @@ -303,7 +309,7 @@ You can try using ffmpeg to decode the raw buffer. Example usage: f"{fb.xres_virtual}x{fb.yres_virtual}", fb.bpp, "RUNNING" if fb.fb_info.state == 0 else "SUSPENDED", - str(file_output), + file_output, ), ) From ea2757c06ce23d9a24e01e2ce7823e4980889ee8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 24 Dec 2024 11:45:23 +0100 Subject: [PATCH 20/51] minor version bump --- volatility3/framework/constants/_version.py | 2 +- volatility3/framework/plugins/linux/graphics/fbdev.py | 3 +++ volatility3/framework/symbols/linux/__init__.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 11edc07d8..9ca2d0a5b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 13 # Number of changes that only add to the interface +VERSION_MINOR = 14 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index ab4289cf1..7b644eccf 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -60,6 +60,9 @@ class Fbdev(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=architectures.LINUX_ARCHS, ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + ), requirements.BooleanRequirement( name="dump", description="Dump framebuffers", diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index ba223f979..5aa27b964 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -76,7 +76,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 1) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 79fe1c50dfcda812cd9d4b307b271ddcc3c26bd9 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 24 Dec 2024 19:42:05 +0000 Subject: [PATCH 21/51] Tweak configuration.py Create a tuple directly and replace random.choice by random.choices. --- volatility3/framework/interfaces/configuration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 2e4f580a7..a376fa813 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -53,7 +53,7 @@ ConfigSimpleType = Optional[Union[SimpleTypes, List[SimpleTypes]]] def path_join(*args) -> str: """Joins configuration paths together.""" # If a path element (particularly the first) is empty, then remove it from the list - args = tuple([arg for arg in args if arg]) + args = tuple(arg for arg in args if arg) return CONFIG_SEPARATOR.join(args) @@ -772,8 +772,7 @@ class ConfigurableInterface(metaclass=ABCMeta): str: The newly generated full configuration path """ random_config_dict = "".join( - random.SystemRandom().choice(string.ascii_uppercase + string.digits) - for _ in range(8) + random.SystemRandom().choices(string.ascii_uppercase + string.digits, k=8) ) new_config_path = path_join(base_config_path, random_config_dict) # TODO: Check that the new_config_path is empty, although it's not critical if it's not since the values are merged in From 4c430d2ec464b3e1fdf8ddd9b51fdf4525078814 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 26 Dec 2024 07:27:45 +0000 Subject: [PATCH 22/51] Use BasicTypes variable This removes a "float", which should be excluded. --- volatility3/framework/interfaces/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index a376fa813..b6f4f889c 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -779,9 +779,9 @@ class ConfigurableInterface(metaclass=ABCMeta): # This should check that each k corresponds to a requirement and each v is of the appropriate type # This would require knowledge of the new configurable itself to verify, and they should do validation in the - # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a simple type + # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a basic type for k, v in kwargs.items(): - if not isinstance(v, (int, str, bool, float, bytes)): + if not isinstance(v, BasicTypes): raise TypeError( "Config values passed to make_subconfig can only be simple types" ) From 834b7d0f072984f232dfc7880c0214b40df424fb Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 28 Dec 2024 14:58:12 +0000 Subject: [PATCH 23/51] Make ETHREAD year check dynamic Change upper bound year check of ETHREAD to be a decade from now. Makes consistent with EPROCESS. --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d63f138b6..5ec84f95f 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -519,7 +519,8 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False - if not (1998 < ctime.year < 2030): + current_year = datetime.datetime.now().year + if not (1998 < ctime.year < current_year + 10): return False except exceptions.InvalidAddressException: From 37873f9e1593fad055b0319085694d67a9568f4b Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 28 Dec 2024 16:58:24 +0000 Subject: [PATCH 24/51] Remove superfluous spaces in intermed.py --- volatility3/framework/symbols/intermed.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 5b4aa22b8..6802af7d6 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -101,7 +101,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): Args: context: The volatility context for the symbol table config_path: The configuration path for the symbol table - name: The name for the symbol table (this is used in symbols e.g. table!symbol ) + name: The name for the symbol table (this is used in symbols e.g. table!symbol) isf_url: The URL pointing to the ISF file location native_types: The NativeSymbolTable that contains the native types for this symbol table table_mapping: A dictionary linking names referenced in the file with symbol tables in the context @@ -111,7 +111,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): """ # Check there are no obvious errors # Open the file and test the version - self._versions = dict([(x.version, x) for x in class_subclasses(ISFormatTable)]) + self._versions = dict((x.version, x) for x in class_subclasses(ISFormatTable)) with resources.ResourceAccessor().open(isf_url) as fp: reader = codecs.getreader("utf-8") json_object = json.load(reader(fp)) # type: ignore @@ -166,9 +166,9 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): format. An interface version such as Major.Minor.Patch means that Major - of the provider must be equal to that of the consumer, and the + of the provider must be equal to that of the consumer, and the provider (the JSON in this instance) must have a greater minor - (indicating that only additive changes have been made) than + (indicating that only additive changes have been made) than the consumer (in this case, the file reader). """ major, minor, patch = (int(x) for x in version.split(".")) From 1bd031b9a86677f6b234e13f93ab2ad9feeb2cc8 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:42:25 +0000 Subject: [PATCH 25/51] Prevent infinite loops in device enumeration extensions #1483 --- .../symbols/windows/extensions/__init__.py | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d63f138b6..6dec08c38 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -405,11 +405,24 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the attached device's objects""" - device = self.AttachedDevice.dereference() - while device: - yield device - device = device.AttachedDevice.dereference() + seen = set() + try: + device = self.AttachedDevice.dereference() + except exceptions.InvalidAddressException: + return + + while device: + if device.vol.offset in seen: + break + seen.add(device.vol.offset) + + yield device + + try: + device = device.AttachedDevice.dereference() + except exceptions.InvalidAddressException: + return class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" @@ -421,10 +434,24 @@ class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" - device = self.DeviceObject.dereference() + seen = set() + + try: + device = self.DeviceObject.dereference() + except exceptions.InvalidAddressException: + return + while device: + if device.vol.offset in seen: + return + seen.add(device.vol.offset) + yield device - device = device.NextDevice.dereference() + + try: + device = device.NextDevice.dereference() + except exceptions.InvalidAddressException: + return def is_valid(self) -> bool: """Determine if the object is valid.""" From bf7f1ca91ed88bf482b19bd2558d79aa70bb5c0e Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:43:56 +0000 Subject: [PATCH 26/51] Prevent infinite loops in device enumeration extensions #1483 --- volatility3/framework/symbols/windows/extensions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 6dec08c38..c38d47f73 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -424,6 +424,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): except exceptions.InvalidAddressException: return + class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" From e64af61efa1a37f6d5c91e34d5375219b1544ee3 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:13:18 +0000 Subject: [PATCH 27/51] Do not analyze processes without VADs #1470 --- volatility3/framework/plugins/windows/direct_system_calls.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index b0c162f46..183e4095c 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -433,6 +433,8 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] vads = self.get_vad_maps(proc) + if not vads: + continue # for each valid process, look for malicious syscall invocations for address, vad_path in self._get_rule_hits( From 33855cf920a8ef76d2bfa414779b7cf96c8c8def Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:20:02 +0000 Subject: [PATCH 28/51] Significantly improve the smear/error handling in the netstat plugin --- .../framework/plugins/windows/netstat.py | 157 +++++++++++++----- .../symbols/windows/extensions/network.py | 8 +- 2 files changed, 120 insertions(+), 45 deletions(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index a1521a8c6..3408c0a3a 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -111,8 +111,21 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The list of indices at which a 1 was found. """ ret = [] + # This value is broken in many samples and was causing essentially infinite loops + # Testing showed that 8192 is the current size across all Windows versions + # We give some leeway in case it increases in later versions, while still keeping it sane + # The problematic samples had values that looked like addresses, so in the billions + if bitmap_size_in_byte > 8192 * 10: + return ret + for idx in range(bitmap_size_in_byte): - current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[0] + try: + current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[ + 0 + ] + except exceptions.InvalidAddressException: + continue + current_offs = idx * 8 for bit in range(8): if current_byte & (1 << bit) != 0: @@ -154,32 +167,37 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) else: # invalid argument. - return None + return vollog.debug(f"Current Port: {port}") # the given port serves as a shifted index into the port pool lists list_index = port >> 8 truncated_port = port & 0xFF - # constructing port_pool object here so callers don't have to - port_pool = context.object( - net_symbol_table + constants.BANG + "_INET_PORT_POOL", - layer_name=layer_name, - offset=port_pool_addr, - ) + try: + # constructing port_pool object here so callers don't have to + port_pool = context.object( + net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name=layer_name, + offset=port_pool_addr, + ) + # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) + inpa = port_pool.PortAssignments[list_index] - # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) - inpa = port_pool.PortAssignments[list_index] - - # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry - assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry + assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + except exceptions.InvalidAddressException: + return if not assignment: - return None + return # the value within assignment.Entry is a) masked and b) points inside of the network object # first decode the pointer - netw_inside = cls._decode_pointer(assignment.Entry) + try: + netw_inside = cls._decode_pointer(assignment.Entry) + except exceptions.InvalidAddressException: + return if netw_inside: # if the value is valid, calculate the actual object address by subtracting the offset @@ -188,16 +206,30 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) yield curr_obj + try: + next_obj_address = cls._decode_pointer(curr_obj.Next) + except exceptions.InvalidAddressException: + return + # if the same port is used on different interfaces multiple objects are created # those can be found by following the pointer within the object's `Next` field until it is empty - while curr_obj.Next: - curr_obj = context.object( - obj_name, - layer_name=layer_name, - offset=cls._decode_pointer(curr_obj.Next) - ptr_offset, - ) + while next_obj_address: + try: + curr_obj = context.object( + obj_name, + layer_name=layer_name, + offset=next_obj_address - ptr_offset, + ) + except exceptions.InvalidAddressException: + return + yield curr_obj + try: + next_obj_address = cls._decode_pointer(curr_obj.Next) + except exceptions.InvalidAddressException: + return + @classmethod def get_tcpip_module( cls, @@ -243,16 +275,25 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The hash table entries which are _not_ empty """ # we are looking for entries whose values are not their own address + # smear sanity check from mass testing + if ht_length > 4096: + return + for index in range(ht_length): current_addr = ht_offset + index * alignment - current_pointer = context.object( - net_symbol_table + constants.BANG + "pointer", - layer_name=layer_name, - offset=current_addr, - ) + try: + current_pointer = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=current_addr, + ) + except exceptions.InvalidAddressException: + continue + # check if addr of pointer is equal to the value pointed to if current_pointer.vol.offset == current_pointer: continue + yield current_pointer @classmethod @@ -292,11 +333,15 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): tcpip_symbol_table + constants.BANG + "PartitionCount" ).address - part_table_addr = context.object( - net_symbol_table + constants.BANG + "pointer", - layer_name=layer_name, - offset=tcpip_module_offset + part_table_symbol, - ) + try: + part_table_addr = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=tcpip_module_offset + part_table_symbol, + ) + except exceptions.InvalidAddressException: + vollog.debug(f"`PartitionTable` not present in memory.") + return # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects part_table = context.object( @@ -304,10 +349,18 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): layer_name=layer_name, offset=part_table_addr, ) - part_count = int.from_bytes( - context.layers[layer_name].read(tcpip_module_offset + part_count_symbol, 1), - "little", - ) + + try: + part_count = int.from_bytes( + context.layers[layer_name].read( + tcpip_module_offset + part_count_symbol, 1 + ), + "little", + ) + except exceptions.InvalidAddressException: + vollog.debug(f"`PartitionCount` not present in memory.") + return + part_table.Partitions.count = part_count vollog.debug( @@ -316,9 +369,21 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset( "ListEntry" ) - for ctr, partition in enumerate(part_table.Partitions): + + try: + partitions = part_table.Partitions + except exceptions.InvalidAddressException: + vollog.debug("Partitions member not present in memory") + return + + for ctr, partition in enumerate(partitions): vollog.debug(f"Parsing partition {ctr}") - if partition.Endpoints.NumEntries > 0: + try: + num_entries = partition.Endpoints.NumEntries + except exceptions.InvalidAddressException: + continue + + if num_entries > 0: for endpoint_entry in cls.parse_hashtable( context, layer_name, @@ -402,6 +467,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): upp_symbol = context.symbol_space.get_symbol( tcpip_symbol_table + constants.BANG + "UdpPortPool" ).address + upp_addr = context.object( net_symbol_table + constants.BANG + "pointer", layer_name=layer_name, @@ -498,13 +564,16 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # then, towards the UDP and TCP port pools # first, find their addresses - upp_addr, tpp_addr = cls.find_port_pools( - context, - layer_name, - net_symbol_table, - tcpip_symbol_table, - tcpip_module_offset, - ) + try: + upp_addr, tpp_addr = cls.find_port_pools( + context, + layer_name, + net_symbol_table, + tcpip_symbol_table, + tcpip_module_offset, + ) + except (exceptions.SymbolError, exceptions.InvalidAddressException): + vollog.debug("Unable to reconstruct port pools") # create port pool objects at the detected address and parse the port bitmap upp_obj = context.object( diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 478deab6b..e41ac6a05 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -219,7 +219,13 @@ class _TCP_ENDPOINT(_TCP_LISTENER): return None def is_valid(self): - if self.State not in self.State.choices.values(): + # netstat calls this before validating the object itself + try: + state = self.State + except exceptions.InvalidAddressException: + return False + + if state not in state.choices.values(): vollog.debug( f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid tcp state {self.State}" ) From 61d6a92f8f32e4fe81d951fc35fd7f36e80a2146 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:25:17 +0000 Subject: [PATCH 29/51] Significantly improve the smear/error handling in the netstat plugin --- volatility3/framework/plugins/windows/netstat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 3408c0a3a..902be5fc8 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -340,7 +340,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset=tcpip_module_offset + part_table_symbol, ) except exceptions.InvalidAddressException: - vollog.debug(f"`PartitionTable` not present in memory.") + vollog.debug("`PartitionTable` not present in memory.") return # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects @@ -358,7 +358,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): "little", ) except exceptions.InvalidAddressException: - vollog.debug(f"`PartitionCount` not present in memory.") + vollog.debug("`PartitionCount` not present in memory.") return part_table.Partitions.count = part_count From 65f602965b14a76dba1596e35a71ab1762257ea7 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:47:48 +0000 Subject: [PATCH 30/51] Address feedback --- volatility3/framework/plugins/windows/suspended_threads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py index 6ddfecca7..cec51ed37 100644 --- a/volatility3/framework/plugins/windows/suspended_threads.py +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -1,7 +1,7 @@ import logging from typing import Dict -from functools import partial +import functools from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements @@ -99,7 +99,7 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, None ) - path_and_symbol = partial( + path_and_symbol = functools.partial( pe_symbols.PESymbols.path_and_symbol_for_address, self.context, self.config_path, From 52a643d5b7f6c57e67acf39eed7c1feb2a0e9dbe Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 29 Dec 2024 20:29:03 +0000 Subject: [PATCH 31/51] Use enumerate for readability --- volatility3/framework/renderers/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 112e93751..093edf8cc 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -83,8 +83,7 @@ class TreeNode(interfaces.renderers.TreeNode): raise TypeError( "Values must be a list of objects made up of simple types and number the same as the columns" ) - for index in range(len(self._treegrid.columns)): - column = self._treegrid.columns[index] + for index, column in enumerate(self._treegrid.columns): val = values[index] if not isinstance(val, (column.type, interfaces.renderers.BaseAbsentValue)): raise TypeError( @@ -413,8 +412,7 @@ class ColumnSortKey(interfaces.renderers.ColumnSortKey): _index = None self._type = None self.ascending = ascending - for i in range(len(treegrid.columns)): - column = treegrid.columns[i] + for i, column in enumerate(treegrid.columns): if column.name.lower() == column_name.lower(): _index = i self._type = column.type From 28ff910d6280edc0dfa21b3b0585a1ab07de9279 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 30 Dec 2024 10:58:38 +0000 Subject: [PATCH 32/51] Use rsplit instead of split Since we want to split rightmost only. --- volatility3/plugins/windows/registry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/__init__.py b/volatility3/plugins/windows/registry/__init__.py index 8915cdfad..aeeaa87f2 100644 --- a/volatility3/plugins/windows/registry/__init__.py +++ b/volatility3/plugins/windows/registry/__init__.py @@ -15,5 +15,5 @@ import os import sys # This is necessary to ensure the core plugins are available, whilst still be overridable -parent_module, module_name = ".".join(__name__.split(".")[:-1]), __name__.split(".")[-1] +parent_module, module_name = __name__.rsplit(".", maxsplit=1) __path__ = [os.path.join(x, module_name) for x in sys.modules[parent_module].__path__] From e9d9345cef488067e7035aa485ff11ed665c4414 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 09:37:40 -0600 Subject: [PATCH 33/51] Windows Cachedump: Handle uncaught InvalidAddressException --- volatility3/framework/plugins/windows/cachedump.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index 6e667984a..6c730e6ae 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -8,7 +8,7 @@ from typing import Tuple from Crypto.Cipher import ARC4, AES from Crypto.Hash import HMAC -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions @@ -140,9 +140,14 @@ class Cachedump(interfaces.plugins.PluginInterface): if cache_item.Name == "NL$Control": continue - data = sechive.read(cache_item.Data + 4, cache_item.DataLength) - if data is None: + try: + data = sechive.read(cache_item.Data + 4, cache_item.DataLength) + except exceptions.InvalidAddressException: continue + + if not data: + continue + ( uname_len, domain_len, From 2153b742a1dbde57b369adb34fedc5e05e5eb40c Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 30 Dec 2024 18:06:04 +0000 Subject: [PATCH 34/51] Fix uncheck read() call and remove variable that would not be definied if exception triggers --- .../plugins/windows/skeleton_key_check.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index f5d7e1b3a..b103bc831 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -289,7 +289,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException as excp: vollog.debug( - f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" + f"Invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None, None @@ -431,15 +431,20 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): # we do not want to fail just because the count is not in memory # 16 was the size on samples I tested, so I chose it as the default + count = 16 + if target_address: - count = int.from_bytes( - self.context.layers[proc_layer_name].read( - target_address, 4 - ), - "little", - ) - else: - count = 16 + try: + count = int.from_bytes( + self.context.layers[proc_layer_name].read( + target_address, 4 + ), + "little", + ) + except exceptions.InvalidAddressException: + vollog.debug( + "Unable to read `cCsystems`. Defaulting to 16." + ) found_count = True From a3844e8bc54f9d597d459005830e733bd73d7256 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 30 Dec 2024 18:08:30 +0000 Subject: [PATCH 35/51] Fix uncheck read() call and remove variable that would not be definied if exception triggers --- volatility3/framework/plugins/windows/skeleton_key_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index b103bc831..6ae07381a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -282,7 +282,6 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): for proc in proc_list: try: - proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() return proc, proc_layer_name From 5af5363c461eab6ae1661665429a1794a2a712fb Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 13:50:05 -0600 Subject: [PATCH 36/51] Windows Handles: Work in fixes from @attrc These changes fix bugs encountered during regression testing related to virtual offset validation and string length checks. --- volatility3/framework/plugins/windows/handles.py | 8 ++++++++ .../framework/symbols/windows/extensions/pool.py | 11 ++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 62eceb973..e3845b376 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -226,6 +226,14 @@ class Handles(interfaces.plugins.PluginInterface): masked_offset = offset & layer_object.maximum_address for entry in table: + # This triggered a backtrace in many testing samples + # in the level == 0 path + # The code above this calls `is_valid` on the `offset` + # It is sent but then does not validate `entry` before + # sending it to `_get_item` + if not self.context.layers[virtual].is_valid(entry.vol.offset): + continue + if level > 0: yield from self._make_handle_array(entry, level - 1, depth) depth += 1 diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index de5c8271b..ff65acdeb 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -376,7 +376,16 @@ class OBJECT_HEADER(objects.StructType): try: # vista and earlier have a Type member - self._vol["object_header_object_type"] = self.Type.Name.String + length = self.Type.member("Name").Length + if length == 0 or length > 128: + string = None + else: + string = self.Type.Name.String + if len(string) == 0 or len(string) > 128: + string = None + + self._vol["object_header_object_type"] = string + except AttributeError: # windows 7 and later have a TypeIndex, but windows 10 # further encodes the index value with nt1!ObHeaderCookie From 3eeb10be2916bb7988d296c7a85785ffb5a7f25e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 10:10:03 -0600 Subject: [PATCH 37/51] Windows Registry: Handle uncaught exceptions A number of calls to `get_key` across multiple plugins are not made within a `try/except` block that handles `registry.RegistryFormatException` - the calls are either unprotected or only check for `KeyError`. This adds the required `try/except` blocks, or updates the existing ones as needed. --- volatility3/framework/plugins/windows/amcache.py | 10 +++++----- volatility3/framework/plugins/windows/hashdump.py | 2 +- volatility3/framework/plugins/windows/lsadump.py | 7 +++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/amcache.py b/volatility3/framework/plugins/windows/amcache.py index 1e918d61c..46a742233 100644 --- a/volatility3/framework/plugins/windows/amcache.py +++ b/volatility3/framework/plugins/windows/amcache.py @@ -543,7 +543,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\InventoryDriverBinary") # type: ignore ) ) - except KeyError: + except (KeyError, registry.RegistryFormatException): # Registry key not found pass @@ -554,7 +554,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\Programs") ) # type: ignore } - except KeyError: + except (KeyError, registry.RegistryFormatException): programs = {} try: @@ -564,7 +564,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), key=_entry_sort_key, ) - except KeyError: + except (KeyError, registry.RegistryFormatException): files = [] for program_id, file_entries in itertools.groupby( @@ -593,7 +593,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\InventoryApplication") # type: ignore ) ) - except KeyError: + except (KeyError, registry.RegistryFormatException): programs = {} try: @@ -603,7 +603,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), key=_entry_sort_key, ) - except KeyError: + except (KeyError, registry.RegistryFormatException): files = [] for program_id, file_entries in itertools.groupby( diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index 0c98ab8ca..621b0ae53 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -332,7 +332,7 @@ class Hashdump(interfaces.plugins.PluginInterface): try: if hive: result = hive.get_key(key) - except KeyError: + except (KeyError, registry.RegistryFormatException): vollog.info( f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image" ) diff --git a/volatility3/framework/plugins/windows/lsadump.py b/volatility3/framework/plugins/windows/lsadump.py index da8dee325..f3925f2a2 100644 --- a/volatility3/framework/plugins/windows/lsadump.py +++ b/volatility3/framework/plugins/windows/lsadump.py @@ -8,7 +8,7 @@ from typing import Optional from Crypto.Cipher import ARC4, DES, AES from Crypto.Hash import MD5, SHA256 -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions @@ -81,7 +81,10 @@ class Lsadump(interfaces.plugins.PluginInterface): if not enc_reg_value: return None - obf_lsa_key = sechive.read(enc_reg_value.Data + 4, enc_reg_value.DataLength) + try: + obf_lsa_key = sechive.read(enc_reg_value.Data + 4, enc_reg_value.DataLength) + except exceptions.InvalidAddressException: + return None if not obf_lsa_key: return None From f3294ef5f12a6b036989585b88105fde62634ce2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 12:58:44 -0600 Subject: [PATCH 38/51] Windows Registry: Handle possible exception in get_node Encountered a `SwappedInvalidAddressException` within the call to `cast` due to an underlying call to `read`. --- volatility3/framework/layers/registry.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc364ad50..ee7286e1e 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -140,7 +140,14 @@ class RegistryHive(linear.LinearlyMappedLayer): """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast("string", max_length=2, encoding="latin-1") + try: + signature = cell.cast("string", max_length=2, encoding="latin-1") + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read cell signature for cell at {cell.vol.offset:x}" + ) + return cell + if signature == "nk": return cell.u.KeyNode elif signature == "sk": From 21077f909f6bda2dec6a3900c7e9ec53268b8213 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 30 Dec 2024 22:28:47 +0000 Subject: [PATCH 39/51] Sort imports and swap two assignments --- volatility3/cli/text_filter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 6bd6878a5..b6f019da9 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -1,7 +1,8 @@ import logging -from typing import Any, List, Optional -from volatility3.framework import constants, interfaces import re +from typing import Any, List, Optional + +from volatility3.framework import constants, interfaces vollog = logging.getLogger(__name__) @@ -67,8 +68,8 @@ class ColumnFilter: ) -> None: self.column_num = column_num self.pattern = pattern - self.exclude = exclude self.regex = regex + self.exclude = exclude def find(self, item) -> bool: """Identifies whether an item is found in the appropriate column""" From 3288ac971397500f61501b86a99678592cbd4128 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 14:01:40 -0600 Subject: [PATCH 40/51] Windows Handles: Handle possibly invalid memory accesses Any number of member accesses here can raise an `InvalidAddressException`; each is now checked, and `None` returned if any `InvalidAddressException` occurs. --- .../framework/plugins/windows/handles.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index e3845b376..85b16d2d4 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -68,7 +68,12 @@ class Handles(interfaces.plugins.PluginInterface): if not self.context.layers[virtual].is_valid(handle_table_entry.Object): return None fast_ref = handle_table_entry.Object.cast("_EX_FAST_REF") - object_header = fast_ref.dereference().cast("_OBJECT_HEADER") + + try: + object_header = fast_ref.dereference().cast("_OBJECT_HEADER") + except exceptions.InvalidAddressException: + return None + object_header.GrantedAccess = handle_table_entry.GrantedAccess except AttributeError: # starting with windows 8 @@ -77,16 +82,26 @@ class Handles(interfaces.plugins.PluginInterface): ) if is_64bit: - if handle_table_entry.ObjectPointerBits == 0: + try: + pointer_bits = handle_table_entry.ObjectPointerBits + except exceptions.InvalidAddressException: return None - offset = handle_table_entry.ObjectPointerBits << 4 + if pointer_bits == 0: + return None + + offset = pointer_bits << 4 else: - if handle_table_entry.InfoTable == 0: + try: + info_table = handle_table_entry.InfoTable + except exceptions.InvalidAddressException: return None - offset = handle_table_entry.InfoTable & ~7 + if info_table == 0: + return None + + offset = info_table & ~7 # print("LowValue: {0:#x} Magic: {1:#x} Offset: {2:#x}".format(handle_table_entry.InfoTable, magic, offset)) object_header = self.context.object( @@ -94,7 +109,10 @@ class Handles(interfaces.plugins.PluginInterface): virtual, offset=offset, ) - object_header.GrantedAccess = handle_table_entry.GrantedAccessBits + try: + object_header.GrantedAccess = handle_table_entry.GrantedAccessBits + except exceptions.InvalidAddressException: + return None object_header.HandleValue = handle_value return object_header From 9a5365e681e971f0e58b23ed42195df09dced6e3 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 16:59:40 -0600 Subject: [PATCH 41/51] Windows Handles: Fix unbound local in exception handler This fixes an unbound local used in a debug message; If the exception is raised during the dereference operation, the `objct` variable may be uninitialized. This uses the offset of `ptr` instead. --- volatility3/framework/plugins/windows/handles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 85b16d2d4..38ccfbfbc 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -178,7 +178,7 @@ class Handles(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, - f"Cannot access _OBJECT_HEADER Name at {objt.vol.offset:#x}", + f"Cannot access _OBJECT_HEADER Name at {ptr.vol.offset:#x}", ) continue From 263c87611b51f1cb9710c2ec71d1f41eb98e9c77 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 31 Dec 2024 10:43:27 -0600 Subject: [PATCH 42/51] Windows Registry: Handle exceptions in read calls These calls to `.read()` can raise an `InvalidAddressException`. Instead of propagating this exception to the caller, this adds debug logging, and pads the data will null bytes. Also updates the docstring for `decode_data()` to indicate that it can raise `TypeError` and `ValueError`. --- .../symbols/windows/extensions/registry.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 9e2f8df3b..97dd7390d 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -276,7 +276,16 @@ class CM_KEY_VALUE(objects.StructType): return RegValueTypes(self.Type) def decode_data(self) -> Union[int, bytes]: - """Properly decodes the data associated with the value node""" + """ + Properly decodes the data associated with the value node. + + If an InvalidAddressException occurs when reading data from the + underlying RegistryHive layer, the data will be padded with null bytes + of the same length. + + Raises ValueError if the data cannot be read + Raises TypeError if the class was not instantiated on a RegistryHive layer + """ # Determine if the data is stored inline datalen = self.DataLength data = b"" @@ -310,14 +319,26 @@ class CM_KEY_VALUE(objects.StructType): and block_offset < layer.maximum_address ): amount = min(BIG_DATA_MAXLEN, datalen) - data += layer.read( - offset=layer.get_cell(block_offset).vol.offset, length=amount - ) + try: + data += layer.read( + offset=layer.get_cell(block_offset).vol.offset, + length=amount, + ) + except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {amount:x} bytes of data, padding with {amount:x}" + ) datalen -= amount else: # Suspect Data actually points to a Cell, # but the length at the start could be negative so just adding 4 to jump past it - data = layer.read(self.Data + 4, datalen) + try: + data = layer.read(self.Data + 4, datalen) + except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {datalen:x} bytes of data, returning {datalen:x} null bytes" + ) + data = b"\x00" * datalen if self.get_type() == RegValueTypes.REG_DWORD: if len(data) != struct.calcsize(" Date: Tue, 31 Dec 2024 11:11:57 -0600 Subject: [PATCH 43/51] Windows Registry: Update docstrings + exceptions This updates the docstrings on several methods to indicate that they may raise an exception. --- .../symbols/windows/extensions/registry.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 97dd7390d..e53338855 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -159,6 +159,11 @@ class CM_KEY_NODE(objects.StructType): """Extension to allow traversal of registry keys.""" def get_volatile(self) -> bool: + """ + Returns a bool indicating whether or not the key is volatile. + + Raises ValueError if the key was not instantiated on a RegistryHive layer + """ if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): raise ValueError( "Cannot determine volatility of registry key without an offset in a RegistryHive layer" @@ -166,7 +171,10 @@ class CM_KEY_NODE(objects.StructType): return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: - """Returns a list of the key nodes.""" + """Returns a list of the key nodes. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") @@ -222,7 +230,10 @@ class CM_KEY_NODE(objects.StructType): yield from self._get_subkeys_recursive(hive, subnode) def get_values(self) -> Iterator["CM_KEY_VALUE"]: - """Returns a list of the Value nodes for a key.""" + """Returns a list of the Value nodes for a key. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") @@ -251,6 +262,11 @@ class CM_KEY_NODE(objects.StructType): return self.Name.cast("string", max_length=namelength, encoding="latin-1") def get_key_path(self) -> str: + """ + Returns the full path to this registry key. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ reg = self._context.layers[self.vol.layer_name] if not isinstance(reg, RegistryHive): raise TypeError("Key was not instantiated on a RegistryHive layer") From fa67f10d3183cd743ca7e44b66d152141a34b2fb Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 31 Dec 2024 11:40:41 -0600 Subject: [PATCH 44/51] Windows Registry: Catch RegistryInvalidIndex refs #1484 This catches uncaught exceptions when casting the cell to a string in `get_node`. --- volatility3/framework/layers/registry.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc364ad50..c684ccd40 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -140,7 +140,13 @@ class RegistryHive(linear.LinearlyMappedLayer): """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast("string", max_length=2, encoding="latin-1") + try: + signature = cell.cast("string", max_length=2, encoding="latin-1") + except (RegistryInvalidIndex, exceptions.InvalidAddressException): + vollog.debug( + f"Failed to get cell signature for cell (0x{cell.vol.offset:x})" + ) + return cell if signature == "nk": return cell.u.KeyNode elif signature == "sk": From 9c02f0d12a13fb77db8fb326f6f68dc31fceec1f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 15:22:09 +0000 Subject: [PATCH 45/51] Linux: Fix kmsf f-strings Closes #1496 --- volatility3/framework/plugins/linux/kmsg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d66e3b9ca..c1d09aff8 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return f"{nsec / 1000000000:lu}.{(nsec % 1000000000) / 1000:06lu}" + return f"{nsec / 1000000000}.{(nsec % 1000000000) / 1000:06}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = f"{caller_name}({caller_id & ~0x80000000:u})" + caller = f"{caller_name}({int(caller_id & ~0x80000000)})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: From ac3e76665b7a44b6c5dbc18e633814bd2371ff75 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 15:29:07 +0000 Subject: [PATCH 46/51] Linux: Fix kmsg unguarded read of msg.len --- volatility3/framework/plugins/linux/kmsg.py | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d66e3b9ca..67114c087 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -317,23 +317,27 @@ class Kmsg_3_5_to_3_11(ABCKmsg): while cur_idx < end_idx: msg_offset = log_buf_ptr + cur_idx # type: ignore msg = self.vmlinux.object(object_type=log_struct_name, offset=msg_offset) - if msg.len == 0: - # As per kernel/printk.c: - # A length == 0 for the next message indicates a wrap-around to - # the beginning of the buffer. - cur_idx = 0 - end_idx = log_next_idx - else: - facility, level, timestamp, caller = self.get_prefix(msg) - level_txt = self.get_level_text(level) - facility_txt = self.get_facility_text(facility) + try: + if msg.len == 0: + # As per kernel/printk.c: + # A length == 0 for the next message indicates a wrap-around to + # the beginning of the buffer. + cur_idx = 0 + end_idx = log_next_idx + else: + facility, level, timestamp, caller = self.get_prefix(msg) + level_txt = self.get_level_text(level) + facility_txt = self.get_facility_text(facility) - for line in self.get_log_lines(msg): - yield facility_txt, level_txt, timestamp, caller, line - for line in self.get_dict_lines(msg): - yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_log_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_dict_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line - cur_idx += msg.len + cur_idx += msg.len + except exceptions.InvalidAddressException: + vollog.warning("Kmsg buffer msg length could not be read") + return class Kmsg_3_11_to_5_10(Kmsg_3_5_to_3_11): From c8e67e526a831dcd05b59fa0adeeda8937c4f81a Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 1 Jan 2025 22:33:25 -0600 Subject: [PATCH 47/51] Convert ValueError to TypeError All other methods in this class raise a `TypeError` if the hive was not instantiated on a registry layer; this changes makes this method consistent with the convention used in the others. All `except` blocks checking for `ValueError` have been audited to ensure that this doesn't break exception handling in existing code within the framework. This also includes a minor version bump because: 1. RegistryHives are currently only instantiated one way, which is through the `hivelist` plugin. `hivelist` uses the correct layers when instantiating the hives. 2. Because there is currently a single source for registry hives, and it's unlikely that a hive from that source will ever be created on the wrong layer, it's unlikely that the existing `ValueError` is being raised anywhere within the framework's code. 3. It seems unlikely that consumers of this framework would be instantiating registry hives independent of the `hivelist` plugin, given that they would effectively have to duplicate the `hivelist` code to do so. For these reasons, we're going to do a minor version bump, even though an argument can be made that this warrants a major version bump according to the SemVer rules. This is a one-off and does not indicate any change in the way that we typically update version numbers. --- volatility3/framework/constants/_version.py | 2 +- .../framework/symbols/windows/extensions/registry.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 9ca2d0a5b..2f0c53093 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 14 # Number of changes that only add to the interface +VERSION_MINOR = 15 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index e53338855..c9544a8ba 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -162,12 +162,10 @@ class CM_KEY_NODE(objects.StructType): """ Returns a bool indicating whether or not the key is volatile. - Raises ValueError if the key was not instantiated on a RegistryHive layer + Raises TypeError if the key was not instantiated on a RegistryHive layer """ if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): - raise ValueError( - "Cannot determine volatility of registry key without an offset in a RegistryHive layer" - ) + raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: From 97b93abe438bf32b32067bd962a19e07d9917406 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 2 Jan 2025 11:26:21 +0000 Subject: [PATCH 48/51] Linux: Remove unnecessary int cast --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index c1d09aff8..894ca575f 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = f"{caller_name}({int(caller_id & ~0x80000000)})" + caller = f"{caller_name}({caller_id & ~0x80000000})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: From 7278bb244f58f36b346d254512e393cde6f63871 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 14:47:18 +0100 Subject: [PATCH 49/51] move get_flags_list at bottom --- .../framework/symbols/linux/extensions/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 34d0fcba9..9546fcf82 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2628,6 +2628,19 @@ class page(objects.StructType): page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) return page_data + def get_flags_list(self) -> List[str]: + """Returns a list of page flags + + Returns: + List of page flags + """ + flags = [] + for name, value in self.pageflags_enum.items(): + if self.flags & (1 << value) != 0: + flags.append(name) + + return flags + class IDR(objects.StructType): IDR_BITS = 8 From b61ba66223a866a29a119eb17f44fba250ecc01e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 14:51:23 +0100 Subject: [PATCH 50/51] multi-architecture vmemmap_start calculation --- .../symbols/linux/extensions/__init__.py | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9546fcf82..b02f80433 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,7 +15,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion from volatility3.framework.constants import linux as linux_constants -from volatility3.framework.layers import linear +from volatility3.framework.layers import linear, intel from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf @@ -2525,16 +2525,13 @@ class address_space(objects.StructType): class page(objects.StructType): - @property - @functools.lru_cache + @functools.cached_property def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values Returns: A dictionary with the pageflags enumeration key/values """ - # FIXME: It would be even better to use @functools.cached_property instead, - # however, this requires Python +3.8 try: pageflags_enum = self._context.symbol_space.get_enumeration( self.get_symbol_table_name() + constants.BANG + "pageflags" @@ -2548,24 +2545,12 @@ class page(objects.StructType): return pageflags_enum - def get_flags_list(self) -> List[str]: - """Returns a list of page flags + @functools.cached_property + def _intel_vmemmap_start(self) -> int: + """Determine the start of the struct page array, for Intel systems. Returns: - List of page flags - """ - flags = [] - for name, value in self.pageflags_enum.items(): - if self.flags & (1 << value) != 0: - flags.append(name) - - return flags - - def to_paddr(self) -> int: - """Converts a page's virtual address to its physical address using the current physical memory model. - - Returns: - int: page physical address + int: vmemmap_start address """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] @@ -2605,13 +2590,39 @@ class page(objects.StructType): "Something went wrong, we shouldn't be here" ) - page_type_size = vmlinux.get_type("page").size + return vmemmap_start + + def _intel_to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current Intel memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] pagec = vmlinux_layer.canonicalize(self.vol.offset) - pfn = (pagec - vmemmap_start) // page_type_size + pfn = (pagec - self._intel_vmemmap_start) // vmlinux.get_type("page").size page_paddr = pfn * vmlinux_layer.page_size return page_paddr + def to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current CPU memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + if isinstance(vmlinux_layer, intel.Intel): + page_paddr = self._intel_to_paddr() + else: + raise exceptions.LayerException( + f"Architecture {type(vmlinux_layer)} vmemmap_start calculation isn't currently supported." + ) + + return page_paddr + def get_content(self) -> Union[str, None]: """Returns the page content @@ -2620,7 +2631,10 @@ class page(objects.StructType): """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - physical_layer = vmlinux.context.layers["memory_layer"] + physical_layer_name = self._context.layers[self.vol.layer_name].config.get( + "memory_layer", self.vol.layer_name + ) + physical_layer = self._context.layers[physical_layer_name] page_paddr = self.to_paddr() if not page_paddr: return None From d2bb5c9f31d7f01fe2e343867c0c7c1926b3ac50 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 3 Jan 2025 10:01:37 +1100 Subject: [PATCH 51/51] linux: fix kmsg fstring bug introduced in #1502 --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 894ca575f..30f67b319 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return f"{nsec / 1000000000}.{(nsec % 1000000000) / 1000:06}" + return f"{nsec // 1000000000}.{(nsec % 1000000000) // 1000:06}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info