diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index e3fb726a1..8851e2b18 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -332,6 +332,7 @@ class CommandLine: parser.error(f"Cannot write configuration: file {args.save_config} already exists") with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + f.write("\n") except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 30fe75e06..769e958fd 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -246,6 +246,7 @@ class VolShell(cli.CommandLine): parser.error(f"Cannot write configuration: file {args.save_config} already exists") with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + f.write("\n") except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index ab81beb5e..85a7d32b7 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -321,7 +321,7 @@ class SizedModule(Module): The mapping should be sorted and should be quicker than reading the data We turn it into JSON to make a common string and use a - quick hash, because collissions are unlikely + quick hash, because collisions are unlikely """ layer = self._context.layers[self.layer_name] if not isinstance(layer, interfaces.layers.TranslationLayerInterface): diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index 9f2cb9fc9..1ad30cfdf 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -169,7 +169,7 @@ class BaseSymbolTableInterface: def optional_set_type_class(self, name: str, clazz: Type[objects.ObjectInterface]) -> bool: """Calls the set_type_class function but does not throw an exception. - Returns whether setting the type class was successfull. + Returns whether setting the type class was successful. Args: name: The name of the type to override the class for clazz: The actual class to override for the provided type name diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 73d46b211..0633637ca 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -88,6 +88,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): self._accessor = resources.ResourceAccessor() self._file_: Optional[IO[Any]] = None self._size: Optional[int] = None + self._maximum_address: Optional[int] = None # Construct the lock now (shared if made before threading) in case we ever need it self._lock: Union[DummyLock, threading.Lock] = DummyLock() if constants.PARALLELISM == constants.Parallelism.Threading: @@ -113,14 +114,15 @@ class FileLayer(interfaces.layers.DataLayerInterface): def maximum_address(self) -> int: """Returns the largest available address in the space.""" # Zero based, so we return the size of the file minus 1 - if self._size: - return self._size + if self._maximum_address: + return self._maximum_address with self._lock: orig = self._file.tell() self._file.seek(0, 2) self._size = self._file.tell() self._file.seek(orig) - return self._size + self._maximum_address = self._size - 1 + return self._maximum_address @property def minimum_address(self) -> int: @@ -189,7 +191,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): """Closes the file handle.""" self._file.close() - def __del__(self) -> None: + def __exit__(self) -> None: self.destroy() @classmethod diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index f1ba1e468..907116e99 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -3,12 +3,17 @@ # import functools import json -from typing import Optional, Dict, Any, Tuple, List, Set +import logging +import re +import struct +from typing import Any, Dict, List, Optional, Set, Tuple -from volatility3.framework import interfaces, exceptions, constants -from volatility3.framework.layers import segmented +from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework.layers import scanners, segmented from volatility3.framework.symbols import intermed +vollog = logging.getLogger(__name__) + class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): """A Qemu suspend-to-disk translation layer.""" @@ -32,6 +37,34 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): SEGMENT_FLAG_XBZRLE = 0x40 SEGMENT_FLAG_HOOK = 0x80 + # See https://qemu.readthedocs.io/en/latest/devel/memory.html for more info + # + # At least the following values could occur for devices using > 3-4 GB RAM: + # +--------------------------------+--------------------------------+------------+-------------+ + # | Architecture | Reference Code | Hole Start | Hole End | + # +--------------------------------+--------------------------------+------------+-------------+ + # | PC i440FX + PIIX "New Default" | qemu/hw/i386/pc_piix.c:98 | 0xc0000000 | 0x100000000 | + # | PC i440FX + PIIX "Old Default" | qemu/hw/i386/pc_piix.c:98 | 0xe0000000 | 0x100000000 | + # | PC Q35 + ICH9 | qemu/hw/i386/pc_q35.c:141 | 0x80000000 | 0x100000000 | + # | MicroVM | qemu/hw/i386/microvm.c:291 | 0xc0000000 | 0x100000000 | + # | Xen | qemu/hw/i386/xen/xen-hvm.c:248 | 0xf0000000 | 0x100000000 | + # +--------------------------------+--------------------------------+------------+-------------+ + # + # For now, we assume that the parameter max-ram-below-4g is not set, since this parameter influences the size + # and location of the memory gap. Deviating hole sizes could eventually be detected for Linux by e.g. scanning + # for dmesg entries with a regex like rb'\[mem (0x[0-9a-f]{4,10})-0x[0-9a-f]{4,10}\] available for PCI devices' + + distro_re = r"(\w+[\d{1,2}\.]*)" + + pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), + re.compile(r"^pc-i440fx-[01]\.\d$"): (0xe0000000, 0xe0000000, 0x100000000), + re.compile(r"^pc-q35-\d\.\d$"): (0xb0000000, 0x80000000, 0x100000000), + re.compile(r"^microvm$"): (0xc0000000, 0xc0000000, 0x100000000), + re.compile(r"^xen$"): (0xf0000000, 0xf0000000, 0x100000000), + re.compile(r"^pc-i440fx-" + distro_re + r"$"): (0xe0000000, 0xc0000000, 0x100000000), + re.compile(r"^pc-q35-" + distro_re + r"$"): (0xb0000000, 0x80000000, 0x100000000), + } + def __init__(self, context: interfaces.context.ContextInterface, config_path: str, @@ -39,8 +72,12 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): metadata: Optional[Dict[str, Any]] = None) -> None: self._qemu_table_name = intermed.IntermediateSymbolTable.create(context, config_path, 'generic', 'qemu') self._configuration = None + self._architecture = None self._compressed: Set[int] = set() self._current_segment_name = b'' + self._pci_hole_start = 0 + self._pci_hole_end = 0 + self._pci_hole_minimum = 0 super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) @classmethod @@ -50,6 +87,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): raise exceptions.LayerException(name, 'No QEMU magic bytes') if header[4:] != b'\x00\x00\x00\x03': raise exceptions.LayerException(name, 'Unsupported QEMU version found') + vollog.debug("QEVM header found") def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, name: str) -> Any: """Reads the JSON configuration from the end of the file""" @@ -73,12 +111,13 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): done = None segments = [] + size_array = {} base_layer = self.context.layers[self._base_layer] while not done: - addr = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', - offset = index, - layer_name = self._base_layer) + # Use struct.unpack here for performance improvements + addr = struct.unpack('>Q', base_layer.read(index, 8))[0] + # Flags are stored in the n least significant bits, where n equals the bit-length of pagesize flags = addr & (page_size - 1) # addr equals the highest multiple of pagesize <= offset @@ -86,19 +125,29 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): addr = addr ^ (addr & (page_size - 1)) index += 8 + if addr >= self._pci_hole_start: + addr += self._pci_hole_end - self._pci_hole_start + if flags & self.SEGMENT_FLAG_MEM_SIZE: namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) while namelen != 0: - # if base_layer.read(index + 1, namelen) == b'pc.ram': - # total_size = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', - # offset = index + 1 + namelen, - # layer_name = self._base_layer) + total_size = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', + offset = index + 1 + namelen, + layer_name = self._base_layer) + size_array[base_layer.read(index + 1, namelen)] = total_size index += 1 + namelen + 8 namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) + highest_possible_maximum = max([x[0] for x in self.pci_hole_table.values()]) + 1 + if size_array.get(b'pc.ram', highest_possible_maximum) < self._pci_hole_minimum: + # Turns off the pci_hole if it's not supposed to be there + vollog.debug( + f"QEVM turning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}") + self._pci_hole_start, self._pci_hole_end = 0, 0 + if flags & (self.SEGMENT_FLAG_COMPRESS | self.SEGMENT_FLAG_PAGE): if not (flags & self.SEGMENT_FLAG_CONTINUE): namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', @@ -130,7 +179,26 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): index = 8 section_info = dict() current_section_id = -1 + arch_detected = False while section_byte != self.QEVM_EOF and index <= base_layer.maximum_address: + if index > 20 and not arch_detected: + # We're past where the QEVM_CONFIGURATION might be, so set the values + # If no architecture has been set, try to determine it using fallback mechanisms + if not self._architecture: + self._architecture = self._fallback_determine_architecture() + if self._architecture is None: + vollog.log(constants.LOGLEVEL_VV, f"QEVM architecture could not be determined") + + # Once all segments have been read, determine the PCI hole if any + for regex in self.pci_hole_table: + if regex.match(self._architecture): + self._pci_hole_minimum, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") + break + else: + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") + arch_detected = True + section_byte = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) @@ -139,6 +207,9 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', offset = index, layer_name = self._base_layer) + self._architecture = self.context.object(self._qemu_table_name + constants.BANG + 'string', + offset = index + 4, layer_name = self._base_layer, + max_length = section_len) index += 4 + section_len elif section_byte == self.QEVM_SECTION_START or section_byte == self.QEVM_SECTION_FULL: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', @@ -189,6 +260,47 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): else: raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}') + def _fallback_determine_architecture(self) -> str: + architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|\w+[\d{1,2}\.]*)' + default_suffix = "-2.0" + base_layer = self.context.layers[self._base_layer] + + vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") + + res = scanners.RegExScanner(architecture_pattern) + for offset in base_layer.scan(context = self.context, scanner = res): + line = base_layer.read(offset, 64) + regex_results = re.search(architecture_pattern, line) + architecture = regex_results.group().decode() + return architecture + + # If that does not work, look in configuration JSON for devices specific to a certain architecture + architecture = None + for device in self._configuration.get('devices', []): + device_name = device.get('vmsd_name', '').lower() + if 'i440fx' in device_name or 'piix' in device_name: + architecture = 'pc-i440fx' + default_suffix + break + elif 'ich9' in device_name: + architecture = 'pc-q35' + default_suffix + break + if architecture: + vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}') + return architecture + + # Still haven't found architecture, switch to fallback-method + architecture_pattern = rb'Standard PC \((i440FX|Q35)' + res = scanners.RegExScanner(architecture_pattern) + for offset in base_layer.scan(context = self.context, scanner = res): + line = base_layer.read(offset, 64) + regex_results = re.search(architecture_pattern, line) + architecture = "pc-" + regex_results.groups()[0].decode().lower() + default_suffix + vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}') + return architecture + + vollog.warning("Could not determine QEMU target architecture!") + return None + def extract_data(self, index, name, version_id): if name == 'ram': if version_id != 4: diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index ac25b5cc2..8a0e96208 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -171,6 +171,8 @@ class ResourceAccessor(object): cache_file.write(block) block = fp.read(block_size) cache_file.close() + else: + vollog.debug(f"Using already cached file at: {temp_filename}") # Re-open the cache with a different mode # Since we don't want people thinking they're able to save to the cache file, # open it in read mode only and allow breakages to happen if they wanted to write diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py new file mode 100644 index 000000000..c62712907 --- /dev/null +++ b/volatility3/framework/plugins/linux/psaux.py @@ -0,0 +1,111 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from typing import Optional + +from volatility3.framework.configuration import requirements +from volatility3.framework import symbols, exceptions, renderers, interfaces +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist +from volatility3.framework.interfaces import plugins + +class PsAux(plugins.PluginInterface): + """ Lists processes with their command line arguments """ + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + description = 'Filter on specific process IDs', + element_type = int, + optional = True) + ] + + def _get_command_line_args(self, task: interfaces.objects.ObjectInterface, + name: str) -> Optional[str]: + """ + Reads the command line arguments of a process + These are stored on the userland stack + Kernel threads re-use the process data structure, but do not have a valid 'mm' pointer + + Parameters: + task: task_struct object of the process + name: string name of the process (from task.comm) + """ + + # kernel theads never have an mm as they do not have userland mappings + try: + mm = task.mm + except exceptions.InvalidAddressException: + mm = None + + if mm: + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + return renderers.UnreadableValue() + + proc_layer = self.context.layers[proc_layer_name] + + # read argv from userland + start = task.mm.arg_start + + # get the size of the arguments with sanity checking + size_to_read = task.mm.arg_end - task.mm.arg_start + if not (0 < size_to_read <= 4096): + return renderers.UnreadableValue() + + # attempt to read it all as partial values are invalid and misleading + try: + argv = proc_layer.read(start, size_to_read) + except exceptions.InvalidAddressException: + return renderers.UnreadableValue() + + # the arguments are null byte terminated, replace the nulls with spaces + s = argv.decode().split('\x00') + args = " ".join(s) + else: + # kernel thread + # [ ] mimics ps on a live system + # also helps identify malware masquerading as a kernel thread, which is fairly common + args = "[" + name + "]" + + # remove trailing space, if present + if len(args) > 1 and args[-1] == " ": + args = args[:-1] + + return args + + def _generator(self, tasks): + """ Generates a listing of processes along with command line arguments """ + + # walk the process list and report the arguments + for task in tasks: + pid = task.pid + + try: + ppid = task.parent.pid + except exceptions.InvalidAddressException: + ppid = 0 + + name = utility.array_to_string(task.comm) + + args = self._get_command_line_args(task, name) + + yield (0, (pid, ppid, name, args)) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + + return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], + self._generator( + pslist.PsList.list_tasks(self.context, + self.config['kernel'], + filter_func = filter_func))) + diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0c5ce395c..d59a95db5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,10 +1,10 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import List, Tuple, Iterator +from typing import Iterator, List, Tuple from volatility3 import framework -from volatility3.framework import exceptions, constants, interfaces, objects +from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -29,7 +29,9 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('files_struct', extensions.files_struct) self.set_type_class('vfsmount', extensions.vfsmount) self.set_type_class('kobject', extensions.kobject) - self.set_type_class('mnt_namespace', extensions.mnt_namespace) + + if 'mnt_namespace' in self.types: + self.set_type_class('mnt_namespace', extensions.mnt_namespace) if 'module' in self.types: self.set_type_class('module', extensions.module) @@ -267,4 +269,4 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): while list_start: list_struct = vmlinux.object(object_type = struct_name, offset = list_start.vol.offset) yield list_struct - list_start = getattr(list_struct, list_member) \ No newline at end of file + list_start = getattr(list_struct, list_member) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 69e8ba94e..b5ee272a0 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -746,7 +746,10 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): trans_layer = self._context.layers[layer] try: - trans_layer.is_valid(self.vol.offset) + is_valid = trans_layer.is_valid(self.vol.offset) + if not is_valid: + return + link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: return @@ -761,9 +764,7 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): while link.vol.offset not in seen: obj_offset = link.vol.offset - relative_offset - try: - trans_layer.is_valid(obj_offset) - except exceptions.InvalidAddressException: + if not trans_layer.is_valid(obj_offset): return obj = self._context.object(symbol_type,