From d4585be7cf4b6fbce85a6fa7a5b448b604036f86 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jul 2024 15:21:08 +0100 Subject: [PATCH 001/110] Linux: Update vmayarascan to scan complete VMA blocks --- .../framework/plugins/linux/vmayarascan.py | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index eda0d7dca..3edbe7784 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -33,9 +33,6 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) ), - requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) - ), requirements.ModuleRequirement( name="kernel", description="Linux kernel", @@ -69,19 +66,29 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): # get the proc_layer object from the context proc_layer = self.context.layers[proc_layer_name] - # scan the process layer with the yarascanner - for offset, rule_name, name, value in proc_layer.scan( - context=self.context, - scanner=yarascan.YaraScanner(rules=rules), - sections=self.get_vma_maps(task), - ): - yield 0, ( - format_hints.Hex(offset), - task.tgid, - rule_name, - name, - value, - ) + for start, end in self.get_vma_maps(task): + for match in rules.match( + data=proc_layer.read(start, end - start, True) + ): + if yarascan.YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield 0, ( + format_hints.Hex(instance.offset + start), + task.UniqueProcessId, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield 0, ( + format_hints.Hex(offset + start), + task.tgid, + match.rule, + name, + value, + ) @staticmethod def get_vma_maps( From bca4cdda744120a5ca868bdfd39119ebf0819201 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Fri, 19 Jul 2024 11:09:52 +0200 Subject: [PATCH 002/110] replace yara-python with yara-x --- doc/requirements.txt | 2 +- test/requirements-testing.txt | 2 +- volatility3/framework/plugins/yarascan.py | 74 ++++++----------------- 3 files changed, 22 insertions(+), 56 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index b715e59f5..c04f71219 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,6 +3,6 @@ sphinx>=4.0.0,<7 sphinx_autodoc_typehints>=1.4.0 sphinx-rtd-theme>=0.4.3 -yara-python +yara-x pycryptodome pefile diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index 7afe19b94..0955d1939 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -5,6 +5,6 @@ pefile>=2017.8.1 #foo # If certain packages are not necessary, place a comment (#) at the start of the line. # This is required for the yara plugins -yara-python>=3.8.0 +yara-x>=0.5.0 pytest>=7.0.0 diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 496ec1844..32a400f9e 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -14,14 +14,9 @@ from volatility3.framework.renderers import format_hints vollog = logging.getLogger(__name__) try: - import yara + import yara_x - if tuple([int(x) for x in yara.__version__.split(".")]) < (3, 8): - raise ImportError except ImportError: - vollog.info( - "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available" - ) raise @@ -34,27 +29,20 @@ class YaraScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules - self.st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( - 4, - 3, - ) def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: - for match in self._rules.match(data=data): - if YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: - yield ( - instance.offset + data_offset, - match.rule, - match_string.identifier, - instance.matched_data, - ) - else: - for offset, name, value in match.strings: - yield (offset + data_offset, match.rule, name, value) + results = self._rules.scan(data) + for match in results.matching_rules: + for match_string in match.patterns: + for instance in match_string.matches: + yield ( + instance.offset + data_offset, + f"{match.namespace}.{match.identifier}", + match_string.identifier, + data[instance.offset : instance.offset + instance.length], + ) class YaraScan(plugins.PluginInterface): @@ -63,9 +51,6 @@ class YaraScan(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 3, 0) - # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string - # or something that makes more sense - @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: """Returns the requirements needed to run yarascan directly, combining the TranslationLayerRequirement @@ -99,16 +84,13 @@ class YaraScan(plugins.PluginInterface): optional=True, ), requirements.StringRequirement( - name="yara_rules", description="Yara rules (as a string)", optional=True + name="yara_string", + description="Yara rules (as a string)", + optional=True, ), - requirements.URIRequirement( - name="yara_file", description="Yara rules (as a file)", optional=True - ), - # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code - # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later requirements.URIRequirement( name="yara_compiled_file", - description="Yara compiled rules (as a file)", + description="Yara-x compiled rules (as a file)", optional=True, ), requirements.IntRequirement( @@ -119,36 +101,20 @@ class YaraScan(plugins.PluginInterface): ), ] - @classmethod - def yara_returns_instances(cls) -> bool: - st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( - 4, - 3, - ) - return st_object - @classmethod def process_yara_options(cls, config: Dict[str, Any]): rules = None - if config.get("yara_rules", None) is not None: - rule = config["yara_rules"] + if config.get("yara_string") is not None: + rule = config["yara_string"] if rule[0] not in ["{", "/"]: rule = f'"{rule}"' if config.get("case", False): rule += " nocase" if config.get("wide", False): rule += " wide ascii" - rules = yara.compile( - sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} - ) - elif config.get("yara_source", None) is not None: - rules = yara.compile(source=config["yara_source"]) - elif config.get("yara_file", None) is not None: - rules = yara.compile( - file=resources.ResourceAccessor().open(config["yara_file"], "rb") - ) - elif config.get("yara_compiled_file", None) is not None: - rules = yara.load( + rules = yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") + elif config.get("yara_compiled_file") is not None: + rules = yara_x.Rules.deserialize_from( file=resources.ResourceAccessor().open( config["yara_compiled_file"], "rb" ) From ce9832a2b3d524637a7bfaa49c7726ff234c121e Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Mon, 22 Jul 2024 13:58:33 +0200 Subject: [PATCH 003/110] both yara-python and yara-x support --- doc/requirements.txt | 1 + test/requirements-testing.txt | 1 + volatility3/framework/plugins/yarascan.py | 101 ++++++++++++++++++++-- 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index c04f71219..d3ba51224 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,6 +3,7 @@ sphinx>=4.0.0,<7 sphinx_autodoc_typehints>=1.4.0 sphinx-rtd-theme>=0.4.3 +yara-python yara-x pycryptodome pefile diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index 0955d1939..51c8f602c 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -5,6 +5,7 @@ pefile>=2017.8.1 #foo # If certain packages are not necessary, place a comment (#) at the start of the line. # This is required for the yara plugins +yara-python>=3.8.0 yara-x>=0.5.0 pytest>=7.0.0 diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 32a400f9e..03a45b8df 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -13,14 +13,73 @@ from volatility3.framework.renderers import format_hints vollog = logging.getLogger(__name__) +USE_YARA_X = False + try: import yara_x + USE_YARA_X = True + except ImportError: - raise + try: + import yara + + if tuple(int(x) for x in yara.__version__.split(".")) < (3, 8): + raise ImportError + + vollog.info("Using yara-python module") + + except ImportError: + vollog.info( + "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available" + ) + raise -class YaraScanner(interfaces.layers.ScannerInterface): +class YaraPythonScanner(interfaces.layers.ScannerInterface): + _version = (2, 0, 0) + + # yara.Rules isn't exposed, so we can't type this properly + def __init__(self, rules) -> None: + super().__init__() + if rules is None: + raise ValueError("No rules provided to YaraScanner") + self._rules = rules + self.st_object = not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) + + def __call__( + self, data: bytes, data_offset: int + ) -> Iterable[Tuple[int, str, str, bytes]]: + for match in self._rules.match(data=data): + if self.st_object: + for match_string in match.strings: + for instance in match_string.instances: + yield ( + instance.offset + data_offset, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield (offset + data_offset, match.rule, name, value) + + @staticmethod + def get_rule(rule): + return yara.compile( + sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} + ) + + @staticmethod + def from_compiled_file(filepath): + return yara.load(file=resources.ResourceAccessor().open(filepath, "rb")) + + @staticmethod + def from_file(filepath): + return yara.compile(file=resources.ResourceAccessor().open(filepath, "rb")) + + +class YaraXScanner(interfaces.layers.ScannerInterface): _version = (2, 0, 0) # yara.Rules isn't exposed, so we can't type this properly @@ -44,6 +103,25 @@ class YaraScanner(interfaces.layers.ScannerInterface): data[instance.offset : instance.offset + instance.length], ) + @staticmethod + def get_rule(rule): + return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") + + @staticmethod + def from_compiled_file(filepath): + return yara_x.Rules.deserialize_from( + file=resources.ResourceAccessor().open(filepath, "rb") + ) + + @staticmethod + def from_file(filepath): + return yara_x.compile( + resources.ResourceAccessor().open(filepath, "rb").read().decode() + ) + + +YaraScanner = YaraXScanner if USE_YARA_X else YaraPythonScanner + class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" @@ -88,9 +166,14 @@ class YaraScan(plugins.PluginInterface): description="Yara rules (as a string)", optional=True, ), + requirements.URIRequirement( + name="yara_file", + description="Yara rules (as a file)", + optional=True, + ), requirements.URIRequirement( name="yara_compiled_file", - description="Yara-x compiled rules (as a file)", + description="Yara compiled rules (as a file)", optional=True, ), requirements.IntRequirement( @@ -112,13 +195,15 @@ class YaraScan(plugins.PluginInterface): rule += " nocase" if config.get("wide", False): rule += " wide ascii" - rules = yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") + rules = YaraScanner.get_rule(rule) + elif config.get("yara_file") is not None: + vollog.debug(f"Plain file: {config["yara_file"]} - yara-x: {USE_YARA_X}") + rules = YaraScanner.from_file(config["yara_file"]) elif config.get("yara_compiled_file") is not None: - rules = yara_x.Rules.deserialize_from( - file=resources.ResourceAccessor().open( - config["yara_compiled_file"], "rb" - ) + vollog.debug( + f"Compiled file: {config["yara_compiled_file"]} - yara-x: {USE_YARA_X}" ) + rules = YaraScanner.from_compiled_file(config["yara_compiled_file"]) else: vollog.error("No yara rules, nor yara rules file were specified") return rules From af2d6206763a661ced9687f1c7b31b7888d3d0ce Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 23 Jul 2024 22:02:34 +0200 Subject: [PATCH 004/110] Improving lsof --- volatility3/framework/plugins/linux/lsof.py | 53 +++++++++++++++++-- .../framework/symbols/linux/__init__.py | 21 +++++++- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index d970ad8a9..f9aeafe14 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -3,7 +3,7 @@ # """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" -import logging +import logging, datetime from typing import List, Callable from volatility3.framework import renderers, interfaces, constants @@ -76,14 +76,59 @@ class Lsof(plugins.PluginInterface): ) for pid, task_comm, _task, fd_fields in fds_generator: - fd_num, _filp, full_path = fd_fields + ( + fd_num, + _filp, + full_path, + inode_num, + imode, + ctime, + mtime, + atime, + file_size, + ) = fd_fields - fields = (pid, task_comm, fd_num, full_path) + fields = ( + pid, + task_comm, + fd_num, + full_path, + inode_num, + imode, + ctime, + mtime, + atime, + file_size, + ) yield (0, fields) def run(self): pids = self.config.get("pid", None) symbol_table = self.config["kernel"] - tree_grid_args = [("PID", int), ("Process", str), ("FD", int), ("Path", str)] + tree_grid_args = [ + ("PID", int), + ("Process", str), + ("FD", int), + ("Path", str), + ("Inode", int), + ("Mode", str), + ("LastChange", datetime.datetime), + ("LastModify", datetime.datetime), + ("LastAccessed", datetime.datetime), + ("Size", int), + ] return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) + + def generate_timeline(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + for row in self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ): + _depth, row_data = row + description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[4]}"' + yield description, timeliner.TimeLinerType.CHANGED, row_data[5] + yield description, timeliner.TimeLinerType.MODIFIED, row_data[6] + yield description, timeliner.TimeLinerType.ACCESSED, row_data[7] diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index c4e2587f4..b9321f369 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,6 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import stat, datetime from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework @@ -265,8 +266,26 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): for fd_num, filp in enumerate(fds): if filp != 0: full_path = LinuxUtilities.path_for_file(context, task, filp) + dentry = filp.get_dentry() + if dentry != 0: + inode_object = dentry.d_inode + inode_num = inode_object.i_ino + file_size = inode_object.i_size # file size in bytes + imode = stat.filemode( + inode_object.i_mode + ) # file type & Permissions - yield fd_num, filp, full_path + # Timestamps + ctime = datetime.datetime.fromtimestamp( + inode_object.i_ctime.tv_sec + ) # last change time + mtime = datetime.datetime.fromtimestamp( + inode_object.i_mtime.tv_sec + ) # last modify time + atime = datetime.datetime.fromtimestamp( + inode_object.i_atime.tv_sec + ) # last access time + yield fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size @classmethod def mask_mods_list( From fceb79ba8b86e95919451ee29ca9fdfccd3ebc3e Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 08:34:04 +0200 Subject: [PATCH 005/110] use with context, bump release --- volatility3/framework/plugins/yarascan.py | 29 +++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 03a45b8df..7957aa0c5 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -27,11 +27,11 @@ except ImportError: if tuple(int(x) for x in yara.__version__.split(".")) < (3, 8): raise ImportError - vollog.info("Using yara-python module") + vollog.debug("Using yara-python module") except ImportError: vollog.info( - "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available" + "Neither yara-x nor yara-python (>3.8.0) module not found, plugin (and dependent plugins) not available" ) raise @@ -51,7 +51,7 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: for match in self._rules.match(data=data): - if self.st_object: + if YaraScan.yara_returns_instances(): for match_string in match.strings: for instance in match_string.instances: yield ( @@ -72,11 +72,13 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): @staticmethod def from_compiled_file(filepath): - return yara.load(file=resources.ResourceAccessor().open(filepath, "rb")) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara.load(file=fp) @staticmethod def from_file(filepath): - return yara.compile(file=resources.ResourceAccessor().open(filepath, "rb")) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara.compile(file=fp) class YaraXScanner(interfaces.layers.ScannerInterface): @@ -109,15 +111,13 @@ class YaraXScanner(interfaces.layers.ScannerInterface): @staticmethod def from_compiled_file(filepath): - return yara_x.Rules.deserialize_from( - file=resources.ResourceAccessor().open(filepath, "rb") - ) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara_x.Rules.deserialize_from(file=fp) @staticmethod def from_file(filepath): - return yara_x.compile( - resources.ResourceAccessor().open(filepath, "rb").read().decode() - ) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara_x.compile(fp.read().decode()) YaraScanner = YaraXScanner if USE_YARA_X else YaraPythonScanner @@ -127,7 +127,7 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 3, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -184,6 +184,11 @@ class YaraScan(plugins.PluginInterface): ), ] + @classmethod + def yara_returns_instances(cls) -> bool: + st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < (4, 3) + return st_object + @classmethod def process_yara_options(cls, config: Dict[str, Any]): rules = None From 1616a898ae5ddfe7369c0ea90306546abd4d929e Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 08:37:27 +0200 Subject: [PATCH 006/110] restore comment --- volatility3/framework/plugins/yarascan.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 7957aa0c5..3bf894d06 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -171,6 +171,8 @@ class YaraScan(plugins.PluginInterface): description="Yara rules (as a file)", optional=True, ), + # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code + # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later requirements.URIRequirement( name="yara_compiled_file", description="Yara compiled rules (as a file)", From c6727ffb701d31067e6617d49904614bedb63ac9 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 08:43:21 +0200 Subject: [PATCH 007/110] fix f-string for previous python release --- volatility3/framework/plugins/yarascan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 3bf894d06..8d2f242d9 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -204,11 +204,11 @@ class YaraScan(plugins.PluginInterface): rule += " wide ascii" rules = YaraScanner.get_rule(rule) elif config.get("yara_file") is not None: - vollog.debug(f"Plain file: {config["yara_file"]} - yara-x: {USE_YARA_X}") + vollog.debug(f"Plain file: {config['yara_file']} - yara-x: {USE_YARA_X}") rules = YaraScanner.from_file(config["yara_file"]) elif config.get("yara_compiled_file") is not None: vollog.debug( - f"Compiled file: {config["yara_compiled_file"]} - yara-x: {USE_YARA_X}" + f"Compiled file: {config['yara_compiled_file']} - yara-x: {USE_YARA_X}" ) rules = YaraScanner.from_compiled_file(config["yara_compiled_file"]) else: From dc8dc9b078d42e41698ff93f2e65c5ee99aa2172 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 14:37:18 +0200 Subject: [PATCH 008/110] fix yara depending plugins --- .../framework/plugins/linux/vmayarascan.py | 4 +- .../framework/plugins/windows/mftscan.py | 6 +-- .../framework/plugins/windows/vadyarascan.py | 49 +++++++++++++------ volatility3/framework/plugins/yarascan.py | 24 ++++----- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index eda0d7dca..6d055707c 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -31,10 +31,10 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) ), requirements.ModuleRequirement( name="kernel", diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 1b1a1b45e..9e6585345 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -29,7 +29,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) ), ] @@ -38,7 +38,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\\*|BAAD/"} + {"yara_string": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File @@ -197,7 +197,7 @@ class ADS(interfaces.plugins.PluginInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\\*|BAAD/"} + {"yara_string": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index dc318dd93..4a84a1285 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -33,7 +33,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="yarascan", plugin=yarascan.YaraScan, version=(1, 3, 0) + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), requirements.ListRequirement( name="pid", @@ -73,26 +73,43 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ) continue - for match in rules.match(data=layer.read(start, size, True)): - if yarascan.YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: + if not yarascan.YaraScan._yara_x: + for match in rules.match(data=layer.read(start, size, True)): + if yarascan.YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield 0, ( + format_hints.Hex(instance.offset + start), + task.UniqueProcessId, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield 0, ( + format_hints.Hex(offset + start), + task.UniqueProcessId, + match.rule, + name, + value, + ) + else: + data = layer.read(start, size, True) + results = rules.scan(data) + for match in results.matching_rules: + for match_string in match.patterns: + for instance in match_string.matches: yield 0, ( format_hints.Hex(instance.offset + start), task.UniqueProcessId, - match.rule, + f"{match.namespace}.{match.identifier}", match_string.identifier, - instance.matched_data, + data[ + instance.offset : instance.offset + + instance.length + ], ) - else: - for offset, name, value in match.strings: - yield 0, ( - format_hints.Hex(offset + start), - task.UniqueProcessId, - match.rule, - name, - value, - ) @staticmethod def get_vad_maps( diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 8d2f242d9..6a4dd9251 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -36,8 +36,8 @@ except ImportError: raise -class YaraPythonScanner(interfaces.layers.ScannerInterface): - _version = (2, 0, 0) +class BaseYaraScanner(interfaces.layers.ScannerInterface): + _version = (2, 1, 0) # yara.Rules isn't exposed, so we can't type this properly def __init__(self, rules) -> None: @@ -45,6 +45,11 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules + + +class YaraPythonScanner(BaseYaraScanner): + def __init__(self, rules) -> None: + super().__init__(rules) self.st_object = not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) def __call__( @@ -81,16 +86,7 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): return yara.compile(file=fp) -class YaraXScanner(interfaces.layers.ScannerInterface): - _version = (2, 0, 0) - - # yara.Rules isn't exposed, so we can't type this properly - def __init__(self, rules) -> None: - super().__init__() - if rules is None: - raise ValueError("No rules provided to YaraScanner") - self._rules = rules - +class YaraXScanner(BaseYaraScanner): def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: @@ -128,6 +124,7 @@ class YaraScan(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (2, 0, 0) + _yara_x = USE_YARA_X @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -188,8 +185,7 @@ class YaraScan(plugins.PluginInterface): @classmethod def yara_returns_instances(cls) -> bool: - st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < (4, 3) - return st_object + return not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) @classmethod def process_yara_options(cls, config: Dict[str, Any]): From 650dd06245918f1b14d8477eff85a743c9e42c4f Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 27 Jul 2024 16:03:59 +0200 Subject: [PATCH 009/110] Modifications following the review --- volatility3/framework/plugins/linux/lsof.py | 74 +++++++++++-------- .../framework/symbols/linux/__init__.py | 51 +++++++------ 2 files changed, 70 insertions(+), 55 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index f9aeafe14..3bbc855f9 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # """A module containing a collection of plugins that produce data typically @@ -12,16 +12,17 @@ from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.framework.symbols import linux from volatility3.plugins.linux import pslist +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class Lsof(plugins.PluginInterface): +class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -46,7 +47,7 @@ class Lsof(plugins.PluginInterface): ] @classmethod - def list_fds( + def list_fds_and_inodes( cls, context: interfaces.context.ContextInterface, symbol_table: str, @@ -67,27 +68,38 @@ class Lsof(plugins.PluginInterface): ) for fd_fields in fd_generator: - yield pid, task_comm, task, fd_fields + fd_num, filp, full_path = fd_fields + inode_metadata = linux.LinuxUtilities.get_inode_metadata(context, filp) + try: + inode_num, file_size, imode, ctime, mtime, atime = next( + inode_metadata + ) + except Exception as e: + vollog.warning( + f"Can't get inode metadata for file descriptor {fd_num}: {e}" + ) + continue + yield pid, task_comm, task, fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size def _generator(self, pids, symbol_table): filter_func = pslist.PsList.create_pid_filter(pids) - fds_generator = self.list_fds( + fds_generator = self.list_fds_and_inodes( self.context, symbol_table, filter_func=filter_func ) - - for pid, task_comm, _task, fd_fields in fds_generator: - ( - fd_num, - _filp, - full_path, - inode_num, - imode, - ctime, - mtime, - atime, - file_size, - ) = fd_fields - + for ( + pid, + task_comm, + task, + fd_num, + filp, + full_path, + inode_num, + imode, + ctime, + mtime, + atime, + file_size, + ) in fds_generator: fields = ( pid, task_comm, @@ -113,22 +125,20 @@ class Lsof(plugins.PluginInterface): ("Path", str), ("Inode", int), ("Mode", str), - ("LastChange", datetime.datetime), - ("LastModify", datetime.datetime), - ("LastAccessed", datetime.datetime), + ("Changed", datetime.datetime), + ("Modified", datetime.datetime), + ("Accessed", datetime.datetime), ("Size", int), ] return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) def generate_timeline(self): + pids = self.config.get("pid", None) + symbol_table = self.config["kernel"] filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - for row in self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ): + for row in self._generator(pids, symbol_table): _depth, row_data = row - description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[4]}"' - yield description, timeliner.TimeLinerType.CHANGED, row_data[5] - yield description, timeliner.TimeLinerType.MODIFIED, row_data[6] - yield description, timeliner.TimeLinerType.ACCESSED, row_data[7] + description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' + yield description, timeliner.TimeLinerType.CHANGED, row_data[6] + yield description, timeliner.TimeLinerType.MODIFIED, row_data[7] + yield description, timeliner.TimeLinerType.ACCESSED, row_data[8] diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index b9321f369..2b97bc5f4 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,9 +1,8 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import stat, datetime from typing import Iterator, List, Tuple, Optional, Union - +import logging, datetime, stat from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility @@ -62,7 +61,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 0) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -266,26 +265,32 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): for fd_num, filp in enumerate(fds): if filp != 0: full_path = LinuxUtilities.path_for_file(context, task, filp) - dentry = filp.get_dentry() - if dentry != 0: - inode_object = dentry.d_inode - inode_num = inode_object.i_ino - file_size = inode_object.i_size # file size in bytes - imode = stat.filemode( - inode_object.i_mode - ) # file type & Permissions - # Timestamps - ctime = datetime.datetime.fromtimestamp( - inode_object.i_ctime.tv_sec - ) # last change time - mtime = datetime.datetime.fromtimestamp( - inode_object.i_mtime.tv_sec - ) # last modify time - atime = datetime.datetime.fromtimestamp( - inode_object.i_atime.tv_sec - ) # last access time - yield fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size + yield fd_num, filp, full_path + + @classmethod + def get_inode_metadata(cls, context: interfaces.context.ContextInterface, filp): + """ + A helper function that gets the inodes metadata from a file descriptor + """ + dentry = filp.get_dentry() + if dentry != 0: + inode_object = dentry.d_inode + inode_num = inode_object.i_ino + file_size = inode_object.i_size # file size in bytes + imode = stat.filemode(inode_object.i_mode) # file type & Permissions + + # Timestamps + ctime = datetime.datetime.fromtimestamp( + inode_object.i_ctime.tv_sec + ) # last change time + mtime = datetime.datetime.fromtimestamp( + inode_object.i_mtime.tv_sec + ) # last modify time + atime = datetime.datetime.fromtimestamp( + inode_object.i_atime.tv_sec + ) # last access time + yield inode_num, file_size, imode, ctime, mtime, atime @classmethod def mask_mods_list( From 7024588076adf95c2d6667c851cddf87e8c68555 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 27 Jul 2024 16:09:52 +0200 Subject: [PATCH 010/110] Code clean --- volatility3/framework/plugins/linux/lsof.py | 1 - volatility3/framework/symbols/linux/__init__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 3bbc855f9..98a215ecf 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -135,7 +135,6 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): def generate_timeline(self): pids = self.config.get("pid", None) symbol_table = self.config["kernel"] - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for row in self._generator(pids, symbol_table): _depth, row_data = row description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 2b97bc5f4..1b3f75e98 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # from typing import Iterator, List, Tuple, Optional, Union -import logging, datetime, stat +import datetime, stat from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility From 815252c9ba31913d22e8836183828059217c4e9d Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 27 Jul 2024 16:35:22 +0200 Subject: [PATCH 011/110] Adding watchdogs --- volatility3/framework/plugins/linux/lsof.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 98a215ecf..c1de48c1a 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -78,7 +78,13 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.warning( f"Can't get inode metadata for file descriptor {fd_num}: {e}" ) - continue + # Yield NotAvailableValue for each field in case of an exception + inode_num = renderers.NotAvailableValue() + file_size = renderers.NotAvailableValue() + imode = renderers.NotAvailableValue() + ctime = renderers.NotAvailableValue() + mtime = renderers.NotAvailableValue() + atime = renderers.NotAvailableValue() yield pid, task_comm, task, fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size def _generator(self, pids, symbol_table): From 59fec176aa5377e7324c6996770d27a46fdebd9a Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 29 Jul 2024 18:49:48 -0500 Subject: [PATCH 012/110] Fix bugs in thrdscan and threads. Add orphan kernel threads plugin --- .../plugins/windows/orphan_kernel_threads.py | 95 +++++++++++++++++++ .../framework/plugins/windows/thrdscan.py | 2 +- .../framework/plugins/windows/threads.py | 12 +-- 3 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 volatility3/framework/plugins/windows/orphan_kernel_threads.py diff --git a/volatility3/framework/plugins/windows/orphan_kernel_threads.py b/volatility3/framework/plugins/windows/orphan_kernel_threads.py new file mode 100644 index 000000000..013ba98ff --- /dev/null +++ b/volatility3/framework/plugins/windows/orphan_kernel_threads.py @@ -0,0 +1,95 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List, Generator + +from volatility3.framework import interfaces, symbols +from volatility3.framework.configuration import requirements +from volatility3.plugins.windows import thrdscan, ssdt + +vollog = logging.getLogger(__name__) + + +class Threads(thrdscan.ThrdScan): + """Lists process threads""" + + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.implementation = self.list_orphan_kernel_threads + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="thrdscan", plugin=thrdscan.ThrdScan, version=(1, 1, 0) + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), + ] + + @classmethod + def list_orphan_kernel_threads( + cls, + context: interfaces.context.ContextInterface, + module_name: str, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Yields thread objects of kernel threads that do not map to a module + + Args: + kernel + + Returns: + A generator of thread objects of orphaned threads + """ + module = context.modules[module_name] + layer_name = module.layer_name + symbol_table = module.symbol_table_name + + collection = ssdt.SSDT.build_module_collection( + context, layer_name, symbol_table + ) + + # used to filter out smeared pointers + if symbols.symbol_table_is_64bit(context, symbol_table): + kernel_start = 0xFFFFF80000000000 + else: + kernel_start = 0x80000000 + + for thread in thrdscan.ThrdScan.scan_threads(context, module_name): + # we don't want smeared or terminated threads + try: + proc = thread.owning_process() + except AttributeError: + continue + + # we only care about kernel threads, 4 = System + # previous methods for determining if a thread was a kernel thread + # such as bit fields and flags are not stable in Win10+ + # so we check if the thread is from the kernel itself or one its child + # kernel processes (MemCompression, Regsitry, ...) + if proc.UniqueProcessId != 4 and proc.InheritedFromUniqueProcessId != 4: + continue + + if thread.StartAddress < kernel_start: + continue + + module_symbols = list( + collection.get_module_symbols_by_absolute_location(thread.StartAddress) + ) + + # alert on threads that do not map to a module + if not module_symbols: + yield thread + diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 6e664e052..ad885e8e9 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -22,8 +22,8 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) _version = (1, 1, 0) def __init__(self, *args, **kwargs): - self.implementation = self.scan_threads super().__init__(*args, **kwargs) + self.implementation = self.scan_threads @classmethod def get_requirements(cls): diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index ae70e717b..98a3169a5 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -18,9 +18,9 @@ class Threads(thrdscan.ThrdScan): _required_framework_version = (2, 4, 0) _version = (1, 0, 0) - def __init__(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.implementation = self.list_process_threads - super().__init__() @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -50,7 +50,6 @@ class Threads(thrdscan.ThrdScan): Args: proc: _EPROCESS object from which to list the VADs - filter_func: Function to take a virtual address descriptor value and return True if it should be filtered out Returns: A list of threads based on the process and filtered based on the filter function @@ -64,22 +63,19 @@ class Threads(thrdscan.ThrdScan): seen.add(thread.vol.offset) yield thread - @classmethod - def filter_func(cls, config: interfaces.configuration.HierarchicalDict) -> Callable: - return pslist.PsList.create_pid_filter(config.get("pid", None)) - @classmethod def list_process_threads( cls, context: interfaces.context.ContextInterface, module_name: str, - filter_func: Callable, ) -> Iterable[interfaces.objects.ObjectInterface]: """Runs through all processes and lists threads for each process""" module = context.modules[module_name] layer_name = module.layer_name symbol_table_name = module.symbol_table_name + filter_func = pslist.PsList.create_pid_filter(context.config.get("pid", None)) + for proc in pslist.PsList.list_processes( context=context, layer_name=layer_name, From 896b40bd22028f8e68d954a9f9080d564c6982e2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 29 Jul 2024 18:50:55 -0500 Subject: [PATCH 013/110] Fix bugs in thrdscan and threads. Add orphan kernel threads plugin --- volatility3/framework/plugins/windows/orphan_kernel_threads.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/orphan_kernel_threads.py b/volatility3/framework/plugins/windows/orphan_kernel_threads.py index 013ba98ff..1cdb1dcdf 100644 --- a/volatility3/framework/plugins/windows/orphan_kernel_threads.py +++ b/volatility3/framework/plugins/windows/orphan_kernel_threads.py @@ -92,4 +92,3 @@ class Threads(thrdscan.ThrdScan): # alert on threads that do not map to a module if not module_symbols: yield thread - From 90b8b5340f9bc886e9b45ecadd80b7d86b8384f3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 1 Aug 2024 11:46:10 +1000 Subject: [PATCH 014/110] Linux: Add inode, timespec, and timespec64 object extensions to support different kernel versions, ensuring we will get aware datetimes when using them. --- .../framework/symbols/linux/__init__.py | 6 + .../symbols/linux/extensions/__init__.py | 132 ++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index c4e2587f4..03353135d 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -29,12 +29,18 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) + self.set_type_class("inode", extensions.inode) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) + # kernels >= 4.18 + self.optional_set_type_class("timespec64", extensions.timespec64) + # kernels < 4.18. Reuses timespec64 obj extension, since both has the same members + self.optional_set_type_class("timespec", extensions.timespec64) + # Mount self.set_type_class("vfsmount", extensions.vfsmount) # Might not exist in older kernels or the current symbols diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e7c6b66d7..be31e298c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -4,10 +4,13 @@ import collections.abc import logging +import stat +from datetime import datetime import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List from volatility3.framework import constants, exceptions, objects, interfaces, symbols +from volatility3.framework import renderers from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1761,3 +1764,132 @@ class kernel_cap_t(kernel_cap_struct): ) return cap_value & self.get_kernel_cap_full() + + +class timespec64(objects.StructType): + def to_datetime(self) -> datetime: + """Returns the respective aware datetime""" + + dt = renderers.conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) + return dt + + +class inode(objects.StructType): + def is_valid(self) -> bool: + # i_count is a 'signed' counter (atomic_t). Smear, or essentially a wrong inode + # pointer, will easily cause an integer overflow here. + return self.i_ino > 0 and self.i_count.counter >= 0 + + def is_dir(self) -> bool: + """Returns True if the inode is a directory""" + return stat.S_ISDIR(self.i_mode) != 0 + + def is_reg(self) -> bool: + """Returns True if the inode is a regular file""" + return stat.S_ISREG(self.i_mode) != 0 + + def is_link(self) -> bool: + """Returns True if the inode is a symlink""" + return stat.S_ISLNK(self.i_mode) != 0 + + def is_fifo(self) -> bool: + """Returns True if the inode is a FIFO""" + return stat.S_ISFIFO(self.i_mode) != 0 + + def is_sock(self) -> bool: + """Returns True if the inode is a socket""" + return stat.S_ISSOCK(self.i_mode) != 0 + + def is_block(self) -> bool: + """Returns True if the inode is a block device""" + return stat.S_ISBLK(self.i_mode) != 0 + + def is_char(self) -> bool: + """Returns True if the inode is a char device""" + return stat.S_ISCHR(self.i_mode) != 0 + + def is_sticky(self) -> bool: + """Returns True if the sticky bit is set""" + return (self.i_mode & stat.S_ISVTX) != 0 + + def get_inode_type(self) -> str: + """Returns inode type name + + Returns: + The inode type name + """ + if self.is_dir(): + return "DIR" + elif self.is_reg(): + return "REG" + elif self.is_link(): + return "LNK" + elif self.is_fifo(): + return "FIFO" + elif self.is_sock(): + return "SOCK" + elif self.is_char(): + return "CHR" + elif self.is_block(): + return "BLK" + else: + return renderers.UnparsableValue() + + def get_inode_number(self) -> int: + """Returns the inode number""" + return int(self.i_ino) + + def ___time_member_to_datetime(self, member) -> datetime: + if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): + # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 + # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 + return renderers.conversion.unixtime_to_datetime( + self.member(f"{member}_sec") + self.has_member(f"{member}_nsec") / 1e9 + ) + elif self.has_member(f"__{member}"): + # 6.6 <= kernels < 6.11 it's a timespec64 + # Ref Linux commit 13bc24457850583a2e7203ded05b7209ab4bc5ef / 12cd44023651666bd44baa36a5c999698890debb + return self.member(f"__{member}").to_datetime() + elif self.has_member(member): + # In kernels < 6.6 it's a timespec64 or timespec + return self.member(member).to_datetime() + else: + raise exceptions.VolatilityException( + "Unsupported kernel inode type implementation" + ) + + def get_access_time(self) -> datetime: + """Returns the inode's last access time + This is updated when inode contents are read + + Returns: + A datetime with the inode's last access time + """ + return self.___time_member_to_datetime("i_atime") + + def get_modification_time(self) -> datetime: + """Returns the inode's last modification time + This is updated when the inode contents change + + Returns: + A datetime with the inode's last data modification time + """ + + return self.___time_member_to_datetime("i_mtime") + + def get_change_time(self) -> datetime: + """Returns the inode's last change time + This is updated when the inode metadata changes + + Returns: + A datetime with the inode's last change time + """ + return self.___time_member_to_datetime("i_ctime") + + def get_file_mode(self) -> str: + """Returns the inode's file mode as string of the form '-rwxrwxrwx'. + + Returns: + The inode's file mode string + """ + return stat.filemode(self.i_mode) From 1dcaf9c0b15dfd00f2b35846488f19f3982ae5b2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 13:50:51 +1000 Subject: [PATCH 015/110] PR review fixes: Rename method name from private to internal --- .../framework/symbols/linux/extensions/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index be31e298c..599fedb6f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1839,7 +1839,7 @@ class inode(objects.StructType): """Returns the inode number""" return int(self.i_ino) - def ___time_member_to_datetime(self, member) -> datetime: + def _time_member_to_datetime(self, member) -> datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 @@ -1865,7 +1865,7 @@ class inode(objects.StructType): Returns: A datetime with the inode's last access time """ - return self.___time_member_to_datetime("i_atime") + return self._time_member_to_datetime("i_atime") def get_modification_time(self) -> datetime: """Returns the inode's last modification time @@ -1875,7 +1875,7 @@ class inode(objects.StructType): A datetime with the inode's last data modification time """ - return self.___time_member_to_datetime("i_mtime") + return self._time_member_to_datetime("i_mtime") def get_change_time(self) -> datetime: """Returns the inode's last change time @@ -1884,7 +1884,7 @@ class inode(objects.StructType): Returns: A datetime with the inode's last change time """ - return self.___time_member_to_datetime("i_ctime") + return self._time_member_to_datetime("i_ctime") def get_file_mode(self) -> str: """Returns the inode's file mode as string of the form '-rwxrwxrwx'. From 930d29046ad8c1cd47d6167e84118d78a3e549a6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 14:15:01 +1000 Subject: [PATCH 016/110] PR review fixes: Avoid using renderers in core functions. --- .../framework/symbols/linux/extensions/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 599fedb6f..1b5e1d286 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,10 +7,10 @@ import logging import stat from datetime import datetime import socket as socket_module -from typing import Generator, Iterable, Iterator, Optional, Tuple, List +from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union from volatility3.framework import constants, exceptions, objects, interfaces, symbols -from volatility3.framework import renderers +from volatility3.framework.renderers import conversion from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1770,7 +1770,7 @@ class timespec64(objects.StructType): def to_datetime(self) -> datetime: """Returns the respective aware datetime""" - dt = renderers.conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) + dt = conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) return dt @@ -1812,7 +1812,7 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 - def get_inode_type(self) -> str: + def get_inode_type(self) -> Union[str, None]: """Returns inode type name Returns: @@ -1833,7 +1833,7 @@ class inode(objects.StructType): elif self.is_block(): return "BLK" else: - return renderers.UnparsableValue() + return None def get_inode_number(self) -> int: """Returns the inode number""" @@ -1843,7 +1843,7 @@ class inode(objects.StructType): if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 - return renderers.conversion.unixtime_to_datetime( + return conversion.unixtime_to_datetime( self.member(f"{member}_sec") + self.has_member(f"{member}_nsec") / 1e9 ) elif self.has_member(f"__{member}"): From efba3a1b7336d5b31f7ac5ee1d8e99d95bcd74f6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 14:17:57 +1000 Subject: [PATCH 017/110] PR review fixes: Convert inode's is_* functions to properties --- .../symbols/linux/extensions/__init__.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1b5e1d286..00f6730eb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1780,34 +1780,42 @@ class inode(objects.StructType): # pointer, will easily cause an integer overflow here. return self.i_ino > 0 and self.i_count.counter >= 0 + @property def is_dir(self) -> bool: """Returns True if the inode is a directory""" return stat.S_ISDIR(self.i_mode) != 0 + @property def is_reg(self) -> bool: """Returns True if the inode is a regular file""" return stat.S_ISREG(self.i_mode) != 0 + @property def is_link(self) -> bool: """Returns True if the inode is a symlink""" return stat.S_ISLNK(self.i_mode) != 0 + @property def is_fifo(self) -> bool: """Returns True if the inode is a FIFO""" return stat.S_ISFIFO(self.i_mode) != 0 + @property def is_sock(self) -> bool: """Returns True if the inode is a socket""" return stat.S_ISSOCK(self.i_mode) != 0 + @property def is_block(self) -> bool: """Returns True if the inode is a block device""" return stat.S_ISBLK(self.i_mode) != 0 + @property def is_char(self) -> bool: """Returns True if the inode is a char device""" return stat.S_ISCHR(self.i_mode) != 0 + @property def is_sticky(self) -> bool: """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 @@ -1818,19 +1826,19 @@ class inode(objects.StructType): Returns: The inode type name """ - if self.is_dir(): + if self.is_dir: return "DIR" - elif self.is_reg(): + elif self.is_reg: return "REG" - elif self.is_link(): + elif self.is_link: return "LNK" - elif self.is_fifo(): + elif self.is_fifo: return "FIFO" - elif self.is_sock(): + elif self.is_sock: return "SOCK" - elif self.is_char(): + elif self.is_char: return "CHR" - elif self.is_block(): + elif self.is_block: return "BLK" else: return None From d34b1ded3e673e2d311d14c8e104668d7ebac78c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 19:46:23 +1000 Subject: [PATCH 018/110] PR review fixes: Remove get_inode_number. It's better to use the type's original member name and handle the casting on the consumer side. --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 00f6730eb..05679523f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1843,10 +1843,6 @@ class inode(objects.StructType): else: return None - def get_inode_number(self) -> int: - """Returns the inode number""" - return int(self.i_ino) - def _time_member_to_datetime(self, member) -> datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 From 230ea09728dc9b756e27936544f03d43647cc0ba Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Aug 2024 17:52:51 +0200 Subject: [PATCH 019/110] Updating code following #1230 merge --- volatility3/framework/plugins/linux/lsof.py | 26 ++++++++--------- .../framework/symbols/linux/__init__.py | 28 ++++++++----------- .../symbols/linux/extensions/__init__.py | 14 ++++++++++ 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index c1de48c1a..fa9d2bf61 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -21,7 +21,6 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) @classmethod @@ -53,7 +52,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): symbol_table: str, filter_func: Callable[[int], bool] = lambda _: False, ): - linuxutils_symbol_table = None # type: ignore + linuxutils_symbol_table = None for task in pslist.PsList.list_tasks(context, symbol_table, filter_func): if linuxutils_symbol_table is None: if constants.BANG not in task.vol.type_name: @@ -71,21 +70,17 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): fd_num, filp, full_path = fd_fields inode_metadata = linux.LinuxUtilities.get_inode_metadata(context, filp) try: - inode_num, file_size, imode, ctime, mtime, atime = next( + inode_num, itype, file_size, imode, ctime, mtime, atime = next( inode_metadata ) except Exception as e: vollog.warning( f"Can't get inode metadata for file descriptor {fd_num}: {e}" ) - # Yield NotAvailableValue for each field in case of an exception - inode_num = renderers.NotAvailableValue() - file_size = renderers.NotAvailableValue() - imode = renderers.NotAvailableValue() - ctime = renderers.NotAvailableValue() - mtime = renderers.NotAvailableValue() - atime = renderers.NotAvailableValue() - yield pid, task_comm, task, fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size + inode_num = itype = file_size = imode = ctime = mtime = atime = ( + renderers.NotAvailableValue() + ) + yield pid, task_comm, task, fd_num, filp, full_path, inode_num, itype, imode, ctime, mtime, atime, file_size def _generator(self, pids, symbol_table): filter_func = pslist.PsList.create_pid_filter(pids) @@ -100,6 +95,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): filp, full_path, inode_num, + itype, imode, ctime, mtime, @@ -112,6 +108,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): fd_num, full_path, inode_num, + itype, imode, ctime, mtime, @@ -130,6 +127,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ("FD", int), ("Path", str), ("Inode", int), + ("Type", str), ("Mode", str), ("Changed", datetime.datetime), ("Modified", datetime.datetime), @@ -144,6 +142,6 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): for row in self._generator(pids, symbol_table): _depth, row_data = row description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' - yield description, timeliner.TimeLinerType.CHANGED, row_data[6] - yield description, timeliner.TimeLinerType.MODIFIED, row_data[7] - yield description, timeliner.TimeLinerType.ACCESSED, row_data[8] + yield description, timeliner.TimeLinerType.CHANGED, row_data[7] + yield description, timeliner.TimeLinerType.MODIFIED, row_data[8] + yield description, timeliner.TimeLinerType.ACCESSED, row_data[9] diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index a96fe9d2f..d52c43dae 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -280,23 +280,19 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): A helper function that gets the inodes metadata from a file descriptor """ dentry = filp.get_dentry() - if dentry != 0: + if dentry: inode_object = dentry.d_inode - inode_num = inode_object.i_ino - file_size = inode_object.i_size # file size in bytes - imode = stat.filemode(inode_object.i_mode) # file type & Permissions - - # Timestamps - ctime = datetime.datetime.fromtimestamp( - inode_object.i_ctime.tv_sec - ) # last change time - mtime = datetime.datetime.fromtimestamp( - inode_object.i_mtime.tv_sec - ) # last modify time - atime = datetime.datetime.fromtimestamp( - inode_object.i_atime.tv_sec - ) # last access time - yield inode_num, file_size, imode, ctime, mtime, atime + if inode_object and inode_object.is_valid(): + itype = inode_object.get_inode_type() or "?" + yield ( + inode_object.i_ino, + itype, + inode_object.i_size, + inode_object.get_file_mode(), + inode_object.get_change_time(), + inode_object.get_modification_time(), + inode_object.get_access_time(), + ) @classmethod def mask_mods_list( diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 05679523f..0ee6e7d95 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1820,6 +1820,16 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 + @property + def is_whiteout(self) -> bool: + """Returns True if the inode is a whiteout""" + return (self.i_mode & 0o140000) == 0o140000 + + @property + def is_overlay(self) -> bool: + """Returns True if the inode is an overlay""" + return (self.i_mode & 0o40000) == 0o40000 + def get_inode_type(self) -> Union[str, None]: """Returns inode type name @@ -1840,6 +1850,10 @@ class inode(objects.StructType): return "CHR" elif self.is_block: return "BLK" + elif self.is_whiteout: + return "WHT" + elif self.is_overlay: + return "OVL" else: return None From 60b1c49e49864ce7cb5ae3a3491b6a7e9e40eef3 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Aug 2024 18:10:11 +0200 Subject: [PATCH 020/110] removing test code --- .../framework/symbols/linux/extensions/__init__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0ee6e7d95..06d2e2bf4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1820,16 +1820,6 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 - @property - def is_whiteout(self) -> bool: - """Returns True if the inode is a whiteout""" - return (self.i_mode & 0o140000) == 0o140000 - - @property - def is_overlay(self) -> bool: - """Returns True if the inode is an overlay""" - return (self.i_mode & 0o40000) == 0o40000 - def get_inode_type(self) -> Union[str, None]: """Returns inode type name From 2e9b5b62faec7d8e7fa67dd1e9243013af7cbcb4 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Aug 2024 18:11:29 +0200 Subject: [PATCH 021/110] removing test code --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 06d2e2bf4..05679523f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1840,10 +1840,6 @@ class inode(objects.StructType): return "CHR" elif self.is_block: return "BLK" - elif self.is_whiteout: - return "WHT" - elif self.is_overlay: - return "OVL" else: return None From d4ed07f95b883183fcd28871e7fc7858e649347a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 20:51:32 +1000 Subject: [PATCH 022/110] PR review fixes: Add fixme to remember we should move wintime_to_datetime/unixtime_to_datetime out of renderers --- volatility3/framework/renderers/conversion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index 864794860..c8ddc19fd 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -11,6 +11,7 @@ from typing import Union from volatility3.framework import interfaces, renderers +# FIXME: Move wintime_to_datetime() and unixtime_to_datetime() out of renderers, possibly framework.objects.utility def wintime_to_datetime( wintime: int, ) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: From 0dfb9d8a0ff9080eef10b7505f7e1955e0f728e1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 20:57:23 +1000 Subject: [PATCH 023/110] Linux mountinfo: Add a method to yield all filesystem superblocks --- .../framework/plugins/linux/mountinfo.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index da743bb60..319c92cca 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -37,7 +37,7 @@ class MountInfo(plugins.PluginInterface): _required_framework_version = (2, 2, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -146,7 +146,7 @@ class MountInfo(plugins.PluginInterface): def _get_tasks_mountpoints( self, tasks: Iterable[interfaces.objects.ObjectInterface], - filtered_by_pids: bool, + filtered_by_pids: bool = False, ): seen_mountpoints = set() for task in tasks: @@ -184,8 +184,8 @@ class MountInfo(plugins.PluginInterface): self, tasks: Iterable[interfaces.objects.ObjectInterface], mnt_ns_ids: List[int], - mount_format: bool, - filtered_by_pids: bool, + mount_format: bool = False, + filtered_by_pids: bool = False, ) -> Iterable[Tuple[int, Tuple]]: show_filter_warning = False for task, mnt, mnt_ns_id in self._get_tasks_mountpoints( @@ -247,6 +247,31 @@ class MountInfo(plugins.PluginInterface): "Could not filter by mount namespace id. This field is not available in this kernel." ) + def get_superblocks(self): + """Yield file system superblocks based on the task's mounted filesystems. + + Yields: + super_block: Kernel's struct super_block object + """ + # No filter so that we get all the mount namespaces from all tasks + pid_filter = pslist.PsList.create_pid_filter() + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=pid_filter + ) + + seen_sb_ptr = set() + for task, mnt, _mnt_ns_id in self._get_tasks_mountpoints(tasks): + path_root = linux.LinuxUtilities.get_path_mnt(task, mnt) + if not path_root: + continue + + sb_ptr = mnt.get_mnt_sb() + if not sb_ptr or sb_ptr in seen_sb_ptr: + continue + seen_sb_ptr.add(sb_ptr) + + yield sb_ptr.dereference(), path_root + def run(self): pids = self.config.get("pids") mount_ns_ids = self.config.get("mntns") From 231f682b2769d7a5a6de0fc96008d44c06f0825c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:01:59 +1000 Subject: [PATCH 024/110] Linux: Improve mount's object extension method docstrings --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 05679523f..de5e432d3 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -47,7 +47,7 @@ class module(generic.GenericIntelProcess): ).choices except exceptions.SymbolError: vollog.debug( - f"Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" + "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" ) # set to empty dict to show that the enum was not found, and so shouldn't be searched for again self._mod_mem_type = {} @@ -936,7 +936,8 @@ class mount(objects.StructType): MNT_RELATIME: "relatime", } - def get_mnt_sb(self): + def get_mnt_sb(self) -> int: + """Returns a pointer to the super_block""" if self.has_member("mnt"): return self.mnt.mnt_sb elif self.has_member("mnt_sb"): @@ -1251,6 +1252,7 @@ class vfsmount(objects.StructType): return self._get_real_mnt().has_parent() def get_mnt_sb(self): + """Returns a pointer to the super_block""" return self.mnt_sb def get_flags_access(self) -> str: From a369a9e23cea620a791129d687bd2bbe9c4d4442 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:05:23 +1000 Subject: [PATCH 025/110] Linux: dentry object extension: Add a method to walk dentries subdirectories --- .../symbols/linux/extensions/__init__.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index de5e432d3..3bfbe168a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -820,6 +820,26 @@ class dentry(objects.StructType): current_dentry = current_dentry.d_parent return None + def get_subdirs(self) -> interfaces.objects.ObjectInterface: + """Walks dentry subdirs + + Yields: + A dentry object + """ + if self.has_member("d_sib") and self.has_member("d_children"): + # kernels >= 6.8 + walk_member = "d_sib" + list_head_member = self.d_children.first + elif self.has_member("d_child") and self.has_member("d_subdirs"): + # 2.5.0 <= kernels < 6.8 + walk_member = "d_child" + list_head_member = self.d_subdirs + else: + raise exceptions.VolatilityException("Unsupported dentry type") + + dentry_type_name = self.get_symbol_table_name() + constants.BANG + "dentry" + yield from list_head_member.to_list(dentry_type_name, walk_member) + class struct_file(objects.StructType): def get_dentry(self) -> interfaces.objects.ObjectInterface: From 41684478ad7c12c2d862ac8612f44315bc6f855b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:21:49 +1000 Subject: [PATCH 026/110] Linux: Add page cache support, including abstractions like RadixTree, XArray, and IDR, to support both older and latest kernel versions --- .../framework/symbols/linux/__init__.py | 346 +++++++++++++++++- .../symbols/linux/extensions/__init__.py | 244 +++++++++++- 2 files changed, 588 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 03353135d..248cb8d75 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,6 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import math +from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework @@ -30,6 +32,9 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) self.set_type_class("inode", extensions.inode) + self.set_type_class("idr", extensions.IDR) + self.set_type_class("address_space", extensions.address_space) + self.set_type_class("page", extensions.page) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) @@ -67,7 +72,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 0) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -425,3 +430,342 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): kernel = context.modules[kernel_module_name] return kernel + + @classmethod + def choose_kernel_tree(cls, vmlinux: interfaces.context.ModuleInterface) -> "Tree": + """Returns the appropriate tree data structure instance for the current kernel implementation. + This is used by the IDR and the PageCache to choose between the XArray and RadixTree. + + Args: + vmlinux: The kernel module object + + Returns: + The appropriate Tree instance for the current kernel + """ + address_space_type = vmlinux.get_type("address_space") + address_space_has_i_pages = address_space_type.has_member("i_pages") + i_pages_type_name = ( + address_space_type.child_template("i_pages").vol.type_name + if address_space_has_i_pages + else "" + ) + i_pages_is_xarray = i_pages_type_name.endswith(constants.BANG + "xarray") + i_pages_is_radix_tree_root = i_pages_type_name.endswith( + constants.BANG + "radix_tree_root" + ) and vmlinux.get_type("radix_tree_root").has_member("xa_head") + + if i_pages_is_xarray or i_pages_is_radix_tree_root: + return XArray(vmlinux) + else: + return RadixTree(vmlinux) + + +class Tree(ABC): + """Abstraction to support both XArray and RadixTree""" + + # Dynamic values, these will be initialized later + CHUNK_SHIFT = None + CHUNK_SIZE = None + CHUNK_MASK = None + + def __init__(self, vmlinux: interfaces.context.ModuleInterface): + self.vmlinux = vmlinux + self.vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + + self.pointer_size = self.vmlinux.get_type("pointer").size + # Dynamically work out the (XA_CHUNK|RADIX_TREE_MAP)_SHIFT values based on + # the node.slots[] array size + node_type = self.vmlinux.get_type(self.node_type_name) + slots_array_size = node_type.child_template("slots").count + + # Calculate the LSB index - 1 + self.CHUNK_SHIFT = slots_array_size.bit_length() - 1 + self.CHUNK_SIZE = 1 << self.CHUNK_SHIFT + self.CHUNK_MASK = self.CHUNK_SIZE - 1 + + @property + @abstractmethod + def node_type_name(self) -> str: + """Returns the Tree implementation node type name + + Returns: + A string with the node type name + """ + raise NotImplementedError() + + @property + def tag_internal_value(self) -> int: + """Returns the internal node flag for the tree""" + raise NotImplementedError() + + @abstractmethod + def node_is_internal(self, nodep) -> bool: + """Checks if the node is internal""" + raise NotImplementedError + + @abstractmethod + def is_node_tagged(self, nodep) -> bool: + """Checks if the node pointer is tagged""" + raise NotImplementedError + + @abstractmethod + def untag_node(self, nodep) -> int: + """Untags a node pointer""" + raise NotImplementedError + + @abstractmethod + def get_tree_height(self, treep) -> int: + """Returns the tree height""" + raise NotImplementedError + + @abstractmethod + def get_node_height(self, nodep) -> int: + """Returns the node height""" + raise NotImplementedError + + @abstractmethod + def get_head_node(self, tree) -> int: + """Returns a pointer to the tree's head""" + raise NotImplementedError + + @abstractmethod + def is_valid_node(self, nodep) -> bool: + """Validates a node pointer""" + raise NotImplementedError + + def nodep_to_node(self, nodep) -> interfaces.objects.ObjectInterface: + """Instanciates a tree node from its pointer + + Args: + nodep: Pointer to the XArray/RadixTree node + + Returns: + A XArray/RadixTree node instance + """ + node = self.vmlinux.object(self.node_type_name, offset=nodep, absolute=True) + return node + + def _slot_to_nodep(self, slot) -> int: + if self.node_is_internal(slot): + nodep = slot & ~self.tag_internal_value + else: + nodep = slot + + return nodep + + def _iter_node(self, nodep, height) -> int: + node = self.nodep_to_node(nodep) + node_slots = node.slots + for off in range(self.CHUNK_SIZE): + slot = node_slots[off] + if slot == 0: + continue + + nodep = self._slot_to_nodep(slot) + + if height == 1: + if self.is_valid_node(nodep): + yield nodep + else: + for child_node in self._iter_node(nodep, height - 1): + yield child_node + + def get_page_addresses(self, root: interfaces.objects.ObjectInterface) -> int: + """Walks the tree data structure + + Args: + root: The tree root object + + Yields: + A tree node pointer + """ + height = self.get_tree_height(root.vol.offset) + + nodep = self.get_head_node(root) + if not nodep: + return + + # Keep the internal flag before untagging it + is_internal = self.node_is_internal(nodep) + if self.is_node_tagged(nodep): + nodep = self.untag_node(nodep) + + if is_internal: + height = self.get_node_height(nodep) + + if height == 0: + if self.is_valid_node(nodep): + yield nodep + else: + for child_node in self._iter_node(nodep, height): + yield child_node + + +class XArray(Tree): + XARRAY_TAG_MASK = 3 + XARRAY_TAG_INTERNAL = 2 + + def get_tree_height(self, treep) -> int: + return 0 + + @property + def node_type_name(self) -> str: + return "xa_node" + + @property + def tag_internal_value(self) -> int: + return self.XARRAY_TAG_INTERNAL + + def get_node_height(self, nodep) -> int: + node = self.nodep_to_node(nodep) + return (node.shift / self.CHUNK_SHIFT) + 1 + + def get_head_node(self, tree) -> int: + return tree.xa_head + + def node_is_internal(self, nodep) -> bool: + return (nodep & self.XARRAY_TAG_MASK) == self.XARRAY_TAG_INTERNAL + + def is_node_tagged(self, nodep) -> bool: + return (nodep & self.XARRAY_TAG_MASK) != 0 + + def untag_node(self, nodep) -> int: + return nodep & (~self.XARRAY_TAG_MASK) + + def is_valid_node(self, nodep) -> bool: + # It should have the tag mask clear + return not self.is_node_tagged(nodep) + + +class RadixTree(Tree): + RADIX_TREE_INTERNAL_NODE = 1 + RADIX_TREE_EXCEPTIONAL_ENTRY = 2 + RADIX_TREE_ENTRY_MASK = 3 + + # Dynamic values. These will be initialized later + RADIX_TREE_INDEX_BITS = None + RADIX_TREE_MAX_PATH = None + RADIX_TREE_HEIGHT_SHIFT = None + RADIX_TREE_HEIGHT_MASK = None + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + char_bits = 8 + self.RADIX_TREE_INDEX_BITS = char_bits * self.pointer_size + self.RADIX_TREE_MAX_PATH = int( + math.ceil(self.RADIX_TREE_INDEX_BITS / float(self.CHUNK_SHIFT)) + ) + self.RADIX_TREE_HEIGHT_SHIFT = self.RADIX_TREE_MAX_PATH + 1 + self.RADIX_TREE_HEIGHT_MASK = (1 << self.RADIX_TREE_HEIGHT_SHIFT) - 1 + + if not self.vmlinux.has_type("radix_tree_root"): + # In kernels 4.20, RADIX_TREE_INTERNAL_NODE flag took RADIX_TREE_EXCEPTIONAL_ENTRY's + # value. RADIX_TREE_EXCEPTIONAL_ENTRY was removed but that's managed in is_valid_node() + # Note that the Radix Tree is still in use for IDR, even after kernels 4.20 when XArray + # mostly replace it + self.RADIX_TREE_INTERNAL_NODE = 2 + + @property + def node_type_name(self) -> str: + return "radix_tree_node" + + @property + def tag_internal_value(self) -> int: + return self.RADIX_TREE_INTERNAL_NODE + + def get_tree_height(self, treep) -> int: + try: + if self.vmlinux.get_type("radix_tree_root").has_member("height"): + # kernels < 4.7.10 + radix_tree_root = self.vmlinux.object( + "radix_tree_root", offset=treep, absolute=True + ) + return radix_tree_root.height + except exceptions.SymbolError: + pass + + # kernels >= 4.7.10 + return 0 + + def _radix_tree_maxindex(self, node, height) -> int: + """Return the maximum key which can be store into a radix tree with this height.""" + + if not self.vmlinux.has_symbol("height_to_maxindex"): + # Kernels >= 4.7 + return (self.CHUNK_SIZE << node.shift) - 1 + else: + # Kernels < 4.7 + height_to_maxindex_array = self.vmlinux.object_from_symbol( + "height_to_maxindex" + ) + maxindex = height_to_maxindex_array[height] + return maxindex + + def get_node_height(self, nodep) -> int: + node = self.nodep_to_node(nodep) + if hasattr(node, "shift"): + # 4.7 <= Kernels < 4.20 + return (node.shift / self.CHUNK_SHIFT) + 1 + elif hasattr(node, "path"): + # 3.15 <= Kernels < 4.7 + return node.path & self.RADIX_TREE_HEIGHT_MASK + elif hasattr(node, "height"): + # Kernels < 3.15 + return node.height + else: + raise exceptions.VolatilityException("Cannot find radix-tree node height") + + def get_head_node(self, tree) -> int: + return tree.rnode + + def node_is_internal(self, nodep) -> bool: + return (nodep & self.RADIX_TREE_INTERNAL_NODE) != 0 + + def is_node_tagged(self, nodep) -> bool: + return self.node_is_internal(nodep) + + def untag_node(self, nodep) -> int: + return nodep & (~self.RADIX_TREE_ENTRY_MASK) + + def is_valid_node(self, nodep) -> bool: + # In kernels 4.20, exceptional nodes were removed and internal entries took their bitmask + if self.vmlinux.has_type("radix_tree_root"): + return ( + nodep & self.RADIX_TREE_ENTRY_MASK + ) != self.RADIX_TREE_EXCEPTIONAL_ENTRY + + return True + + +class PageCache(object): + """Linux Page Cache abstraction""" + + def __init__( + self, + page_cache: interfaces.objects.ObjectInterface, + vmlinux: interfaces.context.ModuleInterface, + ): + """ + Args: + page_cache: Page cache address space + vmlinux: Kernel module object + """ + self.vmlinux = vmlinux + self._page_cache = page_cache + self._tree = LinuxUtilities.choose_kernel_tree(self.vmlinux) + + def get_cached_pages(self) -> interfaces.objects.ObjectInterface: + """Returns all page cache contents + + Yields: + Page objects + """ + + for page_addr in self._tree.get_page_addresses(self._page_cache.i_pages): + if not page_addr: + continue + + page = self.vmlinux.object("page", offset=page_addr, absolute=True) + if page: + yield page diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3bfbe168a..300ab2ed0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -4,10 +4,11 @@ import collections.abc import logging +import functools import stat from datetime import datetime import socket as socket_module -from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union +from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, Dict from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion @@ -1919,3 +1920,244 @@ class inode(objects.StructType): The inode's file mode string """ return stat.filemode(self.i_mode) + + def get_pages(self) -> interfaces.objects.ObjectInterface: + """Gets the inode's cached pages + + Yields: + The inode's cached pages + """ + if not self.i_size: + return + elif not (self.i_mapping and self.i_mapping.nrpages > 0): + return + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + page_cache = linux.PageCache(self.i_mapping.dereference(), vmlinux) + yield from page_cache.get_cached_pages() + + def get_contents(self): + """Get the inode cached pages from the page cache + + Yields: + page_index (int): The page index in the Tree. File offset is page_index * PAGE_SIZE. + page_content (str): The page content + """ + for page_obj in self.get_pages(): + page_index = int(page_obj.index) + page_content = page_obj.get_content() + yield page_index, page_content + + +class address_space(objects.StructType): + @property + def i_pages(self): + """Returns the appropriate member containing the page cache tree""" + if self.has_member("i_pages"): + # Kernel >= 4.17 + return self.member("i_pages") + elif self.has_member("page_tree"): + # Kernel < 4.17 + return self.member("page_tree") + + raise exceptions.VolatilityException("Unsupported page cache tree") + + +class page(objects.StructType): + @property + @functools.cache + def pageflags_enum(self) -> Dict: + """Returns 'pageflags' enumeration key/values + + Returns: + A dictionary with the pageflags enumeration key/values + """ + # FIXME: It would be even better to use @functools.cached_property instead, + # however, this requires Python +3.8 + try: + pageflags_enum = self._context.symbol_space.get_enumeration( + self.get_symbol_table_name() + constants.BANG + "pageflags" + ).choices + except exceptions.SymbolError: + vollog.debug( + "Unable to find pageflags enum. This can happen in kernels < 2.6.26 or wrong ISF" + ) + # set to empty dict to show that the enum was not found, and so shouldn't be searched for again + pageflags_enum = {} + + return pageflags_enum + + def flags_list(self) -> List[str]: + """Returns a list of page flags + + Returns: + List of page flags + """ + flags = [] + for name, value in self.pageflags_enum.items(): + if self.flags & (1 << value) != 0: + flags.append(name) + + return flags + + def to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current physical memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + + vmemmap_start = None + if vmlinux.has_symbol("mem_section"): + # SPARSEMEM_VMEMMAP physical memory model: memmap is virtually contiguous + if vmlinux.has_symbol("vmemmap_base"): + # CONFIG_DYNAMIC_MEMORY_LAYOUT - KASLR kernels >= 4.9 + vmemmap_start = vmlinux.object_from_symbol("vmemmap_base") + else: + # !CONFIG_DYNAMIC_MEMORY_LAYOUT + if vmlinux_layer._maxvirtaddr < 57: + # 4-Level paging -> VMEMMAP_START = __VMEMMAP_BASE_L4 + vmemmap_base_l4 = 0xFFFFEA0000000000 + vmemmap_start = vmemmap_base_l4 + else: + # 5-Level paging -> VMEMMAP_START = __VMEMMAP_BASE_L5 + vmemmap_base_l5 = 0xFFD4000000000000 + vmemmap_start = vmemmap_base_l5 + + # FIXME: Remove this exception once 5-level paging is supported. + raise exceptions.VolatilityException( + "5-level paging is not yet supported" + ) + + elif vmlinux.has_symbol("mem_map"): + # FLATMEM physical memory model, typically 32bit + vmemmap_start = vmlinux.object_from_symbol("mem_map") + + elif vmlinux.has_symbol("node_data"): + raise exceptions.VolatilityException("NUMA systems are not yet supported") + else: + raise exceptions.VolatilityException("Unsupported Linux memory model") + + if not vmemmap_start: + raise exceptions.VolatilityException( + "Something went wrong, we shouldn't be here" + ) + + page_type_size = vmlinux.get_type("page").size + pagec = vmlinux_layer.canonicalize(self.vol.offset) + pfn = (pagec - vmemmap_start) // page_type_size + page_paddr = pfn * vmlinux_layer.page_size + + return page_paddr + + def get_content(self) -> Union[str, None]: + """Returns the page content + + Returns: + The page content + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + physical_layer = vmlinux.context.layers["memory_layer"] + page_paddr = self.to_paddr() + if not page_paddr: + return + + page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) + return page_data + + +class IDR(objects.StructType): + IDR_BITS = 8 + IDR_MASK = (1 << IDR_BITS) - 1 + INT_SIZE = 4 + MAX_IDR_SHIFT = INT_SIZE * 8 - 1 + MAX_IDR_BIT = 1 << MAX_IDR_SHIFT + + def idr_max(self, num_layers: int) -> int: + """Returns the maximum ID which can be allocated given idr::layers + + Args: + num_layers: Number of layers + + Returns: + Maximum ID for a given number of layers + """ + # Kernel < 4.17 + bits = min([self.INT_SIZE, num_layers * self.IDR_BITS, self.MAX_IDR_SHIFT]) + + return (1 << bits) - 1 + + def idr_find(self, idr_id: int) -> int: + """Finds an ID within the IDR data structure. + Based on idr_find_slowpath(), 3.9 <= Kernel < 4.11 + Args: + idr_id: The IDR element ID + + Returns: + A pointer to the given ID element + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if not vmlinux.get_type("idr_layer").has_member("layer"): + vollog.info( + "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" + ) + return + + if idr_id < 0: + return + + cur_layer = self.top + if not cur_layer: + return + + n = (cur_layer.layer + 1) * self.IDR_BITS + + if idr_id > self.idr_max(cur_layer.layer + 1): + return + + assert n != 0 + + while n > 0 and cur_layer: + n -= self.IDR_BITS + assert n == cur_layer.layer * self.IDR_BITS + cur_layer = cur_layer.ary[(idr_id >> n) & self.IDR_MASK] + + return cur_layer.v() + + def _old_kernel_get_page_addresses(self, in_use) -> int: + # Kernels < 4.11 + total = next_id = 0 + while total < in_use: + page_addr = self.idr_find(next_id) + if page_addr: + yield page_addr + total += 1 + + next_id += 1 + + def _new_kernel_get_page_addresses(self, _in_use) -> int: + # Kernels >= 4.11 + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + tree = linux.LinuxUtilities.choose_kernel_tree(vmlinux) + for page_addr in tree.get_page_addresses(root=self.idr_rt): + yield page_addr + + def get_page_addresses(self, in_use=0) -> int: + """Walks the IDR and yield a pointer associated with each element. + + Args: + in_use (int, optional): _description_. Defaults to 0. + + Yields: + A pointer associated with each element. + """ + if self.has_member("idr_rt"): + get_page_addresses_func = self._new_kernel_get_page_addresses + else: + get_page_addresses_func = self._old_kernel_get_page_addresses + + for page_addr in get_page_addresses_func(in_use): + yield page_addr From ac27d6663a3733e002cb064afd155139103c4c43 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:26:29 +1000 Subject: [PATCH 027/110] Linux: Add two page cache plugins, linux.pagecache.Files and linux.pagecache.InodePages --- .../framework/plugins/linux/pagecache.py | 504 ++++++++++++++++++ 1 file changed, 504 insertions(+) create mode 100644 volatility3/framework/plugins/linux/pagecache.py diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py new file mode 100644 index 000000000..545c243e0 --- /dev/null +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -0,0 +1,504 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import math +import logging +import datetime +from dataclasses import dataclass, astuple +from typing import List + +from volatility3.framework import renderers, interfaces +from volatility3.framework.renderers import format_hints +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements +from volatility3.plugins import timeliner +from volatility3.plugins.linux import mountinfo + +vollog = logging.getLogger(__name__) + + +@dataclass +class InodeUser: + """Inode user representation, featuring augmented information and formatted fields. + This is the data the plugin will eventually display. + """ + + superblock_addr: int + mountpoint: str + device: str + inode_num: int + inode_addr: int + type: str + inode_pages: int + cached_pages: int + file_mode: str + access_time: str + modification_time: str + change_time: str + path: str + + +@dataclass +class InodeInternal: + """Inode internal representation containing only the core objects + + Fields: + superblock: 'super_block' struct + mountpoint: Superblock mountpoint path + inode: 'inode' struct + path: Dentry full path + """ + + superblock: interfaces.objects.ObjectInterface + mountpoint: str + inode: interfaces.objects.ObjectInterface + path: str + + def to_user( + self, kernel_layer: interfaces.layers.TranslationLayerInterface + ) -> InodeUser: + """Augment the inode information to be presented to the user + + Args: + kernel_layer: The kernel layer to obtain the page size + + Returns: + An InodeUser dataclass + """ + # Ensure all types are atomic immutable. Otherwise, astuple() will take a long + # time doing a deepcopy of the Volatility objects. + superblock_addr = self.superblock.vol.offset + device = f"{self.superblock.major}:{self.superblock.minor}" + inode_num = int(self.inode.i_ino) + inode_addr = self.inode.vol.offset + inode_type = renderers.UnparsableValue() + # Round up the number of pages to fit the inode's size + inode_pages = int(math.ceil(self.inode.i_size / float(kernel_layer.page_size))) + cached_pages = int(self.inode.i_mapping.nrpages) + file_mode = self.inode.get_file_mode() + access_time_dt = self.inode.get_access_time() + modification_time_str = self.inode.get_modification_time() + change_time_str = self.inode.get_change_time() + + inode_user = InodeUser( + superblock_addr=superblock_addr, + mountpoint=self.mountpoint, + device=device, + inode_num=inode_num, + inode_addr=inode_addr, + type=inode_type, + inode_pages=inode_pages, + cached_pages=cached_pages, + file_mode=file_mode, + access_time=access_time_dt, + modification_time=modification_time_str, + change_time=change_time_str, + path=self.path, + ) + return inode_user + + +class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): + """Lists files from memory""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 1, 0) + ), + requirements.ListRequirement( + name="type", + description="List of space-separated file type filters i.e. --type REG DIR", + element_type=str, + optional=True, + ), + requirements.StringRequirement( + name="find", + description="Filename (full path) to find", + optional=True, + ), + ] + + @staticmethod + def _follow_symlink(inode, symlink_path) -> str: + """Follows (fast) symlinks (kernels >= 4.2.x). + Fast symlinks are filesystem agnostic. + + Args: + inode: The inode (or pointer) to dump + symlink_path: The symlink name + + Returns: + If it can resolve the symlink, it returns a string "symlink_path -> target_path" + Otherwise, it returns the same symlink_path + """ + # i_link (fast symlinks) were introduced in 4.2 + if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: + i_link_str = inode.i_link.dereference().cast( + "string", max_length=255, encoding="utf-8", errors="replace" + ) + symlink_path = f"{symlink_path} -> {i_link_str}" + + return symlink_path + + @classmethod + def _walk_dentry(cls, seen_dentries, root_dentry, parent): + + for dentry in root_dentry.get_subdirs(): + dentry_addr = dentry.vol.offset + + # corruption + if dentry_addr == root_dentry.vol.offset: + continue + + if dentry_addr in seen_dentries: + continue + + seen_dentries.add(dentry_addr) + + inode = dentry.d_inode + if not (inode and inode.is_valid()): + continue + + # This allows us to have consistent paths + if dentry.d_name.name: + name = dentry.d_name.name_as_str() + # Do NOT use os.path.join() below + new_file = parent + "/" + name + else: + continue + + yield new_file, dentry, dentry.d_parent.vol.offset + + if inode.is_dir: + for new_file, dentry, parent_address in cls._walk_dentry( + seen_dentries, dentry, new_file + ): + yield new_file, dentry, parent_address + + @classmethod + def get_inodes( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + ): + """Retrieves the inodes from the superblocks + + Args: + context: The context that the plugin will operate within + config_path: The path to configuration data within the context configuration data + + Yields: + An InodeInternal object + """ + + superblocks_iter = mountinfo.MountInfo( + context=context, + config_path=config_path, + ).get_superblocks() + + seen_inodes = set() + seen_dentries = set() + for superblock, mountpoint in superblocks_iter: + parent = "" if mountpoint == "/" else mountpoint + + # Superblock root dentry + root_dentry = superblock.s_root + if not root_dentry: + continue + + # Dentry sanity check + if not root_dentry.is_root(): + continue + + # More dentry/inode sanity checks + root_inode_ptr = root_dentry.d_inode + if not root_inode_ptr: + continue + root_inode = root_inode_ptr.dereference() + if not root_inode.is_valid(): + continue + + # Inode already processed? + if root_inode_ptr in seen_inodes: + continue + seen_inodes.add(root_inode_ptr) + + root_path = mountpoint + + inode_in = InodeInternal( + superblock=superblock, + mountpoint=mountpoint, + inode=root_inode, + path=root_path, + ) + yield inode_in + + # Children + for file_path, file_dentry, _ in cls._walk_dentry( + seen_dentries, root_dentry, parent + ): + if not file_dentry: + continue + # Dentry/inode sanity checks + file_inode_ptr = file_dentry.d_inode + if not file_inode_ptr: + continue + file_inode = file_inode_ptr.dereference() + if not file_inode.is_valid(): + continue + + # Inode already processed? + if file_inode_ptr in seen_inodes: + continue + seen_inodes.add(file_inode_ptr) + + file_path = cls._follow_symlink(file_inode_ptr, file_path) + inode_in = InodeInternal( + superblock=superblock, + mountpoint=mountpoint, + inode=file_inode, + path=file_path, + ) + yield inode_in + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + inodes_iter = self.get_inodes( + context=self.context, config_path=self.config_path + ) + + types_filter = self.config["type"] + for inode_in in inodes_iter: + if types_filter and inode_in.inode.get_inode_type() not in types_filter: + continue + + if self.config["find"]: + if inode_in.path == self.config["find"]: + inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) + break # Only the first match + else: + inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) + + def generate_timeline(self): + """Generates tuples of (description, timestamp_type, timestamp) + + These need not be generated in any particular order, sorting + will be done later + """ + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + inodes_iter = self.get_inodes( + context=self.context, config_path=self.config_path + ) + for inode_in in inodes_iter: + inode_out = inode_in.to_user(vmlinux_layer) + description = f"Cached Inode for {inode_out.path}" + yield description, timeliner.TimeLinerType.ACCESSED, inode_out.access_time + yield description, timeliner.TimeLinerType.MODIFIED, inode_out.modification_time + yield description, timeliner.TimeLinerType.CHANGE, inode_out.change_time + + @staticmethod + def format_fields_with_headers(headers, generator): + """Uses the headers type to cast the fields obtained from the generator""" + for level, fields in generator: + formatted_fields = [] + for header, field in zip(headers, fields): + header_type = header[1] + + if isinstance( + field, (header_type, interfaces.renderers.BaseAbsentValue) + ): + formatted_field = field + else: + formatted_field = header_type(field) + + formatted_fields.append(formatted_field) + yield level, formatted_fields + + def run(self): + headers = [ + ("SuperblockAddr", format_hints.Hex), + ("MountPoint", str), + ("Device", str), + ("InodeNum", int), + ("InodeAddr", format_hints.Hex), + ("FileType", str), + ("InodePages", int), + ("CachedPages", int), + ("FileMode", str), + ("AccessTime", datetime.datetime), + ("ModificationTime", datetime.datetime), + ("ChangeTime", datetime.datetime), + ("FilePath", str), + ] + + return renderers.TreeGrid( + headers, self.format_fields_with_headers(headers, self._generator()) + ) + + +class InodePages(plugins.PluginInterface): + """Lists and recovers cached inode pages""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="files", plugin=Files, version=(1, 0, 0) + ), + requirements.StringRequirement( + name="find", + description="Filename (full path) to find ", + optional=True, + ), + requirements.IntRequirement( + name="inode", + description="Inode address", + optional=True, + ), + requirements.StringRequirement( + name="dump", + description="Output file path", + optional=True, + ), + ] + + @staticmethod + def write_inode_content_to_file( + inode: interfaces.objects.ObjectInterface, + filename: str, + vmlinux_layer: interfaces.layers.TranslationLayerInterface, + ) -> None: + """Extracts the inode's contents from the page cache and saves them to a file + + Args: + inode: The inode to dump + filename: Filename for writing the inode content + vmlinux_layer: The kernel layer to obtain the page size + """ + if not inode.is_reg: + vollog.error("The inode is not a regular file") + return + + # By using truncate/seek, provided the filesystem supports it, a sparse file will be + # created, saving both disk space and I/O time. + # Additionally, using the page index will guarantee that each page is written at the + # appropriate file position. + try: + with open(filename, "wb") as f: + inode_size = inode.i_size + f.truncate(inode_size) + + for page_idx, page_content in inode.get_contents(): + current_fp = page_idx * vmlinux_layer.page_size + max_length = inode_size - current_fp + page_bytes = page_content[:max_length] + if current_fp + len(page_bytes) > inode_size: + vollog.error( + "Page out of file bounds: inode 0x%x, inode size %d, page index %d", + inode.vol.object, + inode_size, + page_idx, + ) + f.seek(current_fp) + f.write(page_bytes) + + except IOError as e: + vollog.error("Unable to write to file (%s): %s", filename, e) + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + if self.config["inode"] and self.config["find"]: + vollog.error("Cannot use --inode and --find simultaneously") + return + + if self.config["find"]: + inodes_iter = Files.get_inodes( + context=self.context, config_path=self.config_path + ) + for inode_in in inodes_iter: + if inode_in.path == self.config["find"]: + inode = inode_in.inode + break # Only the first match + + elif self.config["inode"]: + inode = vmlinux.object("inode", self.config["inode"], absolute=True) + else: + vollog.error("You must use either --inode or --find") + return + + if not inode.is_reg: + vollog.error("The inode is not a regular file") + return + + inode_size = inode.i_size + if not inode.is_valid(): + vollog.error("Invalid inode at 0x%x", self.config["inode"]) + return + + for page_obj in inode.get_pages(): + page_vaddr = page_obj.vol.offset + page_paddr = page_obj.to_paddr() + page_mapping_addr = page_obj.mapping + page_index = int(page_obj.index) + page_file_offset = page_index * vmlinux_layer.page_size + dump_safe = page_file_offset < inode_size + page_flags_list = page_obj.get_flags() + page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) + fields = ( + page_vaddr, + page_paddr, + page_mapping_addr, + page_index, + dump_safe, + page_flags, + ) + + yield 0, fields + + if self.config["dump"]: + filename = self.config["dump"] + vollog.info("[*] Writing inode at 0x%x to '%s'", inode.vol.offset, filename) + self.write_inode_content_to_file(inode, filename, vmlinux_layer) + + def run(self): + headers = [ + ("PageVAddr", format_hints.Hex), + ("PagePAddr", format_hints.Hex), + ("MappingAddr", format_hints.Hex), + ("Index", int), + ("DumpSafe", bool), + ("Flags", str), + ] + + return renderers.TreeGrid( + headers, Files.format_fields_with_headers(headers, self._generator()) + ) From 55212008f805abdd47f2f3d7d6211c198097f8d4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:39:51 +1000 Subject: [PATCH 028/110] Linux: Add pidhashtable plugin. This is based on the vol2 plugin, removing ancient kernel support, curating code and enhancing comments, while using the new IDR abstraction included also in this effort. --- .../framework/plugins/linux/pidhashtable.py | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 volatility3/framework/plugins/linux/pidhashtable.py diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py new file mode 100644 index 000000000..b24c73e77 --- /dev/null +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -0,0 +1,249 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List + +from volatility3.framework import renderers, interfaces, constants +from volatility3.framework.symbols import linux +from volatility3.framework.renderers import format_hints +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class PIDHashTable(plugins.PluginInterface): + """Enumerates processes through the PID hash table""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + ), + requirements.BooleanRequirement( + name="decorate_comm", + description="Show `user threads` comm in curly brackets, and `kernel threads` comm in square brackets", + optional=True, + default=False, + ), + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.vmlinux = None + self.vmlinux_layer = None + + def _is_valid_task(self, task): + return task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent) + + def _get_pidtype_pid(self): + # The pid_type enumeration is present since 2.5.37, just in case + pid_type_enum = self.vmlinux.get_enumeration("pid_type") + if not pid_type_enum: + vollog.error("Cannot find pid_type enum. Unsupported kernel") + return + + pidtype_pid = pid_type_enum.choices.get("PIDTYPE_PID") + if pidtype_pid is None: + vollog.error("Cannot find PIDTYPE_PID. Unsupported kernel") + return + + # Typically PIDTYPE_PID = 0 + return pidtype_pid + + def _get_pidhash_array(self): + pidhash_shift = self.vmlinux.object_from_symbol("pidhash_shift") + pidhash_size = 1 << pidhash_shift + + array_type_name = self.vmlinux.symbol_table_name + constants.BANG + "array" + + pidhash_ptr = self.vmlinux.object_from_symbol("pid_hash") + # pidhash is an array of hlist_heads + pidhash = self._context.object( + array_type_name, + offset=pidhash_ptr, + subtype=self.vmlinux.get_type("hlist_head"), + count=pidhash_size, + layer_name=self.vmlinux.layer_name, + ) + + return pidhash + + def _walk_upid(self, seen_upids, upid): + while upid and self.vmlinux_layer.is_valid(upid.vol.offset): + if upid.vol.offset in seen_upids: + break + seen_upids.add(upid.vol.offset) + + pid_chain = upid.pid_chain + if not (pid_chain and self.vmlinux_layer.is_valid(pid_chain.vol.offset)): + break + + upid = linux.LinuxUtilities.container_of( + pid_chain.next, "upid", "pid_chain", self.vmlinux + ) + + def _get_upids(self): + # 2.6.24 <= kernels < 4.15 + pidhash = self._get_pidhash_array() + + seen_upids = set() + for hlist in pidhash: + # each entry in the hlist is a upid which is wrapped in a pid + ent = hlist.first + + while ent and self.vmlinux_layer.is_valid(ent.vol.offset): + # upid->pid_chain exists 2.6.24 <= kernel < 4.15 + upid = linux.LinuxUtilities.container_of( + ent.vol.offset, "upid", "pid_chain", self.vmlinux + ) + + if upid.vol.offset in seen_upids: + break + + self._walk_upid(seen_upids, upid) + + ent = ent.next + + return seen_upids + + def _pid_hash_implementation(self): + # 2.6.24 <= kernels < 4.15 + task_pids_off = self.vmlinux.get_type("task_struct").relative_child_offset( + "pids" + ) + pidtype_pid = self._get_pidtype_pid() + + for upid in self._get_upids(): + pid = linux.LinuxUtilities.container_of( + upid, "pid", "numbers", self.vmlinux + ) + if not pid: + continue + + pid_tasks_0 = pid.tasks[pidtype_pid].first + if not pid_tasks_0: + continue + + task = self.vmlinux.object( + "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True + ) + if self._is_valid_task(task): + yield task + + def _task_for_radix_pid_node(self, nodep): + # kernels >= 4.15 + pid = self.vmlinux.object("pid", offset=nodep, absolute=True) + pidtype_pid = self._get_pidtype_pid() + + pid_tasks_0 = pid.tasks[pidtype_pid].first + if not pid_tasks_0: + return + + task_struct_type = self.vmlinux.get_type("task_struct") + if task_struct_type.has_member("pids"): + member = "pids" + elif task_struct_type.has_member("pid_links"): + member = "pid_links" + else: + return None + + task_pids_off = task_struct_type.relative_child_offset(member) + task = self.vmlinux.object( + "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True + ) + return task + + def _pid_namespace_idr(self): + # kernels >= 4.15 + ns_addr = self.vmlinux.get_symbol("init_pid_ns").address + ns = self.vmlinux.object("pid_namespace", offset=ns_addr) + + for page_addr in ns.idr.get_page_addresses(): + task = self._task_for_radix_pid_node(page_addr) + if self._is_valid_task(task): + yield task + + def _determine_pid_func(self): + pid_hash = self.vmlinux.has_symbol("pid_hash") and self.vmlinux.has_symbol( + "pidhash_shift" + ) # 2.5.55 <= kernels < 4.15 + + has_pid_numbers = self.vmlinux.has_type("pid") and self.vmlinux.get_type( + "pid" + ).has_member( + "numbers" + ) # kernels >= 2.6.24 + + has_pid_numbers = self.vmlinux.has_type("upid") and self.vmlinux.get_type( + "upid" + ).has_member( + "pid_chain" + ) # 2.6.24 <= kernels < 4.15 + + # kernels >= 4.15 + pid_idr = self.vmlinux.has_type("pid_namespace") and self.vmlinux.get_type( + "pid_namespace" + ).has_member("idr") + + if pid_idr: + # kernels >= 4.15 + return self._pid_namespace_idr + elif pid_hash and has_pid_numbers and has_pid_numbers: + # 2.6.24 <= kernels < 4.15 + return self._pid_hash_implementation + + return None + + def get_tasks(self) -> interfaces.objects.ObjectInterface: + """Enumerates processes through the PID hash table + + Yields: + task_struct objects + """ + self.vmlinux = self.context.modules[self.config["kernel"]] + self.vmlinux_layer = self.context.layers[self.vmlinux.layer_name] + pid_func = self._determine_pid_func() + if not pid_func: + vollog.error("Cannot determine which PID hash table this kernel is using") + return + + yield from sorted(pid_func(), key=lambda t: (t.tgid, t.pid)) + + def _generator( + self, decorate_comm: bool = False + ) -> interfaces.objects.ObjectInterface: + for task in self.get_tasks(): + offset, pid, tid, ppid, name = pslist.PsList.get_task_fields( + task, decorate_comm + ) + fields = format_hints.Hex(offset), pid, tid, ppid, name + yield 0, fields + + def run(self): + decorate_comm = self.config.get("decorate_comm") + + headers = [ + ("OFFSET", format_hints.Hex), + ("PID", int), + ("TID", int), + ("PPID", int), + ("COMM", str), + ] + return renderers.TreeGrid(headers, self._generator(decorate_comm=decorate_comm)) From 103537801ee0b49ca3475be11e1fe62670938e91 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:44:38 +1000 Subject: [PATCH 029/110] Linux: Add a basic eBPF program enumeration plugin to test and demonstrate using the IDR abstraction --- volatility3/framework/plugins/linux/ebpf.py | 78 +++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 volatility3/framework/plugins/linux/ebpf.py diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py new file mode 100644 index 000000000..33ba71faf --- /dev/null +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -0,0 +1,78 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import binascii +import logging +from typing import List + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements + +vollog = logging.getLogger(__name__) + + +class EBPF(plugins.PluginInterface): + """Enumerate eBPF programs""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + ] + + def get_ebpf_programs(self, vmlinux) -> interfaces.objects.ObjectInterface: + """Enumerate eBPF programs walking its IDR. + + Args: + vmlinux: The kernel symbols object + + Yields: + eBPF program objects + """ + if not vmlinux.has_symbol("prog_idr"): + raise exceptions.VolatilityException( + "Cannot find the eBPF prog idr. Unsupported kernel" + ) + + prog_idr_addr = vmlinux.get_symbol("prog_idr").address + prog_idr = vmlinux.object("idr", offset=prog_idr_addr) + for page_addr in prog_idr.get_page_addresses(): + bpf_prog = vmlinux.object("bpf_prog", offset=page_addr, absolute=True) + yield bpf_prog + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + bpf_prog_types = vmlinux.get_enumeration("bpf_prog_type") + for prog in self.get_ebpf_programs(vmlinux): + prog_addr = prog.vol.offset + prog_type = bpf_prog_types.lookup(prog.type) + prog_tag_addr = prog.tag.vol.offset + prog_tag_size = prog.tag.count + prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) + prog_tag = binascii.hexlify(prog_tag_bytes).decode() + prog_name = ( + utility.array_to_string(prog.aux.name) or renderers.NotAvailableValue() + ) + fields = (format_hints.Hex(prog_addr), prog_name, prog_tag, prog_type) + yield (0, fields) + + def run(self): + headers = [ + ("Address", format_hints.Hex), + ("Name", str), + ("Tag", str), + ("Type", str), + ] + return renderers.TreeGrid(headers, self._generator()) From cc04f665e35989fea4c108de2bdd1b31016145ed Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 23:47:28 +1000 Subject: [PATCH 030/110] Fix inode type --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 545c243e0..c36f8a339 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -72,7 +72,7 @@ class InodeInternal: device = f"{self.superblock.major}:{self.superblock.minor}" inode_num = int(self.inode.i_ino) inode_addr = self.inode.vol.offset - inode_type = renderers.UnparsableValue() + inode_type = self.inode.get_inode_type() or renderers.UnparsableValue() # Round up the number of pages to fit the inode's size inode_pages = int(math.ceil(self.inode.i_size / float(kernel_layer.page_size))) cached_pages = int(self.inode.i_mapping.nrpages) From 3e75c2ae9d29084485f6d2803c64d2c42e064ee0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 4 Aug 2024 14:40:45 +1000 Subject: [PATCH 031/110] Fix @functools.cache . It's available since Python 3.9 --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 300ab2ed0..2d48f677b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1965,7 +1965,7 @@ class address_space(objects.StructType): class page(objects.StructType): @property - @functools.cache + @functools.lru_cache() def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values From 8d6fd3cd78f0fadd223ae93a70a268048b4ccfe9 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 5 Aug 2024 14:28:43 +0200 Subject: [PATCH 032/110] Moved get_inode_metadata, separated inode and FD processing, error handling precision --- volatility3/framework/plugins/linux/lsof.py | 65 ++++++++++++------- .../framework/symbols/linux/__init__.py | 26 +------- 2 files changed, 45 insertions(+), 46 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index fa9d2bf61..167556e7d 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -6,7 +6,7 @@ found in Linux's /proc file system.""" import logging, datetime from typing import List, Callable -from volatility3.framework import renderers, interfaces, constants +from volatility3.framework import renderers, interfaces, constants, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -46,7 +46,30 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ] @classmethod - def list_fds_and_inodes( + def get_inode_metadata(cls, filp: interfaces.objects.ObjectInterface): + try: + dentry = filp.get_dentry() + if dentry: + inode_object = dentry.d_inode + if inode_object and inode_object.is_valid(): + itype = ( + inode_object.get_inode_type() or renderers.NotAvailableValue() + ) + return ( + inode_object.i_ino, + itype, + inode_object.i_size, + inode_object.get_file_mode(), + inode_object.get_change_time(), + inode_object.get_modification_time(), + inode_object.get_access_time(), + ) + except (exceptions.InvalidAddressException, AttributeError) as e: + vollog.warning(f"Can't get inode metadata: {e}") + return tuple(renderers.NotAvailableValue() for _ in range(7)) + + @classmethod + def list_fds( cls, context: interfaces.context.ContextInterface, symbol_table: str, @@ -67,26 +90,27 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ) for fd_fields in fd_generator: - fd_num, filp, full_path = fd_fields - inode_metadata = linux.LinuxUtilities.get_inode_metadata(context, filp) - try: - inode_num, itype, file_size, imode, ctime, mtime, atime = next( - inode_metadata - ) - except Exception as e: - vollog.warning( - f"Can't get inode metadata for file descriptor {fd_num}: {e}" - ) - inode_num = itype = file_size = imode = ctime = mtime = atime = ( - renderers.NotAvailableValue() - ) - yield pid, task_comm, task, fd_num, filp, full_path, inode_num, itype, imode, ctime, mtime, atime, file_size + yield pid, task_comm, task, fd_fields + + @classmethod + def list_fds_and_inodes( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + filter_func: Callable[[int], bool] = lambda _: False, + ): + for pid, task_comm, task, (fd_num, filp, full_path) in cls.list_fds( + context, symbol_table, filter_func + ): + inode_metadata = cls.get_inode_metadata(filp) + yield pid, task_comm, task, fd_num, filp, full_path, inode_metadata def _generator(self, pids, symbol_table): filter_func = pslist.PsList.create_pid_filter(pids) fds_generator = self.list_fds_and_inodes( self.context, symbol_table, filter_func=filter_func ) + for ( pid, task_comm, @@ -94,14 +118,9 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): fd_num, filp, full_path, - inode_num, - itype, - imode, - ctime, - mtime, - atime, - file_size, + inode_metadata, ) in fds_generator: + inode_num, itype, file_size, imode, ctime, mtime, atime = inode_metadata fields = ( pid, task_comm, diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index d52c43dae..03353135d 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,8 +1,8 @@ -# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # from typing import Iterator, List, Tuple, Optional, Union -import datetime, stat + from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility @@ -67,7 +67,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 2, 0) + _version = (2, 1, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -274,26 +274,6 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): yield fd_num, filp, full_path - @classmethod - def get_inode_metadata(cls, context: interfaces.context.ContextInterface, filp): - """ - A helper function that gets the inodes metadata from a file descriptor - """ - dentry = filp.get_dentry() - if dentry: - inode_object = dentry.d_inode - if inode_object and inode_object.is_valid(): - itype = inode_object.get_inode_type() or "?" - yield ( - inode_object.i_ino, - itype, - inode_object.i_size, - inode_object.get_file_mode(), - inode_object.get_change_time(), - inode_object.get_modification_time(), - inode_object.get_access_time(), - ) - @classmethod def mask_mods_list( cls, From c9eb81c95fa530c58e493d77cde506f001e9e4f3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 04:40:48 -0700 Subject: [PATCH 033/110] PR review fixes: Improve eBPF extension objects: bpf_prog and added bpf_prog_aux. Apply changes to the EBPF and Sockstat plugins. --- volatility3/framework/plugins/linux/ebpf.py | 15 ++---- .../framework/plugins/linux/sockstat.py | 14 +++--- .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 48 +++++++++++++++++-- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 33ba71faf..8df506b06 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -1,12 +1,10 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import binascii import logging from typing import List from volatility3.framework import renderers, interfaces, exceptions -from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins from volatility3.framework.configuration import requirements @@ -53,18 +51,11 @@ class EBPF(plugins.PluginInterface): def _generator(self): vmlinux = self.context.modules[self.config["kernel"]] - vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - bpf_prog_types = vmlinux.get_enumeration("bpf_prog_type") for prog in self.get_ebpf_programs(vmlinux): prog_addr = prog.vol.offset - prog_type = bpf_prog_types.lookup(prog.type) - prog_tag_addr = prog.tag.vol.offset - prog_tag_size = prog.tag.count - prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) - prog_tag = binascii.hexlify(prog_tag_bytes).decode() - prog_name = ( - utility.array_to_string(prog.aux.name) or renderers.NotAvailableValue() - ) + prog_type = prog.get_type() or renderers.NotAvailableValue() + prog_tag = prog.get_tag() or renderers.NotAvailableValue() + prog_name = prog.get_name() or renderers.NotAvailableValue() fields = (format_hints.Hex(prog_addr), prog_name, prog_tag, prog_type) yield (0, fields) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 78217fbec..b0503b105 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -151,17 +151,15 @@ class SockHandlers(interfaces.configuration.VersionableInterface): bpfprog = sock_filter.prog - BPF_PROG_TYPE_UNSPEC = 0 # cBPF filter - try: - bpfprog_type = bpfprog.get_type() - if bpfprog_type == BPF_PROG_TYPE_UNSPEC: - return # cBPF filter - except AttributeError: + bpfprog_type = bpfprog.get_type() + if not bpfprog_type: # kernel < 3.18.140, it's a cBPF filter return None - BPF_PROG_TYPE_SOCKET_FILTER = 1 # eBPF filter - if bpfprog_type != BPF_PROG_TYPE_SOCKET_FILTER: + if bpfprog_type == "BPF_PROG_TYPE_UNSPEC": + return None # cBPF filter + + if bpfprog_type != "BPF_PROG_TYPE_SOCKET_FILTER": socket_filter["bpf_filter_type"] = f"UNK({bpfprog_type})" vollog.warning(f"Unexpected BPF type {bpfprog_type} for a socket") return None diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 248cb8d75..7a87135ff 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -38,6 +38,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) + self.optional_set_type_class("bpf_prog_aux", extensions.bpf_prog_aux) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 2d48f677b..2b971fb79 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -5,6 +5,7 @@ import collections.abc import logging import functools +import binascii import stat from datetime import datetime import socket as socket_module @@ -1607,20 +1608,59 @@ class xdp_sock(objects.StructType): class bpf_prog(objects.StructType): - def get_type(self): + def get_type(self) -> Union[str, None]: + """Returns a string with the eBPF program type""" + # The program type was in `bpf_prog_aux::prog_type` from 3.18.140 to # 4.1.52 before it was moved to `bpf_prog::type` if self.has_member("type"): # kernel >= 4.1.52 - return self.type + return self.type.description if self.has_member("aux") and self.aux: if self.aux.has_member("prog_type"): # 3.18.140 <= kernel < 4.1.52 - return self.aux.prog_type + return self.aux.prog_type.description # kernel < 3.18.140 - raise AttributeError("Unable to find the BPF type") + return None + + def get_tag(self) -> Union[str, None]: + """Returns a string with the eBPF program tag""" + # 'tag' was added in kernels 4.10 + if not self.has_member("tag"): + return None + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + + prog_tag_addr = self.tag.vol.offset + prog_tag_size = self.tag.count + prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) + + prog_tag = binascii.hexlify(prog_tag_bytes).decode() + return prog_tag + + def get_name(self) -> Union[str, None]: + """Returns a string with the eBPF program name""" + if not self.has_member("aux"): + # 'prog_aux' was added in kernels 3.18 + return None + + return self.aux.get_name() + + +class bpf_prog_aux(objects.StructType): + def get_name(self) -> Union[str, None]: + """Returns a string with the eBPF program name""" + if not self.has_member("name"): + # 'name' was added in kernels 4.15 + return None + + if not self.name: + return None + + return utility.array_to_string(self.name) class cred(objects.StructType): From bd37aa3930c056fc8511969592f731a29e253c84 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 04:48:49 -0700 Subject: [PATCH 034/110] PR review fixes: Fix pidhashtable plugin --- volatility3/framework/plugins/linux/pidhashtable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index b24c73e77..ef110bdf1 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -191,7 +191,7 @@ class PIDHashTable(plugins.PluginInterface): "numbers" ) # kernels >= 2.6.24 - has_pid_numbers = self.vmlinux.has_type("upid") and self.vmlinux.get_type( + has_pid_chain = self.vmlinux.has_type("upid") and self.vmlinux.get_type( "upid" ).has_member( "pid_chain" @@ -205,7 +205,7 @@ class PIDHashTable(plugins.PluginInterface): if pid_idr: # kernels >= 4.15 return self._pid_namespace_idr - elif pid_hash and has_pid_numbers and has_pid_numbers: + elif pid_hash and has_pid_numbers and has_pid_numbers and has_pid_chain: # 2.6.24 <= kernels < 4.15 return self._pid_hash_implementation From 7df0636f30d47f1d5b9373671aac410549474766 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 04:57:33 -0700 Subject: [PATCH 035/110] PR review fixes: Fix pidhashtable plugin explicit returns mixed with implicit returns --- volatility3/framework/plugins/linux/pidhashtable.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index ef110bdf1..73cdff452 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -50,19 +50,19 @@ class PIDHashTable(plugins.PluginInterface): self.vmlinux_layer = None def _is_valid_task(self, task): - return task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent) + return bool(task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent)) def _get_pidtype_pid(self): # The pid_type enumeration is present since 2.5.37, just in case pid_type_enum = self.vmlinux.get_enumeration("pid_type") if not pid_type_enum: vollog.error("Cannot find pid_type enum. Unsupported kernel") - return + return None pidtype_pid = pid_type_enum.choices.get("PIDTYPE_PID") if pidtype_pid is None: vollog.error("Cannot find PIDTYPE_PID. Unsupported kernel") - return + return None # Typically PIDTYPE_PID = 0 return pidtype_pid @@ -154,7 +154,7 @@ class PIDHashTable(plugins.PluginInterface): pid_tasks_0 = pid.tasks[pidtype_pid].first if not pid_tasks_0: - return + return None task_struct_type = self.vmlinux.get_type("task_struct") if task_struct_type.has_member("pids"): From 339a9a94f57adb2039984facf0cd22df0ba4bf90 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 23:00:09 -0700 Subject: [PATCH 036/110] PR review fixes: pidhashtable plugin add missing typing. --- volatility3/framework/plugins/linux/pidhashtable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 73cdff452..3c429dc2f 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -49,7 +49,7 @@ class PIDHashTable(plugins.PluginInterface): self.vmlinux = None self.vmlinux_layer = None - def _is_valid_task(self, task): + def _is_valid_task(self, task) -> bool: return bool(task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent)) def _get_pidtype_pid(self): From c8cb4465da3d71a879b47d981fdd4871afd9d521 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 23:08:21 -0700 Subject: [PATCH 037/110] PR review fixes: Remove filter function, it isn't needed --- volatility3/framework/plugins/linux/mountinfo.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 319c92cca..dfb2e2f52 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -254,10 +254,7 @@ class MountInfo(plugins.PluginInterface): super_block: Kernel's struct super_block object """ # No filter so that we get all the mount namespaces from all tasks - pid_filter = pslist.PsList.create_pid_filter() - tasks = pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=pid_filter - ) + tasks = pslist.PsList.list_tasks(self.context, self.config["kernel"]) seen_sb_ptr = set() for task, mnt, _mnt_ns_id in self._get_tasks_mountpoints(tasks): From f737b88d03d9be9f4f0a9b43a83e02a88a620068 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 23:12:01 -0700 Subject: [PATCH 038/110] PR review fixes: Improve _walk_dentry() and get_inodes() variable names, arguments and return values --- .../framework/plugins/linux/pagecache.py | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index c36f8a339..cf09f23d8 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List +from typing import List, Set from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints @@ -153,7 +153,23 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): return symlink_path @classmethod - def _walk_dentry(cls, seen_dentries, root_dentry, parent): + def _walk_dentry( + cls, + seen_dentries: Set[int], + root_dentry: interfaces.objects.ObjectInterface, + parent_dir: str, + ): + """Walk dentries recursively + + Args: + seen_dentries: A set to ensure each dentry is processed only once + root_dentry: Root dentry object + parent_dir: Parent directory path + + Yields: + file_path: Filename including path + dentry: Dentry object + """ for dentry in root_dentry.get_subdirs(): dentry_addr = dentry.vol.offset @@ -173,19 +189,16 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): # This allows us to have consistent paths if dentry.d_name.name: - name = dentry.d_name.name_as_str() + basename = dentry.d_name.name_as_str() # Do NOT use os.path.join() below - new_file = parent + "/" + name + file_path = parent_dir + "/" + basename else: continue - yield new_file, dentry, dentry.d_parent.vol.offset + yield file_path, dentry if inode.is_dir: - for new_file, dentry, parent_address in cls._walk_dentry( - seen_dentries, dentry, new_file - ): - yield new_file, dentry, parent_address + yield from cls._walk_dentry(seen_dentries, dentry, parent_dir=file_path) @classmethod def get_inodes( @@ -211,13 +224,15 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): seen_inodes = set() seen_dentries = set() for superblock, mountpoint in superblocks_iter: - parent = "" if mountpoint == "/" else mountpoint + parent_dir = "" if mountpoint == "/" else mountpoint # Superblock root dentry - root_dentry = superblock.s_root - if not root_dentry: + root_dentry_ptr = superblock.s_root + if not root_dentry_ptr: continue + root_dentry = root_dentry_ptr.dereference() + # Dentry sanity check if not root_dentry.is_root(): continue @@ -246,8 +261,8 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): yield inode_in # Children - for file_path, file_dentry, _ in cls._walk_dentry( - seen_dentries, root_dentry, parent + for file_path, file_dentry in cls._walk_dentry( + seen_dentries, root_dentry, parent_dir ): if not file_dentry: continue From 805b3514c3b14c53724cd38d722540d84949d51f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 7 Aug 2024 00:23:19 -0700 Subject: [PATCH 039/110] PR review fixes: Fix page flags list method name, this was introduced earlier in another commit of this PR. --- volatility3/framework/plugins/linux/pagecache.py | 2 +- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index cf09f23d8..fd62cc1e4 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -486,7 +486,7 @@ class InodePages(plugins.PluginInterface): page_index = int(page_obj.index) page_file_offset = page_index * vmlinux_layer.page_size dump_safe = page_file_offset < inode_size - page_flags_list = page_obj.get_flags() + page_flags_list = page_obj.get_flags_list() page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) fields = ( page_vaddr, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 2b971fb79..900cc9b6c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2027,7 +2027,7 @@ class page(objects.StructType): return pageflags_enum - def flags_list(self) -> List[str]: + def get_flags_list(self) -> List[str]: """Returns a list of page flags Returns: From 46842981fb25aa6f1b242ef5990642fdeeb0cf00 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 7 Aug 2024 00:24:58 -0700 Subject: [PATCH 040/110] PR review fixes: Use contextlib.suppress() instead of an empty exception handler --- volatility3/framework/symbols/linux/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 7a87135ff..96bc56a18 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import math +import contextlib from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union @@ -676,15 +677,13 @@ class RadixTree(Tree): return self.RADIX_TREE_INTERNAL_NODE def get_tree_height(self, treep) -> int: - try: + with contextlib.suppress(exceptions.SymbolError): if self.vmlinux.get_type("radix_tree_root").has_member("height"): # kernels < 4.7.10 radix_tree_root = self.vmlinux.object( "radix_tree_root", offset=treep, absolute=True ) return radix_tree_root.height - except exceptions.SymbolError: - pass # kernels >= 4.7.10 return 0 From 10376a2687b8546df88bf6b3a3e11126f8816479 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Wed, 7 Aug 2024 10:38:29 +0200 Subject: [PATCH 041/110] use one class with flag --- README.md | 2 +- pyproject.toml | 2 +- volatility3/framework/__init__.py | 2 +- .../framework/plugins/windows/vadyarascan.py | 7 +- volatility3/framework/plugins/yarascan.py | 81 ++++++++----------- 5 files changed, 38 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 886790df8..1463c2bde 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.7.3 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.8.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/pyproject.toml b/pyproject.toml index 207f762cc..2e1636a43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, ] -requires-python = ">=3.7.3" +requires-python = ">=3.8.0" license = { text = "VSL" } dynamic = ["dependencies", "optional-dependencies", "version"] diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 74db773cf..51310bfa2 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 7, 3) +required_python_version = (3, 8, 0) if ( sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 4a84a1285..7bc3377c3 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -73,8 +73,9 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ) continue + data = layer.read(start, size, True) if not yarascan.YaraScan._yara_x: - for match in rules.match(data=layer.read(start, size, True)): + for match in rules.match(data=data): if yarascan.YaraScan.yara_returns_instances(): for match_string in match.strings: for instance in match_string.instances: @@ -95,9 +96,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): value, ) else: - data = layer.read(start, size, True) - results = rules.scan(data) - for match in results.matching_rules: + for match in rules.scan(data).matching_rules: for match_string in match.patterns: for instance in match_string.matches: yield 0, ( diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 6a4dd9251..310bbd072 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -36,7 +36,7 @@ except ImportError: raise -class BaseYaraScanner(interfaces.layers.ScannerInterface): +class YaraScanner(interfaces.layers.ScannerInterface): _version = (2, 1, 0) # yara.Rules isn't exposed, so we can't type this properly @@ -45,32 +45,44 @@ class BaseYaraScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules - - -class YaraPythonScanner(BaseYaraScanner): - def __init__(self, rules) -> None: - super().__init__(rules) - self.st_object = not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) + self.st_object = ( + None + if USE_YARA_X + else not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) + ) def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: - for match in self._rules.match(data=data): - if YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: + if USE_YARA_X: + for match in self._rules.scan(data).matching_rules: + for match_string in match.patterns: + for instance in match_string.matches: yield ( instance.offset + data_offset, - match.rule, + f"{match.namespace}.{match.identifier}", match_string.identifier, - instance.matched_data, + data[instance.offset : instance.offset + instance.length], ) - else: - for offset, name, value in match.strings: - yield (offset + data_offset, match.rule, name, value) + else: + for match in self._rules.match(data=data): + if YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield ( + instance.offset + data_offset, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield (offset + data_offset, match.rule, name, value) @staticmethod def get_rule(rule): + if USE_YARA_X: + return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") return yara.compile( sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} ) @@ -78,47 +90,18 @@ class YaraPythonScanner(BaseYaraScanner): @staticmethod def from_compiled_file(filepath): with resources.ResourceAccessor().open(filepath, "rb") as fp: + if USE_YARA_X: + return yara_x.Rules.deserialize_from(file=fp) return yara.load(file=fp) @staticmethod def from_file(filepath): with resources.ResourceAccessor().open(filepath, "rb") as fp: + if USE_YARA_X: + return yara_x.compile(fp.read().decode()) return yara.compile(file=fp) -class YaraXScanner(BaseYaraScanner): - def __call__( - self, data: bytes, data_offset: int - ) -> Iterable[Tuple[int, str, str, bytes]]: - results = self._rules.scan(data) - for match in results.matching_rules: - for match_string in match.patterns: - for instance in match_string.matches: - yield ( - instance.offset + data_offset, - f"{match.namespace}.{match.identifier}", - match_string.identifier, - data[instance.offset : instance.offset + instance.length], - ) - - @staticmethod - def get_rule(rule): - return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") - - @staticmethod - def from_compiled_file(filepath): - with resources.ResourceAccessor().open(filepath, "rb") as fp: - return yara_x.Rules.deserialize_from(file=fp) - - @staticmethod - def from_file(filepath): - with resources.ResourceAccessor().open(filepath, "rb") as fp: - return yara_x.compile(fp.read().decode()) - - -YaraScanner = YaraXScanner if USE_YARA_X else YaraPythonScanner - - class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" From de6637c9871ac16f1a5278c4831dfe60778852b9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 00:52:34 -0700 Subject: [PATCH 042/110] PR review fixes: ebpf plugin code improvement. Use the object_from_symbol() instead --- volatility3/framework/plugins/linux/ebpf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 8df506b06..9d41c0ffb 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -43,8 +43,7 @@ class EBPF(plugins.PluginInterface): "Cannot find the eBPF prog idr. Unsupported kernel" ) - prog_idr_addr = vmlinux.get_symbol("prog_idr").address - prog_idr = vmlinux.object("idr", offset=prog_idr_addr) + prog_idr = vmlinux.object_from_symbol("prog_idr") for page_addr in prog_idr.get_page_addresses(): bpf_prog = vmlinux.object("bpf_prog", offset=page_addr, absolute=True) yield bpf_prog From ee10ba8abb7b8c932d9af85a37edefc3fd03fa63 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 01:46:53 -0700 Subject: [PATCH 043/110] PR review fixes: Fix IDR explicit returns mixed with implicit returns and improve and fix code. --- .../symbols/linux/extensions/__init__.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 900cc9b6c..6eafd4173 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2134,7 +2134,7 @@ class IDR(objects.StructType): """Finds an ID within the IDR data structure. Based on idr_find_slowpath(), 3.9 <= Kernel < 4.11 Args: - idr_id: The IDR element ID + idr_id: The IDR lookup ID Returns: A pointer to the given ID element @@ -2144,28 +2144,28 @@ class IDR(objects.StructType): vollog.info( "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" ) - return + return None if idr_id < 0: - return + return None - cur_layer = self.top - if not cur_layer: - return + idr_layer = self.top + if not idr_layer: + return None - n = (cur_layer.layer + 1) * self.IDR_BITS + n = (idr_layer.layer + 1) * self.IDR_BITS - if idr_id > self.idr_max(cur_layer.layer + 1): - return + if idr_id > self.idr_max(idr_layer.layer + 1): + return None assert n != 0 - while n > 0 and cur_layer: + while n > 0 and idr_layer: n -= self.IDR_BITS - assert n == cur_layer.layer * self.IDR_BITS - cur_layer = cur_layer.ary[(idr_id >> n) & self.IDR_MASK] + assert n == idr_layer.layer * self.IDR_BITS + idr_layer = idr_layer.ary[(idr_id >> n) & self.IDR_MASK] - return cur_layer.v() + return idr_layer def _old_kernel_get_page_addresses(self, in_use) -> int: # Kernels < 4.11 From 0f3f33863370f68b7da3069db5b29282dabab932 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 01:52:57 -0700 Subject: [PATCH 044/110] PR review fixes: Since the IDR, XArray and RadixTree can store any value, it renames the function names to a more generic name --- volatility3/framework/plugins/linux/ebpf.py | 2 +- .../framework/plugins/linux/pidhashtable.py | 2 +- volatility3/framework/symbols/linux/__init__.py | 4 ++-- .../symbols/linux/extensions/__init__.py | 16 +++++++++------- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 9d41c0ffb..70082daf5 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -44,7 +44,7 @@ class EBPF(plugins.PluginInterface): ) prog_idr = vmlinux.object_from_symbol("prog_idr") - for page_addr in prog_idr.get_page_addresses(): + for page_addr in prog_idr.get_entries(): bpf_prog = vmlinux.object("bpf_prog", offset=page_addr, absolute=True) yield bpf_prog diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 3c429dc2f..c384cb5cd 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -175,7 +175,7 @@ class PIDHashTable(plugins.PluginInterface): ns_addr = self.vmlinux.get_symbol("init_pid_ns").address ns = self.vmlinux.object("pid_namespace", offset=ns_addr) - for page_addr in ns.idr.get_page_addresses(): + for page_addr in ns.idr.get_entries(): task = self._task_for_radix_pid_node(page_addr) if self._is_valid_task(task): yield task diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 96bc56a18..89f970275 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -572,7 +572,7 @@ class Tree(ABC): for child_node in self._iter_node(nodep, height - 1): yield child_node - def get_page_addresses(self, root: interfaces.objects.ObjectInterface) -> int: + def get_entries(self, root: interfaces.objects.ObjectInterface) -> int: """Walks the tree data structure Args: @@ -762,7 +762,7 @@ class PageCache(object): Page objects """ - for page_addr in self._tree.get_page_addresses(self._page_cache.i_pages): + for page_addr in self._tree.get_entries(self._page_cache.i_pages): if not page_addr: continue diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6eafd4173..29af5c510 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2167,7 +2167,7 @@ class IDR(objects.StructType): return idr_layer - def _old_kernel_get_page_addresses(self, in_use) -> int: + def _old_kernel_get_entries(self) -> int: # Kernels < 4.11 total = next_id = 0 while total < in_use: @@ -2178,14 +2178,14 @@ class IDR(objects.StructType): next_id += 1 - def _new_kernel_get_page_addresses(self, _in_use) -> int: + def _new_kernel_get_entries(self) -> int: # Kernels >= 4.11 vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) tree = linux.LinuxUtilities.choose_kernel_tree(vmlinux) - for page_addr in tree.get_page_addresses(root=self.idr_rt): + for page_addr in tree.get_entries(root=self.idr_rt): yield page_addr - def get_page_addresses(self, in_use=0) -> int: + def get_entries(self) -> int: """Walks the IDR and yield a pointer associated with each element. Args: @@ -2195,9 +2195,11 @@ class IDR(objects.StructType): A pointer associated with each element. """ if self.has_member("idr_rt"): - get_page_addresses_func = self._new_kernel_get_page_addresses + # Kernels >= 4.11 + get_entries_func = self._new_kernel_get_entries else: - get_page_addresses_func = self._old_kernel_get_page_addresses + # Kernels < 4.11 + get_entries_func = self._old_kernel_get_entries - for page_addr in get_page_addresses_func(in_use): + for page_addr in get_entries_func(): yield page_addr From 7a8dea3356c709cda2bdd20b7cb3ff97f1f7987d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 01:57:35 -0700 Subject: [PATCH 045/110] PR review fixes: Code scanning complains about these unused variables. Let's comment them and adapt the FIXME message --- volatility3/framework/symbols/linux/extensions/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 29af5c510..9b03235cb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2063,10 +2063,9 @@ class page(objects.StructType): vmemmap_start = vmemmap_base_l4 else: # 5-Level paging -> VMEMMAP_START = __VMEMMAP_BASE_L5 - vmemmap_base_l5 = 0xFFD4000000000000 - vmemmap_start = vmemmap_base_l5 - - # FIXME: Remove this exception once 5-level paging is supported. + # FIXME: Once 5-level paging is supported, uncomment the following lines and remove the exception + # vmemmap_base_l5 = 0xFFD4000000000000 + # vmemmap_start = vmemmap_base_l5 raise exceptions.VolatilityException( "5-level paging is not yet supported" ) From 06508a4afba235812e7d1b7bfb79467adf2ed6fc Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 02:00:27 -0700 Subject: [PATCH 046/110] PR review fixes: Fix the IDR's old kernel get_entries --- .../framework/symbols/linux/extensions/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9b03235cb..f7b0df6be 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2168,11 +2168,12 @@ class IDR(objects.StructType): def _old_kernel_get_entries(self) -> int: # Kernels < 4.11 + cur = self.cur total = next_id = 0 - while total < in_use: - page_addr = self.idr_find(next_id) - if page_addr: - yield page_addr + while next_id < cur: + entry = self.idr_find(next_id) + if entry: + yield entry total += 1 next_id += 1 From 2c85ea525e15c0cb745f3b806836f22ea44a4b4c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 02:03:28 -0700 Subject: [PATCH 047/110] PR review fixes: Fix page extension object get_content() explicit returns mixed with implicit returns. --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f7b0df6be..be4df6a13 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2102,7 +2102,7 @@ class page(objects.StructType): physical_layer = vmlinux.context.layers["memory_layer"] page_paddr = self.to_paddr() if not page_paddr: - return + return None page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) return page_data From f22575669a6ccd9afaeef126e81a6adad8b880f6 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 9 Aug 2024 10:20:28 +0200 Subject: [PATCH 048/110] Modifications following the review --- volatility3/framework/plugins/linux/lsof.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 167556e7d..9a0fd7417 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -66,7 +66,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ) except (exceptions.InvalidAddressException, AttributeError) as e: vollog.warning(f"Can't get inode metadata: {e}") - return tuple(renderers.NotAvailableValue() for _ in range(7)) + return None @classmethod def list_fds( @@ -103,6 +103,10 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): context, symbol_table, filter_func ): inode_metadata = cls.get_inode_metadata(filp) + if inode_metadata is None: + inode_metadata = tuple( + interfaces.renderers.BaseAbsentValue() for _ in range(7) + ) yield pid, task_comm, task, fd_num, filp, full_path, inode_metadata def _generator(self, pids, symbol_table): From 53f3d12341e722f1058c42436adfea600af94bab Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 9 Aug 2024 23:35:41 -0700 Subject: [PATCH 049/110] linuxutilities code improvement. Remove code duplication --- volatility3/framework/symbols/linux/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 89f970275..90f5cc8a2 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -419,9 +419,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): Returns: A kernel object (vmlinux) """ - symbol_table_arr = volobj.vol.type_name.split("!", 1) - symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None - + symbol_table = volobj.get_symbol_table_name() module_names = context.modules.get_modules_by_symbol_tables(symbol_table) module_names = list(module_names) From 17861618df3744d572955b50b2b1b1ad1d0961e5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 10 Aug 2024 00:13:38 -0700 Subject: [PATCH 050/110] PR review fixes: Rename Tree to IDStorage. Move choose_id_storage() form LinuxUtilities to IDStorage. Use context and kernel_module_name instead of vmlinux --- .../framework/symbols/linux/__init__.py | 87 +++++++++++-------- .../symbols/linux/extensions/__init__.py | 14 +-- 2 files changed, 59 insertions(+), 42 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 90f5cc8a2..632ac2f6b 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -431,17 +431,51 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return kernel + +class IDStorage(ABC): + """Abstraction to support both XArray and RadixTree""" + + # Dynamic values, these will be initialized later + CHUNK_SHIFT = None + CHUNK_SIZE = None + CHUNK_MASK = None + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.vmlinux = context.modules[kernel_module_name] + self.vmlinux_layer = self.vmlinux.context.layers[self.vmlinux.layer_name] + + self.pointer_size = self.vmlinux.get_type("pointer").size + # Dynamically work out the (XA_CHUNK|RADIX_TREE_MAP)_SHIFT values based on + # the node.slots[] array size + node_type = self.vmlinux.get_type(self.node_type_name) + slots_array_size = node_type.child_template("slots").count + + # Calculate the LSB index - 1 + self.CHUNK_SHIFT = slots_array_size.bit_length() - 1 + self.CHUNK_SIZE = 1 << self.CHUNK_SHIFT + self.CHUNK_MASK = self.CHUNK_SIZE - 1 + @classmethod - def choose_kernel_tree(cls, vmlinux: interfaces.context.ModuleInterface) -> "Tree": - """Returns the appropriate tree data structure instance for the current kernel implementation. + def choose_id_storage( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ) -> "IDStorage": + """Returns the appropriate ID storage data structure instance for the current kernel implementation. This is used by the IDR and the PageCache to choose between the XArray and RadixTree. Args: - vmlinux: The kernel module object + context: The context to retrieve required elements (layers, symbol tables) from + kernel_module_name: The name of the kernel module on which to operate Returns: - The appropriate Tree instance for the current kernel + The appropriate ID storage instance for the current kernel """ + vmlinux = context.modules[kernel_module_name] address_space_type = vmlinux.get_type("address_space") address_space_has_i_pages = address_space_type.has_member("i_pages") i_pages_type_name = ( @@ -455,33 +489,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ) and vmlinux.get_type("radix_tree_root").has_member("xa_head") if i_pages_is_xarray or i_pages_is_radix_tree_root: - return XArray(vmlinux) + return XArray(context, kernel_module_name) else: - return RadixTree(vmlinux) - - -class Tree(ABC): - """Abstraction to support both XArray and RadixTree""" - - # Dynamic values, these will be initialized later - CHUNK_SHIFT = None - CHUNK_SIZE = None - CHUNK_MASK = None - - def __init__(self, vmlinux: interfaces.context.ModuleInterface): - self.vmlinux = vmlinux - self.vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - - self.pointer_size = self.vmlinux.get_type("pointer").size - # Dynamically work out the (XA_CHUNK|RADIX_TREE_MAP)_SHIFT values based on - # the node.slots[] array size - node_type = self.vmlinux.get_type(self.node_type_name) - slots_array_size = node_type.child_template("slots").count - - # Calculate the LSB index - 1 - self.CHUNK_SHIFT = slots_array_size.bit_length() - 1 - self.CHUNK_SIZE = 1 << self.CHUNK_SHIFT - self.CHUNK_MASK = self.CHUNK_SIZE - 1 + return RadixTree(context, kernel_module_name) @property @abstractmethod @@ -601,7 +611,7 @@ class Tree(ABC): yield child_node -class XArray(Tree): +class XArray(IDStorage): XARRAY_TAG_MASK = 3 XARRAY_TAG_INTERNAL = 2 @@ -637,7 +647,7 @@ class XArray(Tree): return not self.is_node_tagged(nodep) -class RadixTree(Tree): +class RadixTree(IDStorage): RADIX_TREE_INTERNAL_NODE = 1 RADIX_TREE_EXCEPTIONAL_ENTRY = 2 RADIX_TREE_ENTRY_MASK = 3 @@ -741,17 +751,20 @@ class PageCache(object): def __init__( self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, page_cache: interfaces.objects.ObjectInterface, - vmlinux: interfaces.context.ModuleInterface, ): """ Args: + context: interfaces.context.ContextInterface, + kernel_module_name: The name of the kernel module on which to operate page_cache: Page cache address space - vmlinux: Kernel module object """ - self.vmlinux = vmlinux + self.vmlinux = context.modules[kernel_module_name] + self._page_cache = page_cache - self._tree = LinuxUtilities.choose_kernel_tree(self.vmlinux) + self._idstorage = IDStorage.choose_id_storage(context, kernel_module_name) def get_cached_pages(self) -> interfaces.objects.ObjectInterface: """Returns all page cache contents @@ -760,7 +773,7 @@ class PageCache(object): Page objects """ - for page_addr in self._tree.get_entries(self._page_cache.i_pages): + for page_addr in self._idstorage.get_entries(self._page_cache.i_pages): if not page_addr: continue diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index be4df6a13..d00af7a3f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1972,8 +1972,11 @@ class inode(objects.StructType): elif not (self.i_mapping and self.i_mapping.nrpages > 0): return - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - page_cache = linux.PageCache(self.i_mapping.dereference(), vmlinux) + page_cache = linux.PageCache( + context=self._context, + kernel_module_name="kernel", + page_cache=self.i_mapping.dereference(), + ) yield from page_cache.get_cached_pages() def get_contents(self): @@ -2180,9 +2183,10 @@ class IDR(objects.StructType): def _new_kernel_get_entries(self) -> int: # Kernels >= 4.11 - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - tree = linux.LinuxUtilities.choose_kernel_tree(vmlinux) - for page_addr in tree.get_entries(root=self.idr_rt): + id_storage = linux.IDStorage.choose_id_storage( + self._context, kernel_module_name="kernel" + ) + for page_addr in id_storage.get_entries(root=self.idr_rt): yield page_addr def get_entries(self) -> int: From 8f9d565f6300750ad30c0aaca0aa01cfbcb1a417 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 10 Aug 2024 00:49:18 -0700 Subject: [PATCH 051/110] PR review fixes: Fix minor typo to match verb form from other docstrings --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index fd62cc1e4..f062655a6 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -159,7 +159,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): root_dentry: interfaces.objects.ObjectInterface, parent_dir: str, ): - """Walk dentries recursively + """Walks dentries recursively Args: seen_dentries: A set to ensure each dentry is processed only once From f804b44ff6f8451e8f1091383b8d9c2300f7bfbb Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Aug 2024 01:00:57 -0700 Subject: [PATCH 052/110] Fix test.yaml, it should remove *.bin and not *.lime. There is no *.lime atm. --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 55b2e4b60..668b814ce 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,7 +46,7 @@ jobs: - name: Clean up post-test run: | - rm -rf *.lime + rm -rf *.bin rm -rf *.img cd volatility3/symbols rm -rf linux From 7efa2210a550d55f356b6ebb35cafcdfd4a58d1f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Aug 2024 01:05:44 -0700 Subject: [PATCH 053/110] PR review fixes: Adjust LinuxUtilities version since we moved choose_id_storage() back to the IDStorage class. --- volatility3/framework/plugins/linux/pidhashtable.py | 2 +- volatility3/framework/symbols/linux/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index c384cb5cd..9a2528543 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -34,7 +34,7 @@ class PIDHashTable(plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.VersionRequirement( - name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), requirements.BooleanRequirement( name="decorate_comm", diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 632ac2f6b..91abf7db4 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -74,7 +74,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 2, 0) + _version = (2, 1, 1) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 918584537fafd67b2367e283871889bf291b2951 Mon Sep 17 00:00:00 2001 From: Steven Luke <97394870+sluke-nuix@users.noreply.github.com> Date: Tue, 13 Aug 2024 08:34:35 -0400 Subject: [PATCH 054/110] Update the modules.Modules version requirement. This is a response to PR [#1173](https://github.com/volatilityfoundation/volatility3/pull/1173) --- volatility3/framework/plugins/windows/truecrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index 81250a749..7fd26cb4e 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -33,7 +33,7 @@ class Passphrase(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name="modules", component=modules.Modules, version=(1, 1, 0) + name="modules", component=modules.Modules, version=(2, 0, 0) ), requirements.IntRequirement( name="min-length", From 8d7edfdca6fa84983b4ee734e3f45a9d07c28ff1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Aug 2024 20:36:09 +0100 Subject: [PATCH 055/110] Bump as the release branch has been cut --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index f219fb0af..4df0b9041 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 8 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From 71cdca5883b234680773e000a663750964d4860e Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Thu, 22 Aug 2024 16:45:04 +0200 Subject: [PATCH 056/110] Updating version + docstring --- volatility3/framework/plugins/linux/lsof.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 9a0fd7417..360f89749 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -18,10 +18,10 @@ vollog = logging.getLogger(__name__) class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): - """Lists all memory maps for all processes.""" + """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (1, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From feae6a9aa0f5c7869174e0b906ec83e61b3e14e3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 00:42:52 +1000 Subject: [PATCH 057/110] PR review fixes: Use plugin's open method instead of the builtin open() --- volatility3/framework/plugins/linux/pagecache.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index f062655a6..cf4151c85 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set +from typing import List, Set, Type from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints @@ -408,6 +408,7 @@ class InodePages(plugins.PluginInterface): def write_inode_content_to_file( inode: interfaces.objects.ObjectInterface, filename: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a file @@ -415,6 +416,7 @@ class InodePages(plugins.PluginInterface): Args: inode: The inode to dump filename: Filename for writing the inode content + open_method: class for constructing output files vmlinux_layer: The kernel layer to obtain the page size """ if not inode.is_reg: @@ -426,7 +428,7 @@ class InodePages(plugins.PluginInterface): # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. try: - with open(filename, "wb") as f: + with open_method(filename) as f: inode_size = inode.i_size f.truncate(inode_size) @@ -502,7 +504,7 @@ class InodePages(plugins.PluginInterface): if self.config["dump"]: filename = self.config["dump"] vollog.info("[*] Writing inode at 0x%x to '%s'", inode.vol.offset, filename) - self.write_inode_content_to_file(inode, filename, vmlinux_layer) + self.write_inode_content_to_file(inode, filename, self.open, vmlinux_layer) def run(self): headers = [ From 3c70c1b9f7c2251e35eea276f5d5099b68392a48 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 00:58:04 +1000 Subject: [PATCH 058/110] PR review fixes: Use context and module_name instead of vmlinux in ebpf plugin --- volatility3/framework/plugins/linux/ebpf.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 70082daf5..2267dd922 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -29,15 +29,21 @@ class EBPF(plugins.PluginInterface): ), ] - def get_ebpf_programs(self, vmlinux) -> interfaces.objects.ObjectInterface: + def get_ebpf_programs( + self, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> interfaces.objects.ObjectInterface: """Enumerate eBPF programs walking its IDR. Args: - vmlinux: The kernel symbols object - + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate Yields: eBPF program objects """ + vmlinux = context.modules[vmlinux_module_name] + if not vmlinux.has_symbol("prog_idr"): raise exceptions.VolatilityException( "Cannot find the eBPF prog idr. Unsupported kernel" @@ -49,8 +55,7 @@ class EBPF(plugins.PluginInterface): yield bpf_prog def _generator(self): - vmlinux = self.context.modules[self.config["kernel"]] - for prog in self.get_ebpf_programs(vmlinux): + for prog in self.get_ebpf_programs(self.context, self.config["kernel"]): prog_addr = prog.vol.offset prog_type = prog.get_type() or renderers.NotAvailableValue() prog_tag = prog.get_tag() or renderers.NotAvailableValue() From 695635199e2bd3af11bee522e054719ee1898db6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 01:14:09 +1000 Subject: [PATCH 059/110] PR review fixes: Avoid saving state in the pidhashtable plugin --- .../framework/plugins/linux/pidhashtable.py | 79 +++++++++++-------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 9a2528543..3223aed4a 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -44,17 +44,16 @@ class PIDHashTable(plugins.PluginInterface): ), ] - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.vmlinux = None - self.vmlinux_layer = None - def _is_valid_task(self, task) -> bool: - return bool(task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent)) + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + return bool(task and task.pid > 0 and vmlinux_layer.is_valid(task.parent)) def _get_pidtype_pid(self): + vmlinux = self.context.modules[self.config["kernel"]] + # The pid_type enumeration is present since 2.5.37, just in case - pid_type_enum = self.vmlinux.get_enumeration("pid_type") + pid_type_enum = vmlinux.get_enumeration("pid_type") if not pid_type_enum: vollog.error("Cannot find pid_type enum. Unsupported kernel") return None @@ -68,38 +67,46 @@ class PIDHashTable(plugins.PluginInterface): return pidtype_pid def _get_pidhash_array(self): - pidhash_shift = self.vmlinux.object_from_symbol("pidhash_shift") + vmlinux = self.context.modules[self.config["kernel"]] + + pidhash_shift = vmlinux.object_from_symbol("pidhash_shift") pidhash_size = 1 << pidhash_shift - array_type_name = self.vmlinux.symbol_table_name + constants.BANG + "array" + array_type_name = vmlinux.symbol_table_name + constants.BANG + "array" - pidhash_ptr = self.vmlinux.object_from_symbol("pid_hash") + pidhash_ptr = vmlinux.object_from_symbol("pid_hash") # pidhash is an array of hlist_heads pidhash = self._context.object( array_type_name, offset=pidhash_ptr, - subtype=self.vmlinux.get_type("hlist_head"), + subtype=vmlinux.get_type("hlist_head"), count=pidhash_size, - layer_name=self.vmlinux.layer_name, + layer_name=vmlinux.layer_name, ) return pidhash def _walk_upid(self, seen_upids, upid): - while upid and self.vmlinux_layer.is_valid(upid.vol.offset): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + while upid and vmlinux_layer.is_valid(upid.vol.offset): if upid.vol.offset in seen_upids: break seen_upids.add(upid.vol.offset) pid_chain = upid.pid_chain - if not (pid_chain and self.vmlinux_layer.is_valid(pid_chain.vol.offset)): + if not (pid_chain and vmlinux_layer.is_valid(pid_chain.vol.offset)): break upid = linux.LinuxUtilities.container_of( - pid_chain.next, "upid", "pid_chain", self.vmlinux + pid_chain.next, "upid", "pid_chain", vmlinux ) def _get_upids(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + # 2.6.24 <= kernels < 4.15 pidhash = self._get_pidhash_array() @@ -108,10 +115,10 @@ class PIDHashTable(plugins.PluginInterface): # each entry in the hlist is a upid which is wrapped in a pid ent = hlist.first - while ent and self.vmlinux_layer.is_valid(ent.vol.offset): + while ent and vmlinux_layer.is_valid(ent.vol.offset): # upid->pid_chain exists 2.6.24 <= kernel < 4.15 upid = linux.LinuxUtilities.container_of( - ent.vol.offset, "upid", "pid_chain", self.vmlinux + ent.vol.offset, "upid", "pid_chain", vmlinux ) if upid.vol.offset in seen_upids: @@ -124,16 +131,14 @@ class PIDHashTable(plugins.PluginInterface): return seen_upids def _pid_hash_implementation(self): + vmlinux = self.context.modules[self.config["kernel"]] + # 2.6.24 <= kernels < 4.15 - task_pids_off = self.vmlinux.get_type("task_struct").relative_child_offset( - "pids" - ) + task_pids_off = vmlinux.get_type("task_struct").relative_child_offset("pids") pidtype_pid = self._get_pidtype_pid() for upid in self._get_upids(): - pid = linux.LinuxUtilities.container_of( - upid, "pid", "numbers", self.vmlinux - ) + pid = linux.LinuxUtilities.container_of(upid, "pid", "numbers", vmlinux) if not pid: continue @@ -141,22 +146,24 @@ class PIDHashTable(plugins.PluginInterface): if not pid_tasks_0: continue - task = self.vmlinux.object( + task = vmlinux.object( "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True ) if self._is_valid_task(task): yield task def _task_for_radix_pid_node(self, nodep): + vmlinux = self.context.modules[self.config["kernel"]] + # kernels >= 4.15 - pid = self.vmlinux.object("pid", offset=nodep, absolute=True) + pid = vmlinux.object("pid", offset=nodep, absolute=True) pidtype_pid = self._get_pidtype_pid() pid_tasks_0 = pid.tasks[pidtype_pid].first if not pid_tasks_0: return None - task_struct_type = self.vmlinux.get_type("task_struct") + task_struct_type = vmlinux.get_type("task_struct") if task_struct_type.has_member("pids"): member = "pids" elif task_struct_type.has_member("pid_links"): @@ -165,15 +172,17 @@ class PIDHashTable(plugins.PluginInterface): return None task_pids_off = task_struct_type.relative_child_offset(member) - task = self.vmlinux.object( + task = vmlinux.object( "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True ) return task def _pid_namespace_idr(self): + vmlinux = self.context.modules[self.config["kernel"]] + # kernels >= 4.15 - ns_addr = self.vmlinux.get_symbol("init_pid_ns").address - ns = self.vmlinux.object("pid_namespace", offset=ns_addr) + ns_addr = vmlinux.get_symbol("init_pid_ns").address + ns = vmlinux.object("pid_namespace", offset=ns_addr) for page_addr in ns.idr.get_entries(): task = self._task_for_radix_pid_node(page_addr) @@ -181,24 +190,26 @@ class PIDHashTable(plugins.PluginInterface): yield task def _determine_pid_func(self): - pid_hash = self.vmlinux.has_symbol("pid_hash") and self.vmlinux.has_symbol( + vmlinux = self.context.modules[self.config["kernel"]] + + pid_hash = vmlinux.has_symbol("pid_hash") and vmlinux.has_symbol( "pidhash_shift" ) # 2.5.55 <= kernels < 4.15 - has_pid_numbers = self.vmlinux.has_type("pid") and self.vmlinux.get_type( + has_pid_numbers = vmlinux.has_type("pid") and vmlinux.get_type( "pid" ).has_member( "numbers" ) # kernels >= 2.6.24 - has_pid_chain = self.vmlinux.has_type("upid") and self.vmlinux.get_type( + has_pid_chain = vmlinux.has_type("upid") and vmlinux.get_type( "upid" ).has_member( "pid_chain" ) # 2.6.24 <= kernels < 4.15 # kernels >= 4.15 - pid_idr = self.vmlinux.has_type("pid_namespace") and self.vmlinux.get_type( + pid_idr = vmlinux.has_type("pid_namespace") and vmlinux.get_type( "pid_namespace" ).has_member("idr") @@ -217,8 +228,6 @@ class PIDHashTable(plugins.PluginInterface): Yields: task_struct objects """ - self.vmlinux = self.context.modules[self.config["kernel"]] - self.vmlinux_layer = self.context.layers[self.vmlinux.layer_name] pid_func = self._determine_pid_func() if not pid_func: vollog.error("Cannot determine which PID hash table this kernel is using") From d964e6f61de00b5a8608d48d468d9a59b7923ec7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 02:06:06 +1000 Subject: [PATCH 060/110] PR review fixes: Check for LinuxUtilities version everywhere we use it --- .../symbols/linux/extensions/__init__.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d00af7a3f..51dc37d31 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,6 +13,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion +from volatility3.framework.configuration import requirements from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1608,6 +1609,19 @@ class xdp_sock(objects.StructType): class bpf_prog(objects.StructType): + def _get_vmlinux(self): + linuxutils_required_version = (2, 1, 1) + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + return vmlinux + def get_type(self) -> Union[str, None]: """Returns a string with the eBPF program type""" @@ -1631,7 +1645,7 @@ class bpf_prog(objects.StructType): if not self.has_member("tag"): return None - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] prog_tag_addr = self.tag.vol.offset @@ -2043,13 +2057,26 @@ class page(objects.StructType): return flags + def _get_vmlinux(self): + linuxutils_required_version = (2, 1, 1) + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + return vmlinux + def to_paddr(self) -> int: """Converts a page's virtual address to its physical address using the current physical memory model. Returns: int: page physical address """ - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] vmemmap_start = None @@ -2100,7 +2127,7 @@ class page(objects.StructType): Returns: The page content """ - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] physical_layer = vmlinux.context.layers["memory_layer"] page_paddr = self.to_paddr() @@ -2118,6 +2145,19 @@ class IDR(objects.StructType): MAX_IDR_SHIFT = INT_SIZE * 8 - 1 MAX_IDR_BIT = 1 << MAX_IDR_SHIFT + def _get_vmlinux(self): + linuxutils_required_version = (2, 1, 1) + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + return vmlinux + def idr_max(self, num_layers: int) -> int: """Returns the maximum ID which can be allocated given idr::layers @@ -2141,7 +2181,7 @@ class IDR(objects.StructType): Returns: A pointer to the given ID element """ - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() if not vmlinux.get_type("idr_layer").has_member("layer"): vollog.info( "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" From d627f243e259efce2f730860f47d9116b3c78f3d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 10:55:01 +1000 Subject: [PATCH 061/110] PR review fixes: Add typing info to the get_inodes() class method. --- volatility3/framework/plugins/linux/pagecache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index cf4151c85..e384cbabf 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type +from typing import List, Set, Type, Iterable from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints @@ -205,7 +205,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, config_path: str, - ): + ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: From 90b327e63253404a98cc5a79e3cecaa1b773048c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 11:45:29 +1000 Subject: [PATCH 062/110] PR review fixes: Make mountinfo.get_superblocks() a classmethod and adapt the code using it. --- .../framework/plugins/linux/mountinfo.py | 19 +++++++++--- .../framework/plugins/linux/pagecache.py | 31 ++++++++++++------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index dfb2e2f52..1eaec77bf 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -37,7 +37,7 @@ class MountInfo(plugins.PluginInterface): _required_framework_version = (2, 2, 0) - _version = (1, 1, 0) + _version = (1, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -143,8 +143,8 @@ class MountInfo(plugins.PluginInterface): sb_opts, ) + @staticmethod def _get_tasks_mountpoints( - self, tasks: Iterable[interfaces.objects.ObjectInterface], filtered_by_pids: bool = False, ): @@ -247,17 +247,26 @@ class MountInfo(plugins.PluginInterface): "Could not filter by mount namespace id. This field is not available in this kernel." ) - def get_superblocks(self): + @classmethod + def get_superblocks( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Yield file system superblocks based on the task's mounted filesystems. + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + Yields: super_block: Kernel's struct super_block object """ # No filter so that we get all the mount namespaces from all tasks - tasks = pslist.PsList.list_tasks(self.context, self.config["kernel"]) + tasks = pslist.PsList.list_tasks(context, vmlinux_module_name) seen_sb_ptr = set() - for task, mnt, _mnt_ns_id in self._get_tasks_mountpoints(tasks): + for task, mnt, _mnt_ns_id in cls._get_tasks_mountpoints(tasks): path_root = linux.LinuxUtilities.get_path_mnt(task, mnt) if not path_root: continue diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index e384cbabf..e54891480 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -115,7 +115,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 1, 0) + name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 2, 0) ), requirements.ListRequirement( name="type", @@ -204,22 +204,22 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): def get_inodes( cls, context: interfaces.context.ContextInterface, - config_path: str, + vmlinux_module_name: str, ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: context: The context that the plugin will operate within - config_path: The path to configuration data within the context configuration data + vmlinux_module_name: The name of the kernel module on which to operate Yields: An InodeInternal object """ - superblocks_iter = mountinfo.MountInfo( + superblocks_iter = mountinfo.MountInfo.get_superblocks( context=context, - config_path=config_path, - ).get_superblocks() + vmlinux_module_name=vmlinux_module_name, + ) seen_inodes = set() seen_dentries = set() @@ -289,11 +289,13 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): yield inode_in def _generator(self): - vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] vmlinux_layer = self.context.layers[vmlinux.layer_name] inodes_iter = self.get_inodes( - context=self.context, config_path=self.config_path + context=self.context, + vmlinux_module_name=vmlinux_module_name, ) types_filter = self.config["type"] @@ -316,12 +318,15 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): These need not be generated in any particular order, sorting will be done later """ - vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] vmlinux_layer = self.context.layers[vmlinux.layer_name] inodes_iter = self.get_inodes( - context=self.context, config_path=self.config_path + context=self.context, + vmlinux_module_name=vmlinux_module_name, ) + for inode_in in inodes_iter: inode_out = inode_in.to_user(vmlinux_layer) description = f"Cached Inode for {inode_out.path}" @@ -450,7 +455,8 @@ class InodePages(plugins.PluginInterface): vollog.error("Unable to write to file (%s): %s", filename, e) def _generator(self): - vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] vmlinux_layer = self.context.layers[vmlinux.layer_name] if self.config["inode"] and self.config["find"]: @@ -459,7 +465,8 @@ class InodePages(plugins.PluginInterface): if self.config["find"]: inodes_iter = Files.get_inodes( - context=self.context, config_path=self.config_path + context=self.context, + vmlinux_module_name=vmlinux_module_name, ) for inode_in in inodes_iter: if inode_in.path == self.config["find"]: From 3bf9f8cec0e1c4c75088abe17baadd0bc6d4c3d6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 12:03:10 +1000 Subject: [PATCH 063/110] PR review fixes: Add typing info to pagecache.Files._follow_symlink() --- volatility3/framework/plugins/linux/pagecache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index e54891480..b6c5f7cc0 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -131,7 +131,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): ] @staticmethod - def _follow_symlink(inode, symlink_path) -> str: + def _follow_symlink( + inode: interfaces.objects.ObjectInterface, + symlink_path: str, + ) -> str: """Follows (fast) symlinks (kernels >= 4.2.x). Fast symlinks are filesystem agnostic. From e97abae156721bc1c625729166c08325c164bd3a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 24 Aug 2024 15:02:31 +0100 Subject: [PATCH 064/110] Remove get_vmlinux calls Sorry, I know I asked for it, but I hadn't quite figured out what was going on. Things in the symbols/linux code are considered part of the framework, and therefore it's the framework version that should have been bumped. Since the framework comes packages with LinuxUtilities we can rely on the version numbers to be suitable. This cleans up the mess I caused, sorry for the extra work! 5:S --- volatility3/framework/constants/_version.py | 4 +- .../symbols/linux/extensions/__init__.py | 47 ++----------------- 2 files changed, 6 insertions(+), 45 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 4df0b9041..d30446d63 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 8 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 9 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 51dc37d31..fdd34403a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1609,19 +1609,6 @@ class xdp_sock(objects.StructType): class bpf_prog(objects.StructType): - def _get_vmlinux(self): - linuxutils_required_version = (2, 1, 1) - linuxutils_current_version = linux.LinuxUtilities._version - if not requirements.VersionRequirement.matches_required( - linuxutils_required_version, linuxutils_current_version - ): - raise exceptions.PluginRequirementException( - f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" - ) - - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - return vmlinux - def get_type(self) -> Union[str, None]: """Returns a string with the eBPF program type""" @@ -1645,7 +1632,7 @@ class bpf_prog(objects.StructType): if not self.has_member("tag"): return None - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] prog_tag_addr = self.tag.vol.offset @@ -2057,26 +2044,13 @@ class page(objects.StructType): return flags - def _get_vmlinux(self): - linuxutils_required_version = (2, 1, 1) - linuxutils_current_version = linux.LinuxUtilities._version - if not requirements.VersionRequirement.matches_required( - linuxutils_required_version, linuxutils_current_version - ): - raise exceptions.PluginRequirementException( - f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" - ) - - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - return vmlinux - def to_paddr(self) -> int: """Converts a page's virtual address to its physical address using the current physical memory model. Returns: int: page physical address """ - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] vmemmap_start = None @@ -2127,7 +2101,7 @@ class page(objects.StructType): Returns: The page content """ - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] physical_layer = vmlinux.context.layers["memory_layer"] page_paddr = self.to_paddr() @@ -2145,19 +2119,6 @@ class IDR(objects.StructType): MAX_IDR_SHIFT = INT_SIZE * 8 - 1 MAX_IDR_BIT = 1 << MAX_IDR_SHIFT - def _get_vmlinux(self): - linuxutils_required_version = (2, 1, 1) - linuxutils_current_version = linux.LinuxUtilities._version - if not requirements.VersionRequirement.matches_required( - linuxutils_required_version, linuxutils_current_version - ): - raise exceptions.PluginRequirementException( - f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" - ) - - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - return vmlinux - def idr_max(self, num_layers: int) -> int: """Returns the maximum ID which can be allocated given idr::layers @@ -2181,7 +2142,7 @@ class IDR(objects.StructType): Returns: A pointer to the given ID element """ - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) if not vmlinux.get_type("idr_layer").has_member("layer"): vollog.info( "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" From 6a157a785f3364635de2c7a43eeab054ab7c6b3d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 27 Aug 2024 14:45:52 +0100 Subject: [PATCH 065/110] Bump the modules version number Pull-request #1173 bumped the version of the modules plugin (even though this only needed to be a MINOR version bump, see https://github.com/volatilityfoundation/volatility3/pull/1173#discussion_r1649614761), but failed to verify that other plugins which relied on it were also updated to make use of the new plugin. This was the version system working as intended, but highlighted a review failure that the neither the author, nor the reviewers, verified that the rest of the framework (specifically other plugins which relied on modules) worked correctly with the new code (which this kind of error is designed to fix). Fixes #1244. --- volatility3/framework/plugins/windows/ssdt.py | 2 +- volatility3/framework/plugins/windows/verinfo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/ssdt.py b/volatility3/framework/plugins/windows/ssdt.py index 6a47c36e9..1fcb6cc91 100644 --- a/volatility3/framework/plugins/windows/ssdt.py +++ b/volatility3/framework/plugins/windows/ssdt.py @@ -30,7 +30,7 @@ class SSDT(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="modules", plugin=modules.Modules, version=(1, 0, 0) + name="modules", plugin=modules.Modules, version=(2, 0, 0) ), ] diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 1c6615804..5b3c52bf6 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -46,7 +46,7 @@ class VerInfo(interfaces.plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="modules", plugin=modules.Modules, version=(1, 0, 0) + name="modules", plugin=modules.Modules, version=(2, 0, 0) ), requirements.VersionRequirement( name="dlllist", component=dlllist.DllList, version=(2, 0, 0) From a2196850bc01d5a1a41a2aa29685cdb1d2146e87 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Wed, 28 Aug 2024 10:39:28 +0200 Subject: [PATCH 066/110] increase python version to 3.8 --- .github/workflows/build-pypi.yml | 2 +- .github/workflows/install.yml | 2 +- .github/workflows/test.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 346d0337e..d1a63b4da 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.7"] + python-version: ["3.8"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index e13161085..398ff8ae3 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: host: [ ubuntu-latest, windows-latest ] - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + python-version: [ "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 55b2e4b60..65dfb13c8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.7"] + python-version: ["3.8"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 7a479d617f8107590533eaed9afe105c3cd3b34f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 28 Aug 2024 18:40:41 +0200 Subject: [PATCH 067/110] add OS and framework architectures constants --- .../framework/constants/architectures.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 volatility3/framework/constants/architectures.py diff --git a/volatility3/framework/constants/architectures.py b/volatility3/framework/constants/architectures.py new file mode 100644 index 000000000..a8822d7d3 --- /dev/null +++ b/volatility3/framework/constants/architectures.py @@ -0,0 +1,21 @@ +from volatility3.framework.layers import intel + +WIN_ARCHS = ["Intel32", "Intel64"] +"""Windows supported architectures""" +WIN_ARCHS_LAYERS = [intel.Intel] +"""Windows supported architectures layers""" + +LINUX_ARCHS = ["Intel32", "Intel64"] +"""Linux supported architectures""" +LINUX_ARCHS_LAYERS = [intel.Intel] +"""Linux supported architectures layers""" + +MAC_ARCHS = ["Intel32", "Intel64"] +"""Mac supported architectures""" +MAC_ARCHS_LAYERS = [intel.Intel] +"""Mac supported architectures layers""" + +FRAMEWORK_ARCHS = ["Intel32", "Intel64"] +"""Framework supported architectures""" +FRAMEWORK_ARCHS_LAYERS = [intel.Intel] +"""Framework supported architectures layers""" From 46a26c7dc5f5bc87d93027a6c005acbf85bc8716 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 12:17:16 -0500 Subject: [PATCH 068/110] Address feedback --- .../framework/plugins/windows/orphan_kernel_threads.py | 8 +++++--- volatility3/framework/plugins/windows/thrdscan.py | 3 +-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/orphan_kernel_threads.py b/volatility3/framework/plugins/windows/orphan_kernel_threads.py index 1cdb1dcdf..f4901dc8c 100644 --- a/volatility3/framework/plugins/windows/orphan_kernel_threads.py +++ b/volatility3/framework/plugins/windows/orphan_kernel_threads.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # @@ -48,8 +48,9 @@ class Threads(thrdscan.ThrdScan): """Yields thread objects of kernel threads that do not map to a module Args: - kernel - + cls + context: the context to operate upon + module_name: name of the module to use for scanning Returns: A generator of thread objects of orphaned threads """ @@ -61,6 +62,7 @@ class Threads(thrdscan.ThrdScan): context, layer_name, symbol_table ) + # FIXME - use a proper constant once established # used to filter out smeared pointers if symbols.symbol_table_is_64bit(context, symbol_table): kernel_start = 0xFFFFF80000000000 diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index ad885e8e9..b812a15ff 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -48,8 +48,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) Args: context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - symbol_table: The name of the table containing the kernel symbols + module_name: Name of the module to use for scanning Returns: A list of _ETHREAD objects found by scanning memory for the "Thre" / "Thr\\xE5" pool signatures From 483cb7e4eebbb6b536cdec883caa3f75fa182ce2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 15:08:29 -0500 Subject: [PATCH 069/110] Add smear checks in MFT parsing code --- .../framework/symbols/windows/extensions/mft.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 14c1f08d6..c51dd0348 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -27,6 +27,11 @@ class MFTAttribute(objects.StructType): """This represents an MFT ATTRIBUTE""" def get_resident_filename(self) -> str: + # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems + # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous + if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.NameLength > 512: + return None + # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data try: name = self._context.object( @@ -42,6 +47,11 @@ class MFTAttribute(objects.StructType): return None def get_resident_filecontent(self) -> bytes: + # smear observed in mass testing of samples + # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems + if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.ContentLength > 4194304: + return None + # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data try: bytesobj = self._context.object( From 660e8a7b89387bfc3e883a10a3662708a09fa146 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 15:09:20 -0500 Subject: [PATCH 070/110] Add smear checks in MFT parsing code --- .../framework/symbols/windows/extensions/mft.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index c51dd0348..c0303eafb 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -29,7 +29,10 @@ class MFTAttribute(objects.StructType): def get_resident_filename(self) -> str: # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous - if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.NameLength > 512: + if ( + self.Attr_Header.ContentOffset > 4194304 + or self.Attr_Header.NameLength > 512 + ): return None # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data @@ -49,7 +52,10 @@ class MFTAttribute(objects.StructType): def get_resident_filecontent(self) -> bytes: # smear observed in mass testing of samples # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems - if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.ContentLength > 4194304: + if ( + self.Attr_Header.ContentOffset > 4194304 + or self.Attr_Header.ContentLength > 4194304 + ): return None # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data From 9c058936a88893d0b5dd6a3ee68ab87b037bacf4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 3 Sep 2024 10:30:05 -0500 Subject: [PATCH 071/110] Address feedback --- .../framework/symbols/windows/extensions/mft.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index c0303eafb..ebba882c0 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,6 +2,8 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +from typing import Optional + from volatility3.framework import objects, constants, exceptions @@ -26,11 +28,11 @@ class MFTFileName(objects.StructType): class MFTAttribute(objects.StructType): """This represents an MFT ATTRIBUTE""" - def get_resident_filename(self) -> str: + def get_resident_filename(self) -> Optional[str]: # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous if ( - self.Attr_Header.ContentOffset > 4194304 + self.Attr_Header.ContentOffset > 0x400000 or self.Attr_Header.NameLength > 512 ): return None @@ -49,12 +51,12 @@ class MFTAttribute(objects.StructType): except exceptions.InvalidAddressException: return None - def get_resident_filecontent(self) -> bytes: + def get_resident_filecontent(self) -> Optional[bytes]: # smear observed in mass testing of samples # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems if ( - self.Attr_Header.ContentOffset > 4194304 - or self.Attr_Header.ContentLength > 4194304 + self.Attr_Header.ContentOffset > 0x400000 + or self.Attr_Header.ContentLength > 0x400000 ): return None From 69e6e59daf39ec7a0cf9a82fb13762d4389c95a4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:01:21 -0500 Subject: [PATCH 072/110] Add new pe_symbols API, debug registers plugin, unhooked system calls plugin --- .../plugins/windows/debugregisters.py | 223 ++++++ .../framework/plugins/windows/pe_symbols.py | 732 ++++++++++++++++++ .../plugins/windows/unhooked_system_calls.py | 183 +++++ .../framework/plugins/windows/vadinfo.py | 88 ++- 4 files changed, 1219 insertions(+), 7 deletions(-) create mode 100644 volatility3/framework/plugins/windows/debugregisters.py create mode 100644 volatility3/framework/plugins/windows/pe_symbols.py create mode 100644 volatility3/framework/plugins/windows/unhooked_system_calls.py diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py new file mode 100644 index 000000000..5f3c2e519 --- /dev/null +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -0,0 +1,223 @@ +import logging + +from typing import Tuple, Optional, Generator, List, Dict + +from functools import partial + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +import volatility3.plugins.windows.pslist as pslist +import volatility3.plugins.windows.threads as threads +import volatility3.plugins.windows.vadinfo as vadinfo +import volatility3.plugins.windows.pe_symbols as pe_symbols + +vollog = logging.getLogger(__name__) + + +class DebugRegisters(interfaces.plugins.PluginInterface): + # version 2.6.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 6, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) + ), + ] + + def _get_debug_info( + self, ethread: interfaces.objects.ObjectInterface + ) -> Optional[Tuple[interfaces.objects.ObjectInterface, int, int, int, int, int]]: + """ + Gathers information related to the debug registers for the given thread + """ + try: + dr7 = ethread.Tcb.TrapFrame.Dr7 + state = ethread.Tcb.State + except exceptions.InvalidAddressException: + return None + + # 0 = debug registers not active + # 4 = terminated + if dr7 == 0 or state == 4: + return None + + try: + owner_proc = ethread.owning_process() + except (AttributeError, exceptions.InvalidAddressException): + return None + + dr0 = ethread.Tcb.TrapFrame.Dr0 + dr1 = ethread.Tcb.TrapFrame.Dr1 + dr2 = ethread.Tcb.TrapFrame.Dr2 + dr3 = ethread.Tcb.TrapFrame.Dr3 + + # bail if all are 0 + if not (dr0 or dr1 or dr2 or dr3): + return None + + return owner_proc, dr7, dr0, dr1, dr2, dr3 + + def _get_vads( + self, + vads_cache: Dict[int, List[Tuple[int, int, str]]], + owner_proc: interfaces.objects.ObjectInterface, + ) -> Optional[List[Tuple[int, int, str]]]: + if owner_proc.vol.offset in vads_cache: + vads = vads_cache[owner_proc.vol.offset] + else: + vads = vadinfo.VadInfo.get_proc_vads_with_file_paths(owner_proc) + vads_cache[owner_proc.vol.offset] = vads + + # smear or terminated process + if len(vads) == 0: + return None + + return vads + + def _generator( + self, + ) -> Generator[ + Tuple[ + int, + Tuple[ + str, + int, + int, + int, + int, + format_hints.Hex, + str, + str, + format_hints.Hex, + str, + str, + format_hints.Hex, + str, + str, + format_hints.Hex, + str, + str, + ], + ], + None, + None, + ]: + kernel = self.context.modules[self.config["kernel"]] + + vads_cache: Dict[int, List[Tuple[int, int, str]]] = {} + + proc_modules = None + + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) + + for proc in procs: + for thread in threads.Threads.list_threads(kernel, proc): + debug_info = self._get_debug_info(thread) + if not debug_info: + continue + + owner_proc, dr7, dr0, dr1, dr2, dr3 = debug_info + + vads = self._get_vads(vads_cache, owner_proc) + if not vads: + continue + + # this lookup takes a while, so only perform if we need to + if not proc_modules: + proc_modules = pe_symbols.PESymbols.get_process_modules( + self.context, kernel.layer_name, kernel.symbol_table_name, None + ) + path_and_symbol = partial( + pe_symbols.PESymbols.path_and_symbol_for_address, + self.context, + self.config_path, + proc_modules, + ) + + file0, sym0 = path_and_symbol(vads, dr0) + file1, sym1 = path_and_symbol(vads, dr1) + file2, sym2 = path_and_symbol(vads, dr2) + file3, sym3 = path_and_symbol(vads, dr3) + + # if none map to an actual file VAD then bail + if not ( + isinstance(file0, str) + or isinstance(file1, str) + or isinstance(file2, str) + or isinstance(file3, str) + ): + continue + + process_name = owner_proc.ImageFileName.cast( + "string", + max_length=owner_proc.ImageFileName.vol.count, + errors="replace", + ) + + thread_tid = thread.Cid.UniqueThread + + yield ( + 0, + ( + process_name, + owner_proc.UniqueProcessId, + thread_tid, + thread.Tcb.State, + dr7, + format_hints.Hex(dr0), + file0, + sym0, + format_hints.Hex(dr1), + file1, + sym1, + format_hints.Hex(dr2), + file2, + sym2, + format_hints.Hex(dr3), + file3, + sym3, + ), + ) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("TID", int), + ("State", int), + ("Dr7", int), + ("Dr0", format_hints.Hex), + ("Range0", str), + ("Symbol0", str), + ("Dr1", format_hints.Hex), + ("Range1", str), + ("Symbol1", str), + ("Dr2", format_hints.Hex), + ("Range2", str), + ("Symbol2", str), + ("Dr3", format_hints.Hex), + ("Range3", str), + ("Symbol3", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py new file mode 100644 index 000000000..b7faeaf55 --- /dev/null +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -0,0 +1,732 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + + +import io +import logging + +from typing import Dict, Tuple, Optional, List, Generator, Union + +import pefile + +from volatility3.framework import interfaces, exceptions +from volatility3.framework import renderers, constants +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, vadinfo, modules + +vollog = logging.getLogger(__name__) + + +class PESymbolFinder: + """ + Interface for PE symbol finding classes + This interface provides a standard way for the calling code to + lookup symbols by name or address + """ + + cached_str = Union[str, None] + cached_str_dict = Dict[str, cached_str] + + cached_int = Union[int, None] + cached_int_dict = Dict[str, cached_int] + + cached_value = Union[int, str, None] + cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + + def __init__( + self, + layer_name: str, + mod_name: str, + module_start: int, + symbol_module: Union[interfaces.context.ModuleInterface, pefile.ExportDirData], + ): + self._layer_name = layer_name + self._mod_name = mod_name + self._module_start = module_start + self._symbol_module = symbol_module + + self._address_cache: PESymbolFinder.cached_int_dict = {} + self._name_cache: PESymbolFinder.cached_str_dict = {} + + def _get_cache_key(self, value: cached_value) -> str: + """ + Maintain a cache for symbol lookups to avoid re-walking of PDB symbols or export tables + within the same module for the same address in the same layer + """ + return f"{self._layer_name}|{self._mod_name}|{value}" + + def get_name_for_address(self, address: int) -> cached_str: + cached_key = self._get_cache_key(address) + if cached_key not in self._name_cache: + name = self._do_get_name(address) + self._name_cache[cached_key] = name + + return self._name_cache[cached_key] + + def get_address_for_name(self, name: str) -> cached_int: + cached_key = self._get_cache_key(name) + if cached_key not in self._address_cache: + address = self._do_get_address(name) + self._address_cache[cached_key] = address + + return self._address_cache[cached_key] + + def _do_get_name(self, address: int) -> cached_str: + raise NotImplementedError("_do_get_name must be overwritten") + + def _do_get_address(self, name: str) -> cached_int: + raise NotImplementedError("_do_get_address must be overwritten") + + +class PDBSymbolFinder(PESymbolFinder): + """ + PESymbolFinder implementation for PDB modules + """ + + def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + try: + return self._symbol_module.get_absolute_symbol_address(name) + except exceptions.SymbolError: + return None + + def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + try: + name = self._symbol_module.get_symbols_by_absolute_location(address)[0] + return name.split(constants.BANG)[1] + except (exceptions.SymbolError, IndexError): + return None + + +class ExportSymbolFinder(PESymbolFinder): + """ + PESymbolFinder implementation for PDB modules + """ + + def _get_name(self, export: pefile.ExportData) -> Optional[str]: + # AttributeError throws on empty or ordinal-only exports + try: + return export.name.decode("ascii") + except AttributeError: + return None + + def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + for export in self._symbol_module: + if export.address + self._module_start == address: + return self._get_name(export) + + return None + + def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + for export in self._symbol_module: + sym_name = self._get_name(export) + if sym_name and sym_name == name: + return self._module_start + export.address + + return None + + +class PESymbols(interfaces.plugins.PluginInterface): + """Prints symbols in PE files in process and kernel memory""" + + _required_framework_version = (2, 7, 0) + + _version = (1, 0, 0) + + # used for special handling of the kernel PDB file. See later notes + os_module_name = "ntoskrnl.exe" + + # keys for specifying wanted names and/or addresses + # used for consistent access between the API and plugins + wanted_names = "names" + wanted_addresses = "addresses" + + # how wanted modules/symbols are specified, such as: + # {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} + # {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} + filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + + # holds resolved symbols + # {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} + found_symbols_type = Dict[str, List[Tuple[str, int]]] + + # used to hold informatin about a range (VAD or kernel module) + # (start address, size, file path) + range_type = Tuple[int, int, str] + ranges_type = List[range_type] + + @classmethod + def get_requirements(cls) -> List: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="modules", component=modules.Modules, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pdbutil", component=pdbutil.PDBUtility, version=(1, 0, 0) + ), + requirements.ChoiceRequirement( + name="source", + description="Where to resolve symbols.", + choices=["kernel", "processes"], + optional=False, + ), + requirements.StringRequirement( + name="module", + description='Module in which to resolve symbols. Use "ntoskrnl.exe" to resolve in the base kernel executable.', + optional=False, + ), + requirements.StringRequirement( + name="symbol", + description="Symbol name to resolve", + optional=True, + ), + requirements.IntRequirement( + name="address", + description="Address of symbol to resolve", + optional=True, + ), + ] + + @staticmethod + def _get_pefile_obj( + context: interfaces.context.ContextInterface, + pe_table_name: str, + layer_name: str, + base_address: int, + ) -> Optional[pefile.PE]: + """ + Attempts to pefile object from the bytes of the PE file + + Args: + pe_table_name: name of the pe types table + layer_name: name of the process layer + base_address: base address of the module + + Returns: + the constructed pefile object + """ + pe_data = io.BytesIO() + + try: + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base_address, + layer_name=layer_name, + ) + + for offset, data in dos_header.reconstruct(): + pe_data.seek(offset) + pe_data.write(data) + + pe_ret = pefile.PE(data=pe_data.getvalue(), fast_load=True) + + except exceptions.InvalidAddressException: + pe_ret = None + + return pe_ret + + @staticmethod + def range_info_for_address( + ranges: ranges_type, address: int + ) -> Optional[range_type]: + """ + Helper for getting the range information for an address + """ + for start, size, filepath in ranges: + if start <= address < start + size: + return start, size, filepath + + return None + + @staticmethod + def filepath_for_address(ranges: ranges_type, address: int) -> Optional[str]: + """ + Helper to get the file path for an address + """ + info = PESymbols.range_info_for_address(ranges, address) + if info: + return info[2] + + return None + + @staticmethod + def filename_for_path(filepath: str) -> str: + """ + Consistent way to get the filename + """ + return filepath.split("\\")[-1] + + @staticmethod + def addresses_for_process_symbols( + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + symbol_table_name: str, + symbols: filter_modules_type, + ) -> found_symbols_type: + collected_modules = PESymbols.get_process_modules( + context, layer_name, symbol_table_name, symbols + ) + + found_symbols = PESymbols.find_symbols( + context, config_path, symbols, collected_modules + ) + + for mod_name, unresolved_symbols in symbols.items(): + for symbol in unresolved_symbols: + vollog.debug(f"Unable to resolve symbol {symbol} in module {mod_name}") + + return found_symbols + + @staticmethod + def path_and_symbol_for_address( + context: interfaces.context.ContextInterface, + config_path: str, + collected_modules: Dict[str, List[Tuple[str, int, int]]], + ranges: ranges_type, + address: int, + ) -> Tuple[str, str]: + """ + Method for plugins to determine the file path and symbol name for a given address + + collected_modules: return value from `get_kernel_modules` or `get_process_modules` + ranges: the memory ranges to examine in this layer. + address: address to resolve to its symbol name + """ + + if not address: + return renderers.NotApplicableValue(), renderers.NotApplicableValue() + + filepath = PESymbols.filepath_for_address(ranges, address) + + if not filepath: + return renderers.NotAvailableValue(), renderers.NotAvailableValue() + + filename = PESymbols.filename_for_path(filepath).lower() + + # setup to resolve the address + filter_module: PESymbols.filter_modules_type = { + filename: {PESymbols.wanted_addresses: [address]} + } + + found_symbols = PESymbols.find_symbols( + context, config_path, filter_module, collected_modules + ) + + if not found_symbols or not found_symbols[filename]: + return renderers.NotAvailableValue(), renderers.NotAvailableValue() + + return filepath, found_symbols[filename][0][0] + + @staticmethod + def _get_exported_symbols( + context: interfaces.context.ContextInterface, + pe_table_name: str, + mod_name: str, + module_info: Tuple[str, int, int], + ) -> Optional[ExportSymbolFinder]: + """ + Attempts to locate symbols based on export analysis + + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + """ + + layer_name = module_info[0] + module_start = module_info[1] + + # we need a valid PE with an export table + pe_module = PESymbols._get_pefile_obj( + context, pe_table_name, layer_name, module_start + ) + if not pe_module: + return None + + pe_module.parse_data_directories( + directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_EXPORT"]] + ) + if not hasattr(pe_module, "DIRECTORY_ENTRY_EXPORT"): + return None + + return ExportSymbolFinder( + layer_name, mod_name, module_start, pe_module.DIRECTORY_ENTRY_EXPORT.symbols + ) + + @staticmethod + def _get_pdb_module( + context: interfaces.context.ContextInterface, + config_path: str, + mod_name: str, + module_info: Tuple[str, int, int], + ) -> Optional[PDBSymbolFinder]: + """ + Attempts to locate symbols based on PDB analysis + + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + """ + + mod_symbols = None + + layer_name, module_start, module_size = module_info + + # the PDB name of the kernel file is not consistent for an exe, for example, + # a `ntoskrnl.exe` can have an internal PDB name of any of the ones in the following list + # The code attempts to find all possible PDBs to ensure the best chance of recovery + if mod_name == PESymbols.os_module_name: + pdb_names = ["ntkrnlmp.pdb", "ntkrnlpa.pdb", "ntkrpamp.pdb", "ntoskrnl.pdb"] + + # for non-kernel files, replace the exe, sys, or dll extension with pdb + else: + mod_name = mod_name[:-3] + "pdb" + first_upper = mod_name[0].upper() + mod_name[1:] + pdb_names = [mod_name, first_upper] + + # loop through each PDB name (will be just one for all but the kernel) + for pdb_name in pdb_names: + try: + mod_symbols = pdbutil.PDBUtility.symbol_table_from_pdb( + context, + interfaces.configuration.path_join(config_path, mod_name), + layer_name, + pdb_name, + module_start, + module_size, + ) + + if mod_symbols: + break + + # this exception is expected when the PDB can't be found or downloaded + except exceptions.VolatilityException: + continue + + # this is not expected - it means pdbconv broke when parsing the PDB + except TypeError as e: + vollog.error( + f"Unable to parse PDB file for module {pdb_name} -> {e}. Please file a bug on the GitHub issue tracker." + ) + + # cannot do anything without the symbols + if not mod_symbols: + return None + + pdb_module = context.module( + mod_symbols, layer_name=layer_name, offset=module_start + ) + + return PDBSymbolFinder(layer_name, mod_name, module_start, pdb_module) + + @staticmethod + def _find_symbols_through_pdb( + context: interfaces.context.ContextInterface, + config_path: str, + module_instances: List[Tuple[str, int, int]], + mod_name: str, + ) -> Generator[PDBSymbolFinder, None, None]: + """ + Attempts to resolve the symbols in `wanted_symbols` through PDB analysis + """ + for module_info in module_instances: + mod_module = PESymbols._get_pdb_module( + context, config_path, mod_name, module_info + ) + if mod_module: + yield mod_module + + @staticmethod + def _find_symbols_through_exports( + context: interfaces.context.ContextInterface, + config_path: str, + module_instances: List[Tuple[str, int, int]], + mod_name: str, + ) -> Generator[ExportSymbolFinder, None, None]: + """ + Attempts to resolve the symbols in `wanted_symbols` through export analysis + """ + pe_table_name = intermed.IntermediateSymbolTable.create( + context, config_path, "windows", "pe", class_types=pe.class_types + ) + + # for each process layer and VAD, construct a PE and examine the export table + for module_info in module_instances: + exported_symbols = PESymbols._get_exported_symbols( + context, pe_table_name, mod_name, module_info + ) + if exported_symbols: + yield exported_symbols + + @staticmethod + def _get_symbol_value( + wanted_modules: PESymbolFinder.cached_value_dict, + mod_name: str, + symbol_resolver: PESymbolFinder, + ) -> Generator[Tuple[str, int], None, None]: + """ + Enumerates the symbols specified as wanted by the calling plugin + """ + wanted_symbols = wanted_modules[mod_name] + + if ( + PESymbols.wanted_names not in wanted_symbols + and PESymbols.wanted_addresses not in wanted_symbols + ): + vollog.warning( + f"Invalid `wanted_symbols` sent to `find_symbols` for module {mod_name}. addresses and names keys both misssing." + ) + return + + symbol_keys = [ + (PESymbols.wanted_names, "get_address_for_name"), + (PESymbols.wanted_addresses, "get_name_for_address"), + ] + + for symbol_key, symbol_getter in symbol_keys: + # address or name + if symbol_key in wanted_symbols: + # walk each wanted address or name + for wanted_value in wanted_symbols[symbol_key]: + symbol_value = symbol_resolver.__getattribute__(symbol_getter)( + wanted_value + ) + if symbol_value: + # yield out symbol name, symbol address + if symbol_key == PESymbols.wanted_names: + yield wanted_value, symbol_value # type: ignore + else: + yield symbol_value, wanted_value # type: ignore + + index = wanted_modules[mod_name][symbol_key].index(wanted_value) # type: ignore + + del wanted_modules[mod_name][symbol_key][index] + + # if all names or addresses from a module are found, delete the key + if not wanted_modules[mod_name][symbol_key]: + del wanted_modules[mod_name][symbol_key] + break + + @staticmethod + def _resolve_symbols_through_methods( + context: interfaces.context.ContextInterface, + config_path: str, + module_instances: List[Tuple[str, int, int]], + wanted_modules: PESymbolFinder.cached_value_dict, + mod_name: str, + ) -> Generator[Tuple[str, int], None, None]: + """ + Attempts to resolve every wanted symbol in `mod_name` + Every layer is enumerated for maximum chance of recovery + """ + symbol_resolving_methods = [ + PESymbols._find_symbols_through_pdb, + PESymbols._find_symbols_through_exports, + ] + + for method in symbol_resolving_methods: + for symbol_resolver in method( + context, config_path, module_instances, mod_name + ): + vollog.debug(f"Have resolver for method {method}") + yield from PESymbols._get_symbol_value( + wanted_modules, mod_name, symbol_resolver + ) + + if not wanted_modules[mod_name]: + break + + if not wanted_modules[mod_name]: + break + + @staticmethod + def find_symbols( + context: interfaces.context.ContextInterface, + config_path: str, + wanted_modules: PESymbolFinder.cached_value_dict, + collected_modules: Dict[str, List[Tuple[str, int, int]]], + ) -> found_symbols_type: + """ + Loops through each method of symbol analysis until each wanted symbol is found + Returns the resolved symbols as a dictionary that includes the name and runtime address + """ + found_symbols: PESymbols.found_symbols_type = {} + + for mod_name in wanted_modules: + if mod_name not in collected_modules: + continue + + module_instances = collected_modules[mod_name] + + # try to resolve the symbols for `mod_name` through each method (PDB and export table currently) + for symbol_name, address in PESymbols._resolve_symbols_through_methods( + context, config_path, module_instances, wanted_modules, mod_name + ): + if mod_name not in found_symbols: + found_symbols[mod_name] = [] + + found_symbols[mod_name].append((symbol_name, address)) + + # stop processing the layers (processes) if we found all the symbols for this module + if not wanted_modules[mod_name]: + break + + # stop processing this module if/when all symbols are found + if not wanted_modules[mod_name]: + del wanted_modules[mod_name] + break + + return found_symbols + + @staticmethod + def get_kernel_modules( + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + filter_modules: Optional[filter_modules_type], + ) -> Dict[str, List[Tuple[str, int, int]]]: + """ + Walks the kernel module list and finds the session layer, base, and size of each wanted module + """ + found_modules: Dict[str, List[Tuple[str, int, int]]] = {} + + if filter_modules: + # create a tuple of module names for use with `endswith` + filter_modules_check = tuple([key.lower() for key in filter_modules.keys()]) + else: + filter_modules_check = None + + session_layers = list( + modules.Modules.get_session_layers(context, layer_name, symbol_table) + ) + + # special handling for the kernel + gather_kernel = ( + filter_modules_check and PESymbols.os_module_name in filter_modules_check + ) + + for index, mod in enumerate( + modules.Modules.list_modules(context, layer_name, symbol_table) + ): + try: + mod_name = str(mod.BaseDllName.get_string().lower()) + except exceptions.InvalidAddressException: + continue + + # to analyze, it must either be the kernel or a wanted module + if not filter_modules_check or (gather_kernel and index == 0): + mod_name = PESymbols.os_module_name + elif filter_modules_check and not mod_name.endswith(filter_modules_check): + continue + + # we won't find symbol information if we can't analyze the module + session_layer_name = modules.Modules.find_session_layer( + context, session_layers, mod.DllBase + ) + if not session_layer_name: + continue + + if mod_name not in found_modules: + found_modules[mod_name] = [] + + found_modules[mod_name].append( + (session_layer_name, mod.DllBase, mod.SizeOfImage) + ) + + return found_modules + + @staticmethod + def get_process_modules( + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + filter_modules: Optional[filter_modules_type], + ) -> Dict[str, List[Tuple[str, int, int]]]: + """ + Walks the process list and each process' VAD to determine the base address and size of wanted modules + """ + proc_modules: Dict[str, List[Tuple[str, int, int]]] = {} + + if filter_modules: + # create a tuple of module names for use with `endswith` + filter_modules_check = tuple([key.lower() for key in filter_modules.keys()]) + else: + filter_modules_check = None + + for _, proc_layer_name, vads in vadinfo.VadInfo.get_all_vads_with_file_paths( + context, layer_name, symbol_table + ): + for vad_start, vad_size, filepath in vads: + filename = PESymbols.filename_for_path(filepath) + + if filter_modules_check and not filename.endswith(filter_modules_check): + continue + + # track each module along with the process layer and range to find it + if filename not in proc_modules: + proc_modules[filename] = [] + + proc_modules[filename].append((proc_layer_name, vad_start, vad_size)) + + return proc_modules + + def _generator(self) -> Generator[Tuple[int, Tuple[str, str, int]], None, None]: + kernel = self.context.modules[self.config["kernel"]] + + if self.config["symbol"]: + filter_module = { + self.config["module"].lower(): { + PESymbols.wanted_names: [self.config["symbol"]] + } + } + + elif self.config["address"]: + filter_module = { + self.config["module"].lower(): { + PESymbols.wanted_addresses: [self.config["address"]] + } + } + + else: + vollog.error("--address or --symbol must be specified") + return + + if self.config["source"] == "kernel": + module_resolver = self.get_kernel_modules + else: + module_resolver = self.get_process_modules + + collected_modules = module_resolver( + self.context, kernel.layer_name, kernel.symbol_table_name, filter_module + ) + + found_symbols = PESymbols.find_symbols( + self.context, self.config_path, filter_module, collected_modules + ) + + for module, symbols in found_symbols.items(): + for symbol, address in symbols: + yield (0, (module, symbol, format_hints.Hex(address))) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Module", str), + ("Symbol", str), + ("Address", format_hints.Hex), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py new file mode 100644 index 000000000..0438bc9e3 --- /dev/null +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -0,0 +1,183 @@ +import logging + +from typing import Dict, Tuple, List, Generator + +from volatility3.framework import interfaces, exceptions +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.plugins.windows import pslist, pe_symbols + +vollog = logging.getLogger(__name__) + + +class unhooked_system_calls(interfaces.plugins.PluginInterface): + """Looks for signs of Skeleton Key malware""" + + _required_framework_version = (2, 4, 0) + + system_calls = { + "ntdll.dll": { + pe_symbols.PESymbols.wanted_names: [ + "NtCreateThread", + "NtProtectVirtualMemory", + "NtReadVirtualMemory", + "NtOpenProcess", + "NtWriteFile", + "NtQueryVirtualMemory", + "NtAllocateVirtualMemory", + "NtWorkerFactoryWorkerReady", + "NtAcceptConnectPort", + "NtAddDriverEntry", + "NtAdjustPrivilegesToken", + "NtAlpcCreatePort", + "NtClose", + "NtCreateFile", + "NtCreateMutant", + "NtOpenFile", + "NtOpenIoCompletion", + "NtOpenJobObject", + "NtOpenKey", + "NtOpenKeyEx", + "NtOpenThread", + "NtOpenThreadToken", + "NtOpenThreadTokenEx", + "NtWriteVirtualMemory", + "NtTraceEvent", + "NtTranslateFilePath", + "NtUmsThreadYield", + "NtUnloadDriver", + "NtUnloadKey", + "NtUnloadKey2", + "NtUnloadKeyEx", + "NtCreateKey", + "NtCreateSection", + "NtDeleteKey", + "NtDeleteValueKey", + "NtDuplicateObject", + "NtQueryValueKey", + "NtReplaceKey", + "NtRequestWaitReplyPort", + "NtRestoreKey", + "NtSetContextThread", + "NtSetSecurityObject", + "NtSetValueKey", + "NtSystemDebugControl", + "NtTerminateProcess", + ] + } + } + + _code_bytes_type = Dict[str, Dict[str, Dict[bytes, List[Tuple[int, str]]]]] + + @classmethod + def get_requirements(cls) -> List: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="pe_symbols", plugin=pe_symbols.PESymbols, version=(1, 0, 0) + ), + ] + + def _gather_code_bytes( + self, + kernel: interfaces.context.ModuleInterface, + found_symbols: pe_symbols.PESymbols.found_symbols_type, + ) -> _code_bytes_type: + """ + Enumerates the desired DLLs and function implementations in each process + Groups based on unique implementations of each DLLs' functions + The purpose is to detect when a function has different implementations (code) + in different processes. + This very effectively detects code injection. + """ + code_bytes: unhooked_system_calls._code_bytes_type = {} + + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) + + for proc in procs: + try: + proc_id = proc.UniqueProcessId + proc_name = utility.array_to_string(proc.ImageFileName) + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + for dll_name, functions in found_symbols.items(): + for func_name, func_addr in functions: + try: + fbytes = self.context.layers[proc_layer_name].read( + func_addr, 0x20 + ) + except exceptions.InvalidAddressException: + continue + + if dll_name not in code_bytes: + code_bytes[dll_name] = {} + + if func_name not in code_bytes[dll_name]: + code_bytes[dll_name][func_name] = {} + + if fbytes not in code_bytes[dll_name][func_name]: + code_bytes[dll_name][func_name][fbytes] = [] + + code_bytes[dll_name][func_name][fbytes].append((proc_id, proc_name)) + + return code_bytes + + def _generator(self) -> Generator[Tuple[int, Tuple[str, str, int]], None, None]: + kernel = self.context.modules[self.config["kernel"]] + + found_symbols = pe_symbols.PESymbols.addresses_for_process_symbols( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + unhooked_system_calls.system_calls, + ) + + # code_bytes[dll_name][func_name][func_bytes] + code_bytes = self._gather_code_bytes(kernel, found_symbols) + + for functions in code_bytes.values(): + for func_name, cbb in functions.items(): + cb = list(cbb.values()) + + # same implementation in all + if len(cb) == 1: + yield 0, (func_name, "", len(cb[0])) + else: + # find the processes that are hooked for reporting + max_idx = 0 if len(cb[0]) > len(cb[1]) else 1 + small_idx = (~max_idx) & 1 + + ps = [] + + for pid, pname in cb[small_idx]: + ps.append("{:d}:{}".format(pid, pname)) + + proc_names = ", ".join(ps) + + yield 0, (func_name, proc_names, len(cb[max_idx])) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Function", str), + ("Distinct Implementations", str), + ("Total Implementations", int), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index abc6142fe..97a2f2455 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -3,13 +3,13 @@ # import logging -from typing import Callable, List, Generator, Iterable, Type, Optional +from typing import Callable, List, Generator, Iterable, Type, Optional, Tuple -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import renderers, interfaces, exceptions, symbols from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist +from volatility3.plugins.windows import pslist, pe_symbols vollog = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class VadInfo(interfaces.plugins.PluginInterface): _version = (2, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): # type: ignore super().__init__(*args, **kwargs) self._protect_values = None @@ -107,6 +107,58 @@ class VadInfo(interfaces.plugins.PluginInterface): ) return values # type: ignore + @staticmethod + def get_proc_vads_with_file_paths( + proc: interfaces.objects.ObjectInterface, + ) -> pe_symbols.PESymbols.ranges_type: + """ + Returns a list of the process' vads that map a file + """ + vads = [] + + for vad in proc.get_vad_root().traverse(): + filepath = vad.get_file_name() + if not isinstance(filepath, str) or filepath.count("\\") == 0: + continue + + vads.append((vad.get_start(), vad.get_size(), filepath)) + + return vads + + @classmethod + def get_all_vads_with_file_paths( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table_name: str, + ) -> Generator[ + Tuple[ + interfaces.objects.ObjectInterface, str, pe_symbols.PESymbols.ranges_type + ], + None, + None, + ]: + """ + Yields each set of vads for a process that have a file mapped, along with the process itself and its layer + """ + is_32bit_arch = not symbols.symbol_table_is_64bit(context, symbol_table_name) + + procs = pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, + ) + + for proc in procs: + try: + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + vads = cls.get_proc_vads_with_file_paths(proc) + + yield proc, proc_layer_name, vads + @classmethod def list_vads( cls, @@ -196,11 +248,33 @@ class VadInfo(interfaces.plugins.PluginInterface): return file_handle - def _generator(self, procs): + def _generator( + self, procs: List[interfaces.objects.ObjectInterface] + ) -> Generator[ + Tuple[ + int, + Tuple[ + int, + str, + format_hints.Hex, + format_hints.Hex, + format_hints.Hex, + str, + str, + int, + int, + format_hints.Hex, + str, + str, + ], + ], + None, + None, + ]: kernel = self.context.modules[self.config["kernel"]] kernel_layer = self.context.layers[kernel.layer_name] - def passthrough(_: interfaces.objects.ObjectInterface) -> bool: + def passthrough(x: interfaces.objects.ObjectInterface) -> bool: return False filter_func = passthrough @@ -250,7 +324,7 @@ class VadInfo(interfaces.plugins.PluginInterface): ), ) - def run(self): + def run(self) -> renderers.TreeGrid: kernel = self.context.modules[self.config["kernel"]] filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) From 3bb9264a5770d6828487129b6ebfec509ade8680 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:11:16 -0500 Subject: [PATCH 073/110] formatting --- volatility3/framework/plugins/windows/debugregisters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 5f3c2e519..c70f922ae 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -37,7 +37,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) ), - ] + ] def _get_debug_info( self, ethread: interfaces.objects.ObjectInterface From 30cb5bd3ab4bac9f3ebb224d2f37cb98d6960545 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:14:06 -0500 Subject: [PATCH 074/110] Formatting that my local black doesn't understand --- volatility3/framework/plugins/windows/vadinfo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 97a2f2455..b3b7bf5fb 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -249,8 +249,7 @@ class VadInfo(interfaces.plugins.PluginInterface): return file_handle def _generator( - self, procs: List[interfaces.objects.ObjectInterface] - ) -> Generator[ + self, procs: List[interfaces.objects.ObjectInterface]) -> Generator[ Tuple[ int, Tuple[ From c223ac6e762d1072f04c21276645d7c2dda79dde Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:31:55 -0500 Subject: [PATCH 075/110] more black help --- volatility3/framework/plugins/windows/vadinfo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index b3b7bf5fb..655e54be6 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -141,8 +141,6 @@ class VadInfo(interfaces.plugins.PluginInterface): """ Yields each set of vads for a process that have a file mapped, along with the process itself and its layer """ - is_32bit_arch = not symbols.symbol_table_is_64bit(context, symbol_table_name) - procs = pslist.PsList.list_processes( context=context, layer_name=layer_name, From 7b48ee4be489b4667c57cca55da84abd3c1c3d12 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:36:13 -0500 Subject: [PATCH 076/110] more black help --- volatility3/framework/plugins/windows/vadinfo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 655e54be6..594c0d67c 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -246,8 +246,7 @@ class VadInfo(interfaces.plugins.PluginInterface): return file_handle - def _generator( - self, procs: List[interfaces.objects.ObjectInterface]) -> Generator[ + def _generator(self, procs: List[interfaces.objects.ObjectInterface]) -> Generator[ Tuple[ int, Tuple[ From bcd93616c0628fdb991866290845bbb861889f8f Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:36:35 -0500 Subject: [PATCH 077/110] more black help --- volatility3/framework/plugins/windows/vadinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 594c0d67c..e129ad4ef 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -5,7 +5,7 @@ import logging from typing import Callable, List, Generator, Iterable, Type, Optional, Tuple -from volatility3.framework import renderers, interfaces, exceptions, symbols +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints From fd8777287736baee75e6f1a8843b471a47b6ced2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:47:42 -0500 Subject: [PATCH 078/110] Move VAD enumeration into pe_symbols --- .../framework/plugins/windows/pe_symbols.py | 57 +++++++++++++++++-- .../framework/plugins/windows/vadinfo.py | 50 ---------------- 2 files changed, 52 insertions(+), 55 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index b7faeaf55..51558f71b 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -16,7 +16,7 @@ from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbutil from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, vadinfo, modules +from volatility3.plugins.windows import pslist, modules vollog = logging.getLogger(__name__) @@ -170,9 +170,6 @@ class PESymbols(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) - ), requirements.VersionRequirement( name="modules", component=modules.Modules, version=(2, 0, 0) ), @@ -648,6 +645,56 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_modules + @staticmethod + def get_proc_vads_with_file_paths( + proc: interfaces.objects.ObjectInterface, + ) -> ranges_type: + """ + Returns a list of the process' vads that map a file + """ + vads = [] + + for vad in proc.get_vad_root().traverse(): + filepath = vad.get_file_name() + if not isinstance(filepath, str) or filepath.count("\\") == 0: + continue + + vads.append((vad.get_start(), vad.get_size(), filepath)) + + return vads + + @classmethod + def get_all_vads_with_file_paths( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table_name: str, + ) -> Generator[ + Tuple[ + interfaces.objects.ObjectInterface, str, ranges_type + ], + None, + None, + ]: + """ + Yields each set of vads for a process that have a file mapped, along with the process itself and its layer + """ + procs = pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, + ) + + for proc in procs: + try: + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + vads = PESymbols.get_proc_vads_with_file_paths(proc) + + yield proc, proc_layer_name, vads + @staticmethod def get_process_modules( context: interfaces.context.ContextInterface, @@ -666,7 +713,7 @@ class PESymbols(interfaces.plugins.PluginInterface): else: filter_modules_check = None - for _, proc_layer_name, vads in vadinfo.VadInfo.get_all_vads_with_file_paths( + for _, proc_layer_name, vads in PESymbols.get_all_vads_with_file_paths( context, layer_name, symbol_table ): for vad_start, vad_size, filepath in vads: diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index e129ad4ef..974793a71 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -107,56 +107,6 @@ class VadInfo(interfaces.plugins.PluginInterface): ) return values # type: ignore - @staticmethod - def get_proc_vads_with_file_paths( - proc: interfaces.objects.ObjectInterface, - ) -> pe_symbols.PESymbols.ranges_type: - """ - Returns a list of the process' vads that map a file - """ - vads = [] - - for vad in proc.get_vad_root().traverse(): - filepath = vad.get_file_name() - if not isinstance(filepath, str) or filepath.count("\\") == 0: - continue - - vads.append((vad.get_start(), vad.get_size(), filepath)) - - return vads - - @classmethod - def get_all_vads_with_file_paths( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table_name: str, - ) -> Generator[ - Tuple[ - interfaces.objects.ObjectInterface, str, pe_symbols.PESymbols.ranges_type - ], - None, - None, - ]: - """ - Yields each set of vads for a process that have a file mapped, along with the process itself and its layer - """ - procs = pslist.PsList.list_processes( - context=context, - layer_name=layer_name, - symbol_table=symbol_table_name, - ) - - for proc in procs: - try: - proc_layer_name = proc.add_process_layer() - except exceptions.InvalidAddressException: - continue - - vads = cls.get_proc_vads_with_file_paths(proc) - - yield proc, proc_layer_name, vads - @classmethod def list_vads( cls, From 61cf58d97794359e3e093a97f3b8d2562661aea4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:49:01 -0500 Subject: [PATCH 079/110] black fixes --- volatility3/framework/plugins/windows/pe_symbols.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 51558f71b..10e87c552 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -670,9 +670,7 @@ class PESymbols(interfaces.plugins.PluginInterface): layer_name: str, symbol_table_name: str, ) -> Generator[ - Tuple[ - interfaces.objects.ObjectInterface, str, ranges_type - ], + Tuple[interfaces.objects.ObjectInterface, str, ranges_type], None, None, ]: From 802fc024b5a7666f1e56a641af4fe219c856f6b4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 13:27:04 -0500 Subject: [PATCH 080/110] switch api place --- volatility3/framework/plugins/windows/debugregisters.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index c70f922ae..931ccef37 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -9,7 +9,6 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints import volatility3.plugins.windows.pslist as pslist import volatility3.plugins.windows.threads as threads -import volatility3.plugins.windows.vadinfo as vadinfo import volatility3.plugins.windows.pe_symbols as pe_symbols vollog = logging.getLogger(__name__) @@ -31,9 +30,6 @@ class DebugRegisters(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) - ), requirements.VersionRequirement( name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) ), @@ -80,7 +76,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): if owner_proc.vol.offset in vads_cache: vads = vads_cache[owner_proc.vol.offset] else: - vads = vadinfo.VadInfo.get_proc_vads_with_file_paths(owner_proc) + vads = pe_symbols.PESymbols.get_proc_vads_with_file_paths(owner_proc) vads_cache[owner_proc.vol.offset] = vads # smear or terminated process From 26cd73115b58cf5b1adfb430c9f1070e45dbe727 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 13:31:09 +0100 Subject: [PATCH 081/110] CLI: Filter on rendered values --- volatility3/cli/text_renderer.py | 38 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ab9e44141..1f9cfa9d8 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -179,7 +179,15 @@ class QuickTextRenderer(CLIRenderer): outfd.write("\n{}\n".format("\t".join(line))) def visitor(node: interfaces.renderers.TreeNode, accumulator): - if self.filter and self.filter.filter(node.values): + line = [] + for column_index in range(len(grid.columns)): + column = grid.columns[column_index] + renderer = self._type_renderers.get( + column.type, self._type_renderers["default"] + ) + line.append(renderer(node.values[column_index])) + + if self.filter and self.filter.filter(line): return accumulator accumulator.write("\n") @@ -188,13 +196,6 @@ class QuickTextRenderer(CLIRenderer): "*" * max(0, node.path_depth - 1) + ("" if (node.path_depth <= 1) else " ") ) - line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] - renderer = self._type_renderers.get( - column.type, self._type_renderers["default"] - ) - line.append(renderer(node.values[column_index])) accumulator.write("{}".format("\t".join(line))) accumulator.flush() return accumulator @@ -259,12 +260,18 @@ class CSVRenderer(CLIRenderer): def visitor(node: interfaces.renderers.TreeNode, accumulator): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case row = {"TreeDepth": str(max(0, node.path_depth - 1))} + line = [] for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) row[f"{column.name}"] = renderer(node.values[column_index]) + line.append(row[f"{column.name}"]) + + if self.filter and self.filter.filter(line): + return accumulator + accumulator.writerow(row) return accumulator @@ -317,10 +324,8 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths.get(tree_indent_column, 0), node.path_depth ) - if self.filter and self.filter.filter(node.values): - return accumulator - line = {} + rendered_line = [] for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get( @@ -334,6 +339,11 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths.get(column.name, len(column.name)), field_width ) line[column] = data.split("\n") + rendered_line.append(data) + + if self.filter and self.filter.filter(rendered_line): + return accumulator + accumulator.append((node.path_depth, line)) return accumulator @@ -437,6 +447,7 @@ class JsonRenderer(CLIRenderer): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case acc_map, final_tree = accumulator node_dict: Dict[str, Any] = {"__children": []} + line = [] for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get( @@ -446,6 +457,11 @@ class JsonRenderer(CLIRenderer): if isinstance(data, interfaces.renderers.BaseAbsentValue): data = None node_dict[column.name] = data + line.append(data) + + if self.filter and self.filter.filter(line): + return accumulator + if node.parent: acc_map[node.parent.path]["__children"].append(node_dict) else: From 8259ca90199f548e1592aa3ab05444c9353fe66a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 13:47:26 +0100 Subject: [PATCH 082/110] Core: Bump the framework number so we can differentiate CLI versions --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index d30446d63..a131609c9 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 9 # Number of changes that only add to the interface +VERSION_MINOR = 10 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 4361c3857393b7f7311e9b600d1ea5d4966cdb93 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 14:55:04 +0100 Subject: [PATCH 083/110] Core: Verify plugin requirements of plugins --- .../framework/configuration/requirements.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 1c0622574..931995bb8 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -585,6 +585,24 @@ class PluginRequirement(VersionRequirement): version=version, ) + def unsatisfied( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: + result = super().unsatisfied(context, config_path) + if not result: + component: Type[interfaces.plugins.PluginInterface] = self._component + for requirement in component.get_requirements(): + if isinstance(requirement, PluginRequirement): + result.update( + requirement.unsatisfied( + context, + interfaces.configuration.path_join(config_path, self.name), + ) + ) + if result: + result[config_path] = self + return result + class ModuleRequirement( interfaces.configuration.ConstructableRequirementInterface, From f3085b6c59b05190320722feef58f276357d2155 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 16:06:23 +0100 Subject: [PATCH 084/110] Core: Move the pluginrequirement check to generic versionrequirement --- .../framework/configuration/requirements.py | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 931995bb8..f95dac307 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -527,12 +527,14 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, optional: bool = False, component: Type[interfaces.configuration.VersionableInterface] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: + if description is None: + description = f"Version {".".join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) @@ -550,9 +552,29 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} + + # Check for child requirements + if issubclass(self._component, interfaces.configuration.ConfigurableInterface): + result = {} + for requirement in self._component.get_requirements(): + if not requirement.optional and isinstance( + requirement, VersionRequirement + ): + result.update( + requirement.unsatisfied( + context, + config_path, + ) + ) + + if result: + result.update({config_path: self}) + return result + context.config[interfaces.configuration.path_join(config_path, self.name)] = ( True ) + return {} @classmethod @@ -585,24 +607,6 @@ class PluginRequirement(VersionRequirement): version=version, ) - def unsatisfied( - self, context: interfaces.context.ContextInterface, config_path: str - ) -> Dict[str, interfaces.configuration.RequirementInterface]: - result = super().unsatisfied(context, config_path) - if not result: - component: Type[interfaces.plugins.PluginInterface] = self._component - for requirement in component.get_requirements(): - if isinstance(requirement, PluginRequirement): - result.update( - requirement.unsatisfied( - context, - interfaces.configuration.path_join(config_path, self.name), - ) - ) - if result: - result[config_path] = self - return result - class ModuleRequirement( interfaces.configuration.ConstructableRequirementInterface, From ace590e8f0669797b5770773eeadcbcb27752a78 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 16:08:48 +0100 Subject: [PATCH 085/110] Core: Fix up f-string containing a string --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index f95dac307..3a862a132 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -534,7 +534,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): version: Optional[Tuple[int, ...]] = None, ) -> None: if description is None: - description = f"Version {".".join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" + description = f"Version {'.'.join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) From bf000ff0a0bafc0197ee1dd074f1df450362d02f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 21:42:59 +0100 Subject: [PATCH 086/110] Core: Add recursion protection to VersionRequirement check --- .../framework/configuration/requirements.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 3a862a132..49ca49b59 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -546,13 +546,26 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): self._version = version def unsatisfied( - self, context: interfaces.context.ContextInterface, config_path: str + self, + context: interfaces.context.ContextInterface, + config_path: str, + accumulator: Optional[ + List[interfaces.configuration.VersionableInterface] + ] = None, ) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} + if accumulator is None: + accumulator = set([self._component]) + else: + if self._component in accumulator: + return {config_path: self} + else: + accumulator.add(self._component) + # Check for child requirements if issubclass(self._component, interfaces.configuration.ConfigurableInterface): result = {} @@ -562,8 +575,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): ): result.update( requirement.unsatisfied( - context, - config_path, + context, config_path, accumulator.copy() ) ) From 9152f33181fd347366468d00c47a2c22ae71cfc2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 21:58:09 +0100 Subject: [PATCH 087/110] Core: Allow circular dependencies as long as they are all met --- volatility3/framework/configuration/requirements.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 49ca49b59..dcb7505ea 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -558,16 +558,20 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): if not self.matches_required(self._version, self._component.version): return {config_path: self} + recurse = True if accumulator is None: accumulator = set([self._component]) else: if self._component in accumulator: - return {config_path: self} + recurse = False else: accumulator.add(self._component) # Check for child requirements - if issubclass(self._component, interfaces.configuration.ConfigurableInterface): + if ( + issubclass(self._component, interfaces.configuration.ConfigurableInterface) + and recurse + ): result = {} for requirement in self._component.get_requirements(): if not requirement.optional and isinstance( From fd2c97e1bafb5b30403a1788eae4c32723438746 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 9 Sep 2024 00:44:00 +0100 Subject: [PATCH 088/110] CLI: Use enumerate for renderers (thanks @gcmoreira) --- volatility3/cli/text_renderer.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 1f9cfa9d8..da0cdf62a 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -180,8 +180,7 @@ class QuickTextRenderer(CLIRenderer): def visitor(node: interfaces.renderers.TreeNode, accumulator): line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) @@ -261,8 +260,7 @@ class CSVRenderer(CLIRenderer): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case row = {"TreeDepth": str(max(0, node.path_depth - 1))} line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) @@ -326,8 +324,7 @@ class PrettyTextRenderer(CLIRenderer): line = {} rendered_line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) @@ -357,8 +354,7 @@ class PrettyTextRenderer(CLIRenderer): format_string_list = [ "{0:<" + str(max_column_widths.get(tree_indent_column, 0)) + "s}" ] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): format_string_list.append( "{" + str(column_index + 1) @@ -448,8 +444,7 @@ class JsonRenderer(CLIRenderer): acc_map, final_tree = accumulator node_dict: Dict[str, Any] = {"__children": []} line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) From b7e604d63f667663f9a00415493ec4f8b12a5024 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 10 Sep 2024 14:50:40 -0500 Subject: [PATCH 089/110] Address feedback --- .../plugins/windows/debugregisters.py | 33 +- .../framework/plugins/windows/pe_symbols.py | 394 ++++++++++++++---- .../plugins/windows/unhooked_system_calls.py | 7 +- .../framework/plugins/windows/vadinfo.py | 4 +- 4 files changed, 331 insertions(+), 107 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 931ccef37..65b2e625b 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -1,3 +1,6 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + import logging from typing import Tuple, Optional, Generator, List, Dict @@ -35,11 +38,16 @@ class DebugRegisters(interfaces.plugins.PluginInterface): ), ] + @staticmethod def _get_debug_info( - self, ethread: interfaces.objects.ObjectInterface + ethread: interfaces.objects.ObjectInterface, ) -> Optional[Tuple[interfaces.objects.ObjectInterface, int, int, int, int, int]]: """ Gathers information related to the debug registers for the given thread + Args: + ethread: the thread (_ETHREAD) to examine + Returns: + Tuple[interfaces.objects.ObjectInterface, int, int, int, int, int]: The owner process of the thread and the values for dr7, dr0, dr1, dr2, dr3 """ try: dr7 = ethread.Tcb.TrapFrame.Dr7 @@ -68,23 +76,6 @@ class DebugRegisters(interfaces.plugins.PluginInterface): return owner_proc, dr7, dr0, dr1, dr2, dr3 - def _get_vads( - self, - vads_cache: Dict[int, List[Tuple[int, int, str]]], - owner_proc: interfaces.objects.ObjectInterface, - ) -> Optional[List[Tuple[int, int, str]]]: - if owner_proc.vol.offset in vads_cache: - vads = vads_cache[owner_proc.vol.offset] - else: - vads = pe_symbols.PESymbols.get_proc_vads_with_file_paths(owner_proc) - vads_cache[owner_proc.vol.offset] = vads - - # smear or terminated process - if len(vads) == 0: - return None - - return vads - def _generator( self, ) -> Generator[ @@ -115,7 +106,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): ]: kernel = self.context.modules[self.config["kernel"]] - vads_cache: Dict[int, List[Tuple[int, int, str]]] = {} + vads_cache: Dict[int, pe_symbols.ranges_type] = {} proc_modules = None @@ -133,7 +124,9 @@ class DebugRegisters(interfaces.plugins.PluginInterface): owner_proc, dr7, dr0, dr1, dr2, dr3 = debug_info - vads = self._get_vads(vads_cache, owner_proc) + vads = pe_symbols.PESymbols.get_vads_for_process_cache( + vads_cache, owner_proc + ) if not vads: continue diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 10e87c552..c9785a1c9 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -1,9 +1,9 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 - import io import logging +import ntpath from typing import Dict, Tuple, Optional, List, Generator, Union @@ -17,9 +17,37 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbutil from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import pslist, modules +from volatility3.framework.constants.windows import KERNEL_MODULE_NAMES vollog = logging.getLogger(__name__) +# keys for specifying wanted names and/or addresses +# used for consistent access between the API and plugins +wanted_names_identifier = "names" +wanted_addresses_identifier = "addresses" + +# how wanted modules/symbols are specified, such as: +# {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} +# {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} +filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + +# holds resolved symbols +# {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} +found_symbols_type = Dict[str, List[Tuple[str, int]]] + +# used to hold informatin about a range (VAD or kernel module) +# (start address, size, file path) +range_type = Tuple[int, int, str] +ranges_type = List[range_type] + +# collected_modules are modules and their symbols found when walking vads or kernel modules +# Tuple of (process or kernel layer name, range start, range size) +collected_module_instance = Tuple[str, int, int] +collected_modules_info = List[collected_module_instance] +collected_modules_type = Dict[str, collected_modules_info] + +PESymbolFinders = Union[interfaces.context.ModuleInterface, pefile.ExportDirData] + class PESymbolFinder: """ @@ -28,21 +56,19 @@ class PESymbolFinder: lookup symbols by name or address """ - cached_str = Union[str, None] - cached_str_dict = Dict[str, cached_str] + cached_str_dict = Dict[str, Optional[str]] - cached_int = Union[int, None] - cached_int_dict = Dict[str, cached_int] + cached_int_dict = Dict[str, Optional[int]] cached_value = Union[int, str, None] - cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + cached_value_dict = Dict[str, Dict[str, List[str]] | Dict[str, List[int]]] def __init__( self, layer_name: str, mod_name: str, module_start: int, - symbol_module: Union[interfaces.context.ModuleInterface, pefile.ExportDirData], + symbol_module: PESymbolFinders, ): self._layer_name = layer_name self._mod_name = mod_name @@ -56,10 +82,25 @@ class PESymbolFinder: """ Maintain a cache for symbol lookups to avoid re-walking of PDB symbols or export tables within the same module for the same address in the same layer + + Args: + value: The value (address or name) being cached + + Returns: + str: The constructed cache key that includes the layer and module name """ return f"{self._layer_name}|{self._mod_name}|{value}" - def get_name_for_address(self, address: int) -> cached_str: + def get_name_for_address(self, address: int) -> Optional[str]: + """ + Returns the name for the given address within the particular layer and module + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ cached_key = self._get_cache_key(address) if cached_key not in self._name_cache: name = self._do_get_name(address) @@ -67,7 +108,16 @@ class PESymbolFinder: return self._name_cache[cached_key] - def get_address_for_name(self, name: str) -> cached_int: + def get_address_for_name(self, name: str) -> Optional[int]: + """ + Returns the name for the given address within the particular layer and module + + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ cached_key = self._get_cache_key(name) if cached_key not in self._address_cache: address = self._do_get_address(name) @@ -75,10 +125,30 @@ class PESymbolFinder: return self._address_cache[cached_key] - def _do_get_name(self, address: int) -> cached_str: + def _do_get_name(self, address: int) -> Optional[str]: + """ + Returns the name for the given address within the particular layer and module. + This method must be overwritten by sub classes. + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ raise NotImplementedError("_do_get_name must be overwritten") - def _do_get_address(self, name: str) -> cached_int: + def _do_get_address(self, name: str) -> Optional[int]: + """ + Returns the name for the given address within the particular layer and module + This method must be overwritten by sub classes. + + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ raise NotImplementedError("_do_get_address must be overwritten") @@ -87,13 +157,31 @@ class PDBSymbolFinder(PESymbolFinder): PESymbolFinder implementation for PDB modules """ - def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + def _do_get_address(self, name: str) -> Optional[int]: + """ + _do_get_address implementation for PDBSymbolFinder + + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ try: return self._symbol_module.get_absolute_symbol_address(name) except exceptions.SymbolError: return None - def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + def _do_get_name(self, address: int) -> Optional[str]: + """ + _do_get_name implementation for PDBSymbolFinder + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ try: name = self._symbol_module.get_symbols_by_absolute_location(address)[0] return name.split(constants.BANG)[1] @@ -113,14 +201,32 @@ class ExportSymbolFinder(PESymbolFinder): except AttributeError: return None - def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + def _do_get_name(self, address: int) -> Optional[str]: + """ + _do_get_name implementation for ExportSymbolFinder + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ for export in self._symbol_module: if export.address + self._module_start == address: return self._get_name(export) return None - def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + def _do_get_address(self, name: str) -> Optional[int]: + """ + _do_get_address implementation for ExportSymbolFinder + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ + for export in self._symbol_module: sym_name = self._get_name(export) if sym_name and sym_name == name: @@ -139,25 +245,6 @@ class PESymbols(interfaces.plugins.PluginInterface): # used for special handling of the kernel PDB file. See later notes os_module_name = "ntoskrnl.exe" - # keys for specifying wanted names and/or addresses - # used for consistent access between the API and plugins - wanted_names = "names" - wanted_addresses = "addresses" - - # how wanted modules/symbols are specified, such as: - # {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} - # {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} - filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] - - # holds resolved symbols - # {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} - found_symbols_type = Dict[str, List[Tuple[str, int]]] - - # used to hold informatin about a range (VAD or kernel module) - # (start address, size, file path) - range_type = Tuple[int, int, str] - ranges_type = List[range_type] - @classmethod def get_requirements(cls) -> List: # Since we're calling the plugin, make sure we have the plugin's requirements @@ -187,13 +274,15 @@ class PESymbols(interfaces.plugins.PluginInterface): description='Module in which to resolve symbols. Use "ntoskrnl.exe" to resolve in the base kernel executable.', optional=False, ), - requirements.StringRequirement( - name="symbol", + requirements.ListRequirement( + name="symbols", + element_type=str, description="Symbol name to resolve", optional=True, ), - requirements.IntRequirement( - name="address", + requirements.ListRequirement( + name="addresses", + element_type=int, description="Address of symbol to resolve", optional=True, ), @@ -242,7 +331,15 @@ class PESymbols(interfaces.plugins.PluginInterface): ranges: ranges_type, address: int ) -> Optional[range_type]: """ - Helper for getting the range information for an address + Helper for getting the range information for an address. + Finds the range holding the `address` parameter + + Args: + address: the address to find the range for + + Returns: + Tuple[int, int, str]: The starting address, size, and file path of the range + """ for start, size, filepath in ranges: if start <= address < start + size: @@ -254,6 +351,13 @@ class PESymbols(interfaces.plugins.PluginInterface): def filepath_for_address(ranges: ranges_type, address: int) -> Optional[str]: """ Helper to get the file path for an address + + Args: + ranges: The set of VADs with mapped files to find the address + address: The address to find inside of the VADs set + + Returns: + str: The full path of the file, if found and present """ info = PESymbols.range_info_for_address(ranges, address) if info: @@ -264,9 +368,15 @@ class PESymbols(interfaces.plugins.PluginInterface): @staticmethod def filename_for_path(filepath: str) -> str: """ - Consistent way to get the filename + Consistent way to get the filename regardless of platform + + Args: + str: the file path from `filepath_for_address` + + Returns: + str: the bsae file name of the full path """ - return filepath.split("\\")[-1] + return ntpath.basename(filepath) @staticmethod def addresses_for_process_symbols( @@ -276,6 +386,18 @@ class PESymbols(interfaces.plugins.PluginInterface): symbol_table_name: str, symbols: filter_modules_type, ) -> found_symbols_type: + """ + Used to easily resolve the addresses of names inside of modules. + + See the usage of this function for system call resolution in unhooked_system_calls.py + for an easy to understand example. + + Args: + symbols: The dictionary of symbols requested by the caller + + Returns: + found_symbols_type: The dictionary of symbols that were resolved + """ collected_modules = PESymbols.get_process_modules( context, layer_name, symbol_table_name, symbols ) @@ -294,16 +416,22 @@ class PESymbols(interfaces.plugins.PluginInterface): def path_and_symbol_for_address( context: interfaces.context.ContextInterface, config_path: str, - collected_modules: Dict[str, List[Tuple[str, int, int]]], + collected_modules: collected_modules_type, ranges: ranges_type, address: int, ) -> Tuple[str, str]: """ Method for plugins to determine the file path and symbol name for a given address - collected_modules: return value from `get_kernel_modules` or `get_process_modules` - ranges: the memory ranges to examine in this layer. - address: address to resolve to its symbol name + See debugregisters.py for an example of how this function is used along with get_vads_for_process_cache + for resolving symbols in processes. + + Args: + collected_modules: return value from `get_kernel_modules` or `get_process_modules` + ranges: the memory ranges to examine in this layer. + address: address to resolve to its symbol name + Returns: + Tuple[str|renderers.NotApplicableValue|renderers.NotAvailableValue, str|renderers.NotApplicableValue|renderers.NotAvailableValue] """ if not address: @@ -317,15 +445,15 @@ class PESymbols(interfaces.plugins.PluginInterface): filename = PESymbols.filename_for_path(filepath).lower() # setup to resolve the address - filter_module: PESymbols.filter_modules_type = { - filename: {PESymbols.wanted_addresses: [address]} + filter_module: filter_modules_type = { + filename: {wanted_addresses_identifier: [address]} } found_symbols = PESymbols.find_symbols( context, config_path, filter_module, collected_modules ) - if not found_symbols or not found_symbols[filename]: + if not found_symbols or filename not in found_symbols: return renderers.NotAvailableValue(), renderers.NotAvailableValue() return filepath, found_symbols[filename][0][0] @@ -335,13 +463,18 @@ class PESymbols(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, pe_table_name: str, mod_name: str, - module_info: Tuple[str, int, int], + module_info: collected_module_instance, ) -> Optional[ExportSymbolFinder]: """ Attempts to locate symbols based on export analysis - mod_name: lower case name of the module to resolve symbols in - module_info: (layer_name, module_start, module_size) of the module to examine + Args: + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + + Returns: + Optional[ExportSymbolFinder]: If the export table can be resolved, then the ExportSymbolFinder + instance for it """ layer_name = module_info[0] @@ -361,7 +494,10 @@ class PESymbols(interfaces.plugins.PluginInterface): return None return ExportSymbolFinder( - layer_name, mod_name, module_start, pe_module.DIRECTORY_ENTRY_EXPORT.symbols + layer_name, + mod_name.lower(), + module_start, + pe_module.DIRECTORY_ENTRY_EXPORT.symbols, ) @staticmethod @@ -369,13 +505,17 @@ class PESymbols(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, config_path: str, mod_name: str, - module_info: Tuple[str, int, int], + module_info: collected_module_instance, ) -> Optional[PDBSymbolFinder]: """ - Attempts to locate symbols based on PDB analysis + Attempts to locate symbols based on PDB analysis through each layer where the mod_name module was found - mod_name: lower case name of the module to resolve symbols in - module_info: (layer_name, module_start, module_size) of the module to examine + Args: + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + + Returns: + Optional[PDBSymbolFinder]: If the export table can be resolved, then the ExportSymbolFinder """ mod_symbols = None @@ -386,15 +526,17 @@ class PESymbols(interfaces.plugins.PluginInterface): # a `ntoskrnl.exe` can have an internal PDB name of any of the ones in the following list # The code attempts to find all possible PDBs to ensure the best chance of recovery if mod_name == PESymbols.os_module_name: - pdb_names = ["ntkrnlmp.pdb", "ntkrnlpa.pdb", "ntkrpamp.pdb", "ntoskrnl.pdb"] + pdb_names = [fn + ".pdb" for fn in KERNEL_MODULE_NAMES] # for non-kernel files, replace the exe, sys, or dll extension with pdb else: + # in testing we found where some DLLs, such amsi.dll, have its PDB string as Amsi.dll + # in certain Windows versions mod_name = mod_name[:-3] + "pdb" first_upper = mod_name[0].upper() + mod_name[1:] pdb_names = [mod_name, first_upper] - # loop through each PDB name (will be just one for all but the kernel) + # loop through each PDB name (all the kernel names or the dll name as lower() + first char upper case) for pdb_name in pdb_names: try: mod_symbols = pdbutil.PDBUtility.symbol_table_from_pdb( @@ -433,11 +575,17 @@ class PESymbols(interfaces.plugins.PluginInterface): def _find_symbols_through_pdb( context: interfaces.context.ContextInterface, config_path: str, - module_instances: List[Tuple[str, int, int]], + module_instances: collected_modules_info, mod_name: str, ) -> Generator[PDBSymbolFinder, None, None]: """ - Attempts to resolve the symbols in `wanted_symbols` through PDB analysis + Attempts to resolve the symbols in `mod_name` through PDB analysis + + Args: + module_instances: the set of layers in which the module was found + mod_name: name of the module to resolve symbols in + Returns: + Generator[PDBSymbolFinder]: a PDBSymbolFinder instance for each layer in which the module was found """ for module_info in module_instances: mod_module = PESymbols._get_pdb_module( @@ -450,11 +598,17 @@ class PESymbols(interfaces.plugins.PluginInterface): def _find_symbols_through_exports( context: interfaces.context.ContextInterface, config_path: str, - module_instances: List[Tuple[str, int, int]], + module_instances: collected_modules_info, mod_name: str, ) -> Generator[ExportSymbolFinder, None, None]: """ - Attempts to resolve the symbols in `wanted_symbols` through export analysis + Attempts to resolve the symbols in `mod_name` through export analysis + + Args: + module_instances: the set of layers in which the module was found + mod_name: name of the module to resolve symbols in + Returns: + Generator[ExportSymbolFinder]: an ExportSymbolFinder instance for each layer in which the module was found """ pe_table_name = intermed.IntermediateSymbolTable.create( context, config_path, "windows", "pe", class_types=pe.class_types @@ -476,12 +630,21 @@ class PESymbols(interfaces.plugins.PluginInterface): ) -> Generator[Tuple[str, int], None, None]: """ Enumerates the symbols specified as wanted by the calling plugin + + removes entries from wanted_modules as they found to avoid PDB or export analysis after resolving all symbols + + Args: + wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. + mod_name: the name of module to resolve symbols in + + Returns: + Tuple[str, int]: the name and address of resolved symbols """ wanted_symbols = wanted_modules[mod_name] if ( - PESymbols.wanted_names not in wanted_symbols - and PESymbols.wanted_addresses not in wanted_symbols + wanted_names_identifier not in wanted_symbols + and wanted_addresses_identifier not in wanted_symbols ): vollog.warning( f"Invalid `wanted_symbols` sent to `find_symbols` for module {mod_name}. addresses and names keys both misssing." @@ -489,8 +652,8 @@ class PESymbols(interfaces.plugins.PluginInterface): return symbol_keys = [ - (PESymbols.wanted_names, "get_address_for_name"), - (PESymbols.wanted_addresses, "get_name_for_address"), + (wanted_names_identifier, "get_address_for_name"), + (wanted_addresses_identifier, "get_name_for_address"), ] for symbol_key, symbol_getter in symbol_keys: @@ -503,7 +666,7 @@ class PESymbols(interfaces.plugins.PluginInterface): ) if symbol_value: # yield out symbol name, symbol address - if symbol_key == PESymbols.wanted_names: + if symbol_key == wanted_names_identifier: yield wanted_value, symbol_value # type: ignore else: yield symbol_value, wanted_value # type: ignore @@ -521,13 +684,20 @@ class PESymbols(interfaces.plugins.PluginInterface): def _resolve_symbols_through_methods( context: interfaces.context.ContextInterface, config_path: str, - module_instances: List[Tuple[str, int, int]], + module_instances: collected_modules_info, wanted_modules: PESymbolFinder.cached_value_dict, mod_name: str, ) -> Generator[Tuple[str, int], None, None]: """ Attempts to resolve every wanted symbol in `mod_name` Every layer is enumerated for maximum chance of recovery + + Args: + module_instances: the set of layers in which the module was found + wanted_modules: The symbols to resolve tied to their module names + mod_name: name of the module to resolve symbols in + Returns: + Generator[Tuple[str, int]]: resolved symbol names and addresses """ symbol_resolving_methods = [ PESymbols._find_symbols_through_pdb, @@ -554,13 +724,19 @@ class PESymbols(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, config_path: str, wanted_modules: PESymbolFinder.cached_value_dict, - collected_modules: Dict[str, List[Tuple[str, int, int]]], + collected_modules: collected_modules_type, ) -> found_symbols_type: """ Loops through each method of symbol analysis until each wanted symbol is found Returns the resolved symbols as a dictionary that includes the name and runtime address + + Args: + wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. + collected_modules: return value from `get_kernel_modules` or `get_process_modules` + Returns: + found_symbols_type: The set of symbols resolved to their name and/or address """ - found_symbols: PESymbols.found_symbols_type = {} + found_symbols: found_symbols_type = {} for mod_name in wanted_modules: if mod_name not in collected_modules: @@ -594,11 +770,16 @@ class PESymbols(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, filter_modules: Optional[filter_modules_type], - ) -> Dict[str, List[Tuple[str, int, int]]]: + ) -> collected_modules_type: """ Walks the kernel module list and finds the session layer, base, and size of each wanted module + + Args: + filter_modules: The modules to filter the gathering to. If left as None, all kernel modules are gathered. + Returns: + collected_modules_type: The collection of modules found with at least one layer present """ - found_modules: Dict[str, List[Tuple[str, int, int]]] = {} + found_modules: collected_modules_type = {} if filter_modules: # create a tuple of module names for use with `endswith` @@ -645,16 +826,55 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_modules + @staticmethod + def get_vads_for_process_cache( + vads_cache: Dict[int, ranges_type], + owner_proc: interfaces.objects.ObjectInterface, + ) -> Optional[ranges_type]: + """ + Creates and utilizes a cache of a process' VADs for efficient lookups + + Returns the vad information of the VAD hosting the address, if found + + Args: + vads_cache: The existing cache of VADs + owner_proc: The process being inspected + Returns: + Optional[ranges_type]: The range holding the address, if found + """ + if owner_proc.vol.offset in vads_cache: + vads = vads_cache[owner_proc.vol.offset] + else: + vads = PESymbols.get_proc_vads_with_file_paths(owner_proc) + vads_cache[owner_proc.vol.offset] = vads + + # smear or terminated process + if len(vads) == 0: + return None + + return vads + @staticmethod def get_proc_vads_with_file_paths( proc: interfaces.objects.ObjectInterface, ) -> ranges_type: """ Returns a list of the process' vads that map a file - """ - vads = [] - for vad in proc.get_vad_root().traverse(): + Args: + proc: The process to gather the VADs for + + Returns: + ranges_type: The list of VADs for this process that map a file + """ + vads: ranges_type = [] + + try: + vad_root = proc.get_vad_root() + except exceptions.InvalidAddressException: + return vads + + for vad in vad_root.traverse(): filepath = vad.get_file_name() if not isinstance(filepath, str) or filepath.count("\\") == 0: continue @@ -676,6 +896,9 @@ class PESymbols(interfaces.plugins.PluginInterface): ]: """ Yields each set of vads for a process that have a file mapped, along with the process itself and its layer + + Args: + Generator[Tuple[interfaces.objects.ObjectInterface, str, ranges_type]]: Yields tuple of process objects, layers, and VADs mapping files """ procs = pslist.PsList.list_processes( context=context, @@ -689,7 +912,7 @@ class PESymbols(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: continue - vads = PESymbols.get_proc_vads_with_file_paths(proc) + vads = cls.get_proc_vads_with_file_paths(proc) yield proc, proc_layer_name, vads @@ -699,11 +922,16 @@ class PESymbols(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, filter_modules: Optional[filter_modules_type], - ) -> Dict[str, List[Tuple[str, int, int]]]: + ) -> collected_modules_type: """ Walks the process list and each process' VAD to determine the base address and size of wanted modules + + Args: + filter_modules: The modules to filter the gathering to. If left as None, all process modules are gathered. + Returns: + collected_modules_type: The collection of modules found with at least one layer present """ - proc_modules: Dict[str, List[Tuple[str, int, int]]] = {} + proc_modules: collected_modules_type = {} if filter_modules: # create a tuple of module names for use with `endswith` @@ -711,7 +939,7 @@ class PESymbols(interfaces.plugins.PluginInterface): else: filter_modules_check = None - for _, proc_layer_name, vads in PESymbols.get_all_vads_with_file_paths( + for _proc, proc_layer_name, vads in PESymbols.get_all_vads_with_file_paths( context, layer_name, symbol_table ): for vad_start, vad_size, filepath in vads: @@ -731,17 +959,17 @@ class PESymbols(interfaces.plugins.PluginInterface): def _generator(self) -> Generator[Tuple[int, Tuple[str, str, int]], None, None]: kernel = self.context.modules[self.config["kernel"]] - if self.config["symbol"]: + if self.config["symbols"]: filter_module = { self.config["module"].lower(): { - PESymbols.wanted_names: [self.config["symbol"]] + wanted_names_identifier: self.config["symbols"] } } - elif self.config["address"]: + elif self.config["addresses"]: filter_module = { self.config["module"].lower(): { - PESymbols.wanted_addresses: [self.config["address"]] + wanted_addresses_identifier: self.config["addresses"] } } diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index 0438bc9e3..68f4c4b80 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -1,3 +1,6 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + import logging from typing import Dict, Tuple, List, Generator @@ -18,7 +21,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): system_calls = { "ntdll.dll": { - pe_symbols.PESymbols.wanted_names: [ + pe_symbols.wanted_names_identifier: [ "NtCreateThread", "NtProtectVirtualMemory", "NtReadVirtualMemory", @@ -90,7 +93,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): def _gather_code_bytes( self, kernel: interfaces.context.ModuleInterface, - found_symbols: pe_symbols.PESymbols.found_symbols_type, + found_symbols: pe_symbols.found_symbols_type, ) -> _code_bytes_type: """ Enumerates the desired DLLs and function implementations in each process diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 974793a71..2c6ed4daf 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -9,7 +9,7 @@ from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist, pe_symbols +from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class VadInfo(interfaces.plugins.PluginInterface): _version = (2, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb - def __init__(self, *args, **kwargs): # type: ignore + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._protect_values = None From b40c20dd7fe0d1e348f94531c290a9aceee34dcb Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 10 Sep 2024 14:55:00 -0500 Subject: [PATCH 090/110] Revert back to union to avoid failed tests --- volatility3/framework/plugins/windows/pe_symbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index c9785a1c9..30e9b49d1 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -61,7 +61,7 @@ class PESymbolFinder: cached_int_dict = Dict[str, Optional[int]] cached_value = Union[int, str, None] - cached_value_dict = Dict[str, Dict[str, List[str]] | Dict[str, List[int]]] + cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] def __init__( self, From 037eb1ce036ae7dea48c427742ae889c9b2ec7a3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 11 Sep 2024 16:30:41 +1000 Subject: [PATCH 091/110] Linux Check_creds plugins pointer verification improvements --- volatility3/framework/plugins/linux/check_creds.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index ab6ee4935..45df966d2 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -16,6 +16,8 @@ class Check_creds(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) + @classmethod def get_requirements(cls): return [ @@ -46,7 +48,11 @@ class Check_creds(interfaces.plugins.PluginInterface): tasks = pslist.PsList.list_tasks(self.context, vmlinux.name) for task in tasks: - cred_addr = task.cred.dereference().vol.offset + task_cred_ptr = task.cred + if not (task_cred_ptr and task_cred_ptr.is_readable()): + continue + + cred_addr = task_cred_ptr.dereference().vol.offset if cred_addr not in creds: creds[cred_addr] = [] From c77c662b70c6751087bf947c400a045c81e7a8ec Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 11 Sep 2024 21:09:08 +1000 Subject: [PATCH 092/110] Linux pidhashtable plugin pointer verification improvements --- .../framework/plugins/linux/pidhashtable.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 3223aed4a..edafe97e0 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -20,7 +20,7 @@ class PIDHashTable(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -45,9 +45,7 @@ class PIDHashTable(plugins.PluginInterface): ] def _is_valid_task(self, task) -> bool: - vmlinux = self.context.modules[self.config["kernel"]] - vmlinux_layer = self.context.layers[vmlinux.layer_name] - return bool(task and task.pid > 0 and vmlinux_layer.is_valid(task.parent)) + return bool(task and task.pid > 0 and task.parent.is_readable()) def _get_pidtype_pid(self): vmlinux = self.context.modules[self.config["kernel"]] @@ -96,7 +94,7 @@ class PIDHashTable(plugins.PluginInterface): seen_upids.add(upid.vol.offset) pid_chain = upid.pid_chain - if not (pid_chain and vmlinux_layer.is_valid(pid_chain.vol.offset)): + if not (pid_chain.next and pid_chain.next.is_readable()): break upid = linux.LinuxUtilities.container_of( @@ -105,7 +103,6 @@ class PIDHashTable(plugins.PluginInterface): def _get_upids(self): vmlinux = self.context.modules[self.config["kernel"]] - vmlinux_layer = self.context.layers[vmlinux.layer_name] # 2.6.24 <= kernels < 4.15 pidhash = self._get_pidhash_array() @@ -115,7 +112,7 @@ class PIDHashTable(plugins.PluginInterface): # each entry in the hlist is a upid which is wrapped in a pid ent = hlist.first - while ent and vmlinux_layer.is_valid(ent.vol.offset): + while ent and ent.is_readable(): # upid->pid_chain exists 2.6.24 <= kernel < 4.15 upid = linux.LinuxUtilities.container_of( ent.vol.offset, "upid", "pid_chain", vmlinux @@ -143,7 +140,7 @@ class PIDHashTable(plugins.PluginInterface): continue pid_tasks_0 = pid.tasks[pidtype_pid].first - if not pid_tasks_0: + if not (pid_tasks_0 and pid_tasks_0.is_readable()): continue task = vmlinux.object( @@ -160,7 +157,7 @@ class PIDHashTable(plugins.PluginInterface): pidtype_pid = self._get_pidtype_pid() pid_tasks_0 = pid.tasks[pidtype_pid].first - if not pid_tasks_0: + if not (pid_tasks_0 and pid_tasks_0.is_readable()): return None task_struct_type = vmlinux.get_type("task_struct") From 9e8471799adcf090e20ea98a20309076193e9009 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 10:44:43 +1000 Subject: [PATCH 093/110] Improving code and adding the credential virtual addresses to the output. --- .../framework/plugins/linux/check_creds.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 45df966d2..3e292ae33 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -2,21 +2,18 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import logging - from volatility3.framework import interfaces, renderers +from volatility3.framework.renderers import format_hints from volatility3.framework.configuration import requirements from volatility3.plugins.linux import pslist -vollog = logging.getLogger(__name__) - class Check_creds(interfaces.plugins.PluginInterface): """Checks if any processes are sharing credential structures""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls): @@ -54,18 +51,22 @@ class Check_creds(interfaces.plugins.PluginInterface): cred_addr = task_cred_ptr.dereference().vol.offset - if cred_addr not in creds: - creds[cred_addr] = [] - + creds.setdefault(cred_addr, []) creds[cred_addr].append(task.pid) - for _, pids in creds.items(): + for cred_addr, pids in creds.items(): if len(pids) > 1: - pid_str = "" - for pid in pids: - pid_str = pid_str + f"{pid:d}, " - pid_str = pid_str[:-2] - yield (0, [str(pid_str)]) + pid_str = ", ".join([str(pid) for pid in pids]) + + fields = [ + format_hints.Hex(cred_addr), + pid_str, + ] + yield (0, fields) def run(self): - return renderers.TreeGrid([("PIDs", str)], self._generator()) + headers = [ + ("CredVAddr", format_hints.Hex), + ("PIDs", str), + ] + return renderers.TreeGrid(headers, self._generator()) From 57de357ffdfe87dbcad8c219228a4a0d0e17c173 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 16:05:40 +1000 Subject: [PATCH 094/110] Timeliner plugin: Fix issue with filtering TimeLinerInterface plugins and using the filter argument --- volatility3/framework/plugins/timeliner.py | 23 +++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index f657a2918..abe802e8f 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -45,6 +45,7 @@ class Timeliner(interfaces.plugins.PluginInterface): orders the results by time.""" _required_framework_version = (2, 0, 0) + _version = (1, 1, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -245,6 +246,17 @@ class Timeliner(interfaces.plugins.PluginInterface): filter_list = self.config["plugin-filter"] # Identify plugins that we can run which output datetimes for plugin_class in self.usable_plugins: + if not issubclass(plugin_class, TimeLinerInterface): + continue + + if filter_list and not any( + [ + filter in plugin_class.__module__ + "." + plugin_class.__name__ + for filter in filter_list + ] + ): + continue + try: automagics = automagic.choose_automagic(self.automagics, plugin_class) @@ -276,15 +288,8 @@ class Timeliner(interfaces.plugins.PluginInterface): config_value, ) - if isinstance(plugin, TimeLinerInterface): - if not len(filter_list) or any( - [ - filter - in plugin.__module__ + "." + plugin.__class__.__name__ - for filter in filter_list - ] - ): - plugins_to_run.append(plugin) + plugins_to_run.append(plugin) + except exceptions.UnsatisfiedException as excp: # Remove the failed plugin from the list and continue vollog.debug( From be05ace29b134156fe5f7584921887426fc2f41f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 16:06:40 +1000 Subject: [PATCH 095/110] Timeliner plugin: Add exception information --- volatility3/framework/plugins/timeliner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index abe802e8f..56fe465e4 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -199,9 +199,10 @@ class Timeliner(interfaces.plugins.PluginInterface): ), ) ) - except Exception: + except Exception as e: vollog.log( - logging.INFO, f"Exception occurred running plugin: {plugin_name}" + logging.INFO, + f"Exception occurred running plugin: {plugin_name}: {e}", ) vollog.log(logging.DEBUG, traceback.format_exc()) From a7dcd6d9e8adfe3124aa13db1b1536e6799d822c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 16:19:13 +1000 Subject: [PATCH 096/110] Minor: Add comment on TimeLinerInterface subclass filter --- volatility3/framework/plugins/timeliner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 56fe465e4..d1cb9f460 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -248,6 +248,7 @@ class Timeliner(interfaces.plugins.PluginInterface): # Identify plugins that we can run which output datetimes for plugin_class in self.usable_plugins: if not issubclass(plugin_class, TimeLinerInterface): + # get_usable_plugins() should filter this, but adding a safeguard just in case continue if filter_list and not any( From 48ae43d64edd457b65eb40174fb00b54202aabda Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 17:22:52 +1000 Subject: [PATCH 097/110] Bumping the major version since the output changed --- volatility3/framework/plugins/linux/check_creds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 3e292ae33..b7f73c3eb 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -13,7 +13,7 @@ class Check_creds(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): From 997abeda6d9014a028f6c2b7a9e11352320c142f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 17:27:02 +1000 Subject: [PATCH 098/110] Linux lsof: Add namespace dentry name --- .../framework/symbols/linux/__init__.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 91abf7db4..57b45667e 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -169,13 +169,30 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): Returns: str: Sock pipe pathname relative to the task's root directory. """ + # FIXME: This function must be moved to the 'dentry' object extension + # Also, the scope of this function went beyond the sock pipe path, so we need to rename this. + # Once https://github.com/volatilityfoundation/volatility3/pull/1263 is merged, replace the + # dentry inode getters + + if not (filp and filp.is_readable()): + return f" {filp:x}" + dentry = filp.get_dentry() + if not (dentry and dentry.is_readable()): + return f" {dentry:x}" kernel_module = cls.get_module_from_volobj_type(context, dentry) sym_addr = dentry.d_op.d_dname + if not (sym_addr and sym_addr.is_readable()): + return f" {sym_addr:x}" + symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) + inode = dentry.d_inode + if not (inode and inode.is_readable() and inode.is_valid()): + return f" {inode:x}" + if len(symbs) == 1: sym = symbs[0].split(constants.BANG)[1] @@ -191,13 +208,36 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): elif sym == "simple_dname": pre_name = cls._get_path_file(task, filp) - else: - pre_name = f"" + elif sym == "ns_dname": + # From Kernels 3.19 - ret = f"{pre_name}:[{dentry.d_inode.i_ino:d}]" + # In Kernels >= 6.9, see Linux kernel commit 1fa08aece42512be072351f482096d5796edf7ca + # ns_common->stashed change from 'atomic64_t' to 'dentry*' + try: + ns_common_type = kernel_module.get_type("ns_common") + stashed_template = ns_common_type.child_template("stashed") + stashed_type_full_name = stashed_template.vol.type_name + stashed_type_name = stashed_type_full_name.split(constants.BANG)[-1] + if stashed_type_name == "atomic64_t": + # 3.19 <= Kernels < 6.9 + ns_ops = dentry.d_fsdata.dereference().cast( + "proc_ns_operations" + ) + else: + # Kernels >= 6.9 + ns_common = inode.i_private.dereference().cast("ns_common") + ns_ops = ns_common.ops + + pre_name = utility.pointer_to_string(ns_ops.name, 255) + except IndexError: + ret = "" + else: + pre_name = f" {sym}" + + ret = f"{pre_name}:[{inode.i_ino:d}]" else: - ret = f" {sym_addr:x}" + ret = f" {sym_addr:x}" return ret From cd2af74e6d0c554e81d1e67a8020195cfca59983 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 17:58:21 +1000 Subject: [PATCH 099/110] Improve pointers address verification and return message chain --- .../framework/symbols/linux/__init__.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 57b45667e..2410d2627 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -217,29 +217,32 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ns_common_type = kernel_module.get_type("ns_common") stashed_template = ns_common_type.child_template("stashed") stashed_type_full_name = stashed_template.vol.type_name - stashed_type_name = stashed_type_full_name.split(constants.BANG)[-1] + stashed_type_name = stashed_type_full_name.split(constants.BANG)[1] if stashed_type_name == "atomic64_t": # 3.19 <= Kernels < 6.9 - ns_ops = dentry.d_fsdata.dereference().cast( - "proc_ns_operations" - ) + fsdata_ptr = dentry.d_fsdata + if not (fsdata_ptr and fsdata_ptr.is_readable()): + raise IndexError + + ns_ops = fsdata_ptr.dereference().cast("proc_ns_operations") else: # Kernels >= 6.9 - ns_common = inode.i_private.dereference().cast("ns_common") + private_ptr = inode.i_private + if not (private_ptr and private_ptr.is_readable()): + raise IndexError + + ns_common = private_ptr.dereference().cast("ns_common") ns_ops = ns_common.ops pre_name = utility.pointer_to_string(ns_ops.name, 255) except IndexError: - ret = "" + pre_name = "" else: pre_name = f" {sym}" - - ret = f"{pre_name}:[{inode.i_ino:d}]" - else: - ret = f" {sym_addr:x}" + pre_name = f" {sym_addr:x}" - return ret + return f"{pre_name}:[{inode.i_ino:d}]" @classmethod def path_for_file(cls, context, task, filp) -> str: From 67ee382c3229f10d4e29958b6a5bf257e29ed8f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 13 Sep 2024 18:53:33 +0200 Subject: [PATCH 100/110] use default req value in config_value call --- volatility3/framework/interfaces/configuration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 3bb3cb019..da0a4556c 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -494,8 +494,7 @@ class SimpleTypeRequirement(RequirementInterface): """Validates the instance requirement based upon its `instance_type`.""" config_path = path_join(config_path, self.name) - - value = self.config_value(context, config_path, None) + value = self.config_value(context, config_path, self.default) if not isinstance(value, self.instance_type): vollog.log( constants.LOGLEVEL_V, @@ -536,7 +535,7 @@ class ClassRequirement(RequirementInterface): """Checks to see if a class can be recovered.""" config_path = path_join(config_path, self.name) - value = self.config_value(context, config_path, None) + value = self.config_value(context, config_path, self.default) self._cls = None if value is not None and isinstance(value, str): if "." in value: From ba0c975e73207ee6555bd80067460ddbea6426a2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 13 Sep 2024 18:50:47 -0500 Subject: [PATCH 101/110] Address all feedback --- .../framework/plugins/windows/pe_symbols.py | 126 +++++++++--------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 30e9b49d1..85bfb572e 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -1,11 +1,12 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +import copy import io import logging import ntpath -from typing import Dict, Tuple, Optional, List, Generator, Union +from typing import Dict, Tuple, Optional, List, Generator, Union, Callable import pefile @@ -29,11 +30,13 @@ wanted_addresses_identifier = "addresses" # how wanted modules/symbols are specified, such as: # {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} # {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} -filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] +filter_module_info = Union[Dict[str, List[str]], Dict[str, List[int]]] +filter_modules_type = Dict[str, filter_module_info] # holds resolved symbols # {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} -found_symbols_type = Dict[str, List[Tuple[str, int]]] +found_symbols_module = List[Tuple[str, int]] +found_symbols_type = Dict[str, found_symbols_module] # used to hold informatin about a range (VAD or kernel module) # (start address, size, file path) @@ -61,7 +64,8 @@ class PESymbolFinder: cached_int_dict = Dict[str, Optional[int]] cached_value = Union[int, str, None] - cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + cached_module_lists = Union[Dict[str, List[str]], Dict[str, List[int]]] + cached_value_dict = Dict[str, cached_module_lists] def __init__( self, @@ -402,11 +406,11 @@ class PESymbols(interfaces.plugins.PluginInterface): context, layer_name, symbol_table_name, symbols ) - found_symbols = PESymbols.find_symbols( + found_symbols, missing_symbols = PESymbols.find_symbols( context, config_path, symbols, collected_modules ) - for mod_name, unresolved_symbols in symbols.items(): + for mod_name, unresolved_symbols in missing_symbols.items(): for symbol in unresolved_symbols: vollog.debug(f"Unable to resolve symbol {symbol} in module {mod_name}") @@ -449,7 +453,7 @@ class PESymbols(interfaces.plugins.PluginInterface): filename: {wanted_addresses_identifier: [address]} } - found_symbols = PESymbols.find_symbols( + found_symbols, _missing_msybols = PESymbols.find_symbols( context, config_path, filter_module, collected_modules ) @@ -624,61 +628,46 @@ class PESymbols(interfaces.plugins.PluginInterface): @staticmethod def _get_symbol_value( - wanted_modules: PESymbolFinder.cached_value_dict, - mod_name: str, + wanted_symbols: filter_module_info, symbol_resolver: PESymbolFinder, - ) -> Generator[Tuple[str, int], None, None]: + ) -> Generator[Tuple[str, int, str, int], None, None]: """ Enumerates the symbols specified as wanted by the calling plugin - removes entries from wanted_modules as they found to avoid PDB or export analysis after resolving all symbols - Args: - wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. - mod_name: the name of module to resolve symbols in + wanted_symbols: the set of symbols for a particular module + symbol_resolver: method in a layer to resolve the symbols Returns: - Tuple[str, int]: the name and address of resolved symbols + Tuple[str, int, str, int]: the index and value of the found symbol in the wanted list, and the name and address of resolved symbol """ - wanted_symbols = wanted_modules[mod_name] - if ( wanted_names_identifier not in wanted_symbols and wanted_addresses_identifier not in wanted_symbols ): vollog.warning( - f"Invalid `wanted_symbols` sent to `find_symbols` for module {mod_name}. addresses and names keys both misssing." + f"Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." ) return - symbol_keys = [ - (wanted_names_identifier, "get_address_for_name"), - (wanted_addresses_identifier, "get_name_for_address"), + symbol_keys: List[Tuple[str, Callable]] = [ + (wanted_names_identifier, symbol_resolver.get_address_for_name), + (wanted_addresses_identifier, symbol_resolver.get_name_for_address), ] for symbol_key, symbol_getter in symbol_keys: # address or name if symbol_key in wanted_symbols: # walk each wanted address or name - for wanted_value in wanted_symbols[symbol_key]: - symbol_value = symbol_resolver.__getattribute__(symbol_getter)( - wanted_value - ) + for value_index, wanted_value in enumerate(wanted_symbols[symbol_key]): + symbol_value = symbol_getter(wanted_value) + if symbol_value: - # yield out symbol name, symbol address + # yield out deleteion key, deletion index, symbol name, symbol address if symbol_key == wanted_names_identifier: - yield wanted_value, symbol_value # type: ignore + yield symbol_key, value_index, wanted_value, symbol_value # type: ignore else: - yield symbol_value, wanted_value # type: ignore - - index = wanted_modules[mod_name][symbol_key].index(wanted_value) # type: ignore - - del wanted_modules[mod_name][symbol_key][index] - - # if all names or addresses from a module are found, delete the key - if not wanted_modules[mod_name][symbol_key]: - del wanted_modules[mod_name][symbol_key] - break + yield symbol_key, value_index, symbol_value, wanted_value # type: ignore @staticmethod def _resolve_symbols_through_methods( @@ -687,7 +676,7 @@ class PESymbols(interfaces.plugins.PluginInterface): module_instances: collected_modules_info, wanted_modules: PESymbolFinder.cached_value_dict, mod_name: str, - ) -> Generator[Tuple[str, int], None, None]: + ) -> Tuple[found_symbols_module, PESymbolFinder.cached_module_lists]: """ Attempts to resolve every wanted symbol in `mod_name` Every layer is enumerated for maximum chance of recovery @@ -697,35 +686,53 @@ class PESymbols(interfaces.plugins.PluginInterface): wanted_modules: The symbols to resolve tied to their module names mod_name: name of the module to resolve symbols in Returns: - Generator[Tuple[str, int]]: resolved symbol names and addresses + Tuple[found_symbols_module, PESymbolFinder.cached_module_lists]: The set of found symbols and the ones that could not be resolved """ symbol_resolving_methods = [ PESymbols._find_symbols_through_pdb, PESymbols._find_symbols_through_exports, ] + found: found_symbols_module = [] + + # the symbols wanted from this module by the caller + wanted = wanted_modules[mod_name] + + # make a copy to remove from inside this function for returning to the caller + remaining = copy.deepcopy(wanted) + for method in symbol_resolving_methods: + # every layer where this module was found through the given method for symbol_resolver in method( context, config_path, module_instances, mod_name ): vollog.debug(f"Have resolver for method {method}") - yield from PESymbols._get_symbol_value( - wanted_modules, mod_name, symbol_resolver - ) + for ( + symbol_key, + value_index, + symbol_name, + symbol_address, + ) in PESymbols._get_symbol_value(remaining, symbol_resolver): + found.append((symbol_name, symbol_address)) + del remaining[symbol_key][value_index] - if not wanted_modules[mod_name]: + # everything was resolved, stop this resolver + if not remaining: break - if not wanted_modules[mod_name]: + # stop all resolving + if not remaining: break + return found, remaining + @staticmethod def find_symbols( context: interfaces.context.ContextInterface, config_path: str, wanted_modules: PESymbolFinder.cached_value_dict, collected_modules: collected_modules_type, - ) -> found_symbols_type: + ) -> Tuple[found_symbols_type, PESymbolFinder.cached_value_dict]: """ Loops through each method of symbol analysis until each wanted symbol is found Returns the resolved symbols as a dictionary that includes the name and runtime address @@ -734,9 +741,10 @@ class PESymbols(interfaces.plugins.PluginInterface): wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. collected_modules: return value from `get_kernel_modules` or `get_process_modules` Returns: - found_symbols_type: The set of symbols resolved to their name and/or address + Tuple[found_symbols_type, PESymbolFinder.cached_value_dict]: The set of found symbols but the ones that could not be resolved """ found_symbols: found_symbols_type = {} + missing_symbols: PESymbolFinder.cached_value_dict = {} for mod_name in wanted_modules: if mod_name not in collected_modules: @@ -745,24 +753,20 @@ class PESymbols(interfaces.plugins.PluginInterface): module_instances = collected_modules[mod_name] # try to resolve the symbols for `mod_name` through each method (PDB and export table currently) - for symbol_name, address in PESymbols._resolve_symbols_through_methods( + ( + found_in_module, + missing_in_module, + ) = PESymbols._resolve_symbols_through_methods( context, config_path, module_instances, wanted_modules, mod_name - ): - if mod_name not in found_symbols: - found_symbols[mod_name] = [] + ) - found_symbols[mod_name].append((symbol_name, address)) + if found_in_module: + found_symbols[mod_name] = found_in_module - # stop processing the layers (processes) if we found all the symbols for this module - if not wanted_modules[mod_name]: - break + if missing_in_module: + missing_symbols[mod_name] = missing_in_module - # stop processing this module if/when all symbols are found - if not wanted_modules[mod_name]: - del wanted_modules[mod_name] - break - - return found_symbols + return found_symbols, missing_symbols @staticmethod def get_kernel_modules( @@ -986,7 +990,7 @@ class PESymbols(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, filter_module ) - found_symbols = PESymbols.find_symbols( + found_symbols, _missing_symbols = PESymbols.find_symbols( self.context, self.config_path, filter_module, collected_modules ) From 21d21cf4b8ec3aafa5e98c563b207b21c41d5c3f Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 16:26:38 -0500 Subject: [PATCH 102/110] Break properly in all paths. Help callers to ensure always lower case module name matching. --- volatility3/framework/plugins/windows/pe_symbols.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 85bfb572e..d0ddcac57 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -380,7 +380,7 @@ class PESymbols(interfaces.plugins.PluginInterface): Returns: str: the bsae file name of the full path """ - return ntpath.basename(filepath) + return ntpath.basename(filepath).lower() @staticmethod def addresses_for_process_symbols( @@ -717,11 +717,12 @@ class PESymbols(interfaces.plugins.PluginInterface): del remaining[symbol_key][value_index] # everything was resolved, stop this resolver - if not remaining: + if not remaining[symbol_key]: break # stop all resolving - if not remaining: + if not remaining[symbol_key]: + del remaining[symbol_key] break return found, remaining From 291bc878ad771388f3514b2c66d31b7325b2bc01 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 16:31:33 -0500 Subject: [PATCH 103/110] Break in a cleaner flow --- volatility3/framework/plugins/windows/pe_symbols.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index d0ddcac57..44254513f 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -701,6 +701,8 @@ class PESymbols(interfaces.plugins.PluginInterface): # make a copy to remove from inside this function for returning to the caller remaining = copy.deepcopy(wanted) + done_processing = False + for method in symbol_resolving_methods: # every layer where this module was found through the given method for symbol_resolver in method( @@ -717,12 +719,14 @@ class PESymbols(interfaces.plugins.PluginInterface): del remaining[symbol_key][value_index] # everything was resolved, stop this resolver + # remove this key from the remaining symbols to resolve if not remaining[symbol_key]: + del remaining[symbol_key] + done_processing = True break # stop all resolving - if not remaining[symbol_key]: - del remaining[symbol_key] + if done_processing: break return found, remaining From 322f79fb5040e3e7ebdfa9de8c82a639729a39f2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 17:18:26 -0500 Subject: [PATCH 104/110] Bail as early as possible --- .../framework/plugins/windows/pe_symbols.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 44254513f..0bf03e7d6 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -437,7 +437,6 @@ class PESymbols(interfaces.plugins.PluginInterface): Returns: Tuple[str|renderers.NotApplicableValue|renderers.NotAvailableValue, str|renderers.NotApplicableValue|renderers.NotAvailableValue] """ - if not address: return renderers.NotApplicableValue(), renderers.NotApplicableValue() @@ -458,7 +457,7 @@ class PESymbols(interfaces.plugins.PluginInterface): ) if not found_symbols or filename not in found_symbols: - return renderers.NotAvailableValue(), renderers.NotAvailableValue() + return filepath, renderers.NotAvailableValue() return filepath, found_symbols[filename][0][0] @@ -718,11 +717,14 @@ class PESymbols(interfaces.plugins.PluginInterface): found.append((symbol_name, symbol_address)) del remaining[symbol_key][value_index] - # everything was resolved, stop this resolver - # remove this key from the remaining symbols to resolve - if not remaining[symbol_key]: - del remaining[symbol_key] - done_processing = True + # everything was resolved, stop this resolver + # remove this key from the remaining symbols to resolve + if not remaining[symbol_key]: + del remaining[symbol_key] + done_processing = True + break + + if done_processing: break # stop all resolving @@ -885,6 +887,7 @@ class PESymbols(interfaces.plugins.PluginInterface): for vad in vad_root.traverse(): filepath = vad.get_file_name() + if not isinstance(filepath, str) or filepath.count("\\") == 0: continue From 79b8ff7d05b56316d90be682f56db22401f2c265 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 15 Sep 2024 18:29:05 -0500 Subject: [PATCH 105/110] Address final feedback --- .../plugins/windows/debugregisters.py | 23 ++++++++----------- .../framework/plugins/windows/pe_symbols.py | 10 ++++---- .../plugins/windows/unhooked_system_calls.py | 8 +++++++ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 65b2e625b..945ba1df0 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -148,12 +148,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): file3, sym3 = path_and_symbol(vads, dr3) # if none map to an actual file VAD then bail - if not ( - isinstance(file0, str) - or isinstance(file1, str) - or isinstance(file2, str) - or isinstance(file3, str) - ): + if not (file0 or file1 or file2 or file3): continue process_name = owner_proc.ImageFileName.cast( @@ -173,17 +168,17 @@ class DebugRegisters(interfaces.plugins.PluginInterface): thread.Tcb.State, dr7, format_hints.Hex(dr0), - file0, - sym0, + file0 or renderers.NotApplicableValue(), + sym0 or renderers.NotApplicableValue(), format_hints.Hex(dr1), - file1, - sym1, + file1 or renderers.NotApplicableValue(), + sym1 or renderers.NotApplicableValue(), format_hints.Hex(dr2), - file2, - sym2, + file2 or renderers.NotApplicableValue(), + sym2 or renderers.NotApplicableValue(), format_hints.Hex(dr3), - file3, - sym3, + file3 or renderers.NotApplicableValue(), + sym3 or renderers.NotApplicableValue(), ), ) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 0bf03e7d6..955098d6b 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -423,7 +423,7 @@ class PESymbols(interfaces.plugins.PluginInterface): collected_modules: collected_modules_type, ranges: ranges_type, address: int, - ) -> Tuple[str, str]: + ) -> Tuple[Optional[str], Optional[str]]: """ Method for plugins to determine the file path and symbol name for a given address @@ -438,12 +438,12 @@ class PESymbols(interfaces.plugins.PluginInterface): Tuple[str|renderers.NotApplicableValue|renderers.NotAvailableValue, str|renderers.NotApplicableValue|renderers.NotAvailableValue] """ if not address: - return renderers.NotApplicableValue(), renderers.NotApplicableValue() + return None, None filepath = PESymbols.filepath_for_address(ranges, address) if not filepath: - return renderers.NotAvailableValue(), renderers.NotAvailableValue() + return None, None filename = PESymbols.filename_for_path(filepath).lower() @@ -452,12 +452,12 @@ class PESymbols(interfaces.plugins.PluginInterface): filename: {wanted_addresses_identifier: [address]} } - found_symbols, _missing_msybols = PESymbols.find_symbols( + found_symbols, _missing_symbols = PESymbols.find_symbols( context, config_path, filter_module, collected_modules ) if not found_symbols or filename not in found_symbols: - return filepath, renderers.NotAvailableValue() + return filepath, None return filepath, found_symbols[filename][0][0] diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index 68f4c4b80..c3d98254d 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -71,6 +71,13 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): } } + # This data structure is used to track unique implementations of functions across processes + # The outer dictionary holds the module name (e.g., ntdll.dll) + # The next dictionary holds the function names (NtTerminateProcess, NtSetValueKey, etc.) inside a module + # The innermost dictionary holds the unique implementation (bytes) of a function across processes + # Each implementation is tracked along with the process(es) that host it + # For systems without malware, all functions should have the same implementation + # When API hooking/module unhooking is done, the victim (infected) processes will have unique implementations _code_bytes_type = Dict[str, Dict[str, Dict[bytes, List[Tuple[int, str]]]]] @classmethod @@ -127,6 +134,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: continue + # see the definition of _code_bytes_type for details of this data structure if dll_name not in code_bytes: code_bytes[dll_name] = {} From b788733683256e2cfd436750f616054f707252e9 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 15 Sep 2024 19:48:55 -0500 Subject: [PATCH 106/110] More comments on unhooked system calls --- .../plugins/windows/debugregisters.py | 4 ++++ .../plugins/windows/unhooked_system_calls.py | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 945ba1df0..57dd1822c 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -1,6 +1,10 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# Full details on the techniques used in these plugins to detect EDR-evading malware +# can be found in our 20 page whitepaper submitted to DEFCON along with the presentation +# https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + import logging from typing import Tuple, Optional, Generator, List, Dict diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index c3d98254d..1a1e59940 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -1,6 +1,10 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# Full details on the techniques used in these plugins to detect EDR-evading malware +# can be found in our 20 page whitepaper submitted to DEFCON along with the presentation +# https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + import logging from typing import Dict, Tuple, List, Generator @@ -162,20 +166,30 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): # code_bytes[dll_name][func_name][func_bytes] code_bytes = self._gather_code_bytes(kernel, found_symbols) + # walk the functions that were evaluated for functions in code_bytes.values(): + # cbb is the distinct groups of bytes (instructions) + # for this function across processes for func_name, cbb in functions.items(): + # the dict key here is the raw instructions, which is not helpful to look at + # the values are the list of tuples for the (proc_id, proc_name) pairs for this set of bytes (instructions) cb = list(cbb.values()) - # same implementation in all + # if all processes map to the same implementation, then no malware is present if len(cb) == 1: yield 0, (func_name, "", len(cb[0])) else: - # find the processes that are hooked for reporting + # if there are differing implementations then it means + # that malware has overwritten system call(s) in infected processes + # max_idx and small_idx find which implementation of a system call has the least processes + # as all observed malware and open source projects only infected a few targets, leaving the + # rest with the original EDR hooks in place max_idx = 0 if len(cb[0]) > len(cb[1]) else 1 small_idx = (~max_idx) & 1 ps = [] + # gather processes on small_idx since these are the malware infected ones for pid, pname in cb[small_idx]: ps.append("{:d}:{}".format(pid, pname)) From 10ac21da2cbca02d47dc9aa938c2fe0d560af23d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 16 Sep 2024 12:16:43 +0100 Subject: [PATCH 107/110] Windows: Remove the unnecessary requirement on verinfo Fixes #1267 --- volatility3/framework/plugins/windows/verinfo.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 5b3c52bf6..57b8dcd3f 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -48,9 +48,6 @@ class VerInfo(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="modules", plugin=modules.Modules, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="dlllist", component=dlllist.DllList, version=(2, 0, 0) - ), requirements.BooleanRequirement( name="extensive", description="Search physical layer for version information", From f77003b670be71e012d0caa52f936a5adba8f162 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Sep 2024 12:50:58 +1000 Subject: [PATCH 108/110] Fix changes introduced to volatility3.framework.constants in PRs #838 and #1247 --- volatility3/framework/constants/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 8743a64b0..27fae4ba1 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -134,4 +134,5 @@ def __getattr__(name): ]: warnings.warn(f"{name} is deprecated", FutureWarning) return globals()[f"{deprecated_tag}{name}"] - return None + + return getattr(__import__(__name__), name) From 09fa859a92878b5d035e0dd5c44c0521661630fc Mon Sep 17 00:00:00 2001 From: eve Date: Wed, 25 Sep 2024 09:02:51 +0100 Subject: [PATCH 109/110] Windows: change warnings around large memory maps to debug level as per issue #1256 --- volatility3/framework/plugins/windows/vadyarascan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 7bc3377c3..efcc70d07 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -18,7 +18,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -68,7 +68,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): layer = self.context.layers[layer_name] for start, size in self.get_vad_maps(task): if size > sanity_check: - vollog.warn( + vollog.debug( f"VAD at 0x{start:x} over sanity-check size, not scanning" ) continue From 7f37135739c9ff951e73680f7cdf4333b47ee231 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 25 Sep 2024 13:13:33 -0500 Subject: [PATCH 110/110] Linux: Update sockstat to render process names Currently, process names are not displayed for sockets in the sockstat plugin, making analysis more painful than it needs to be. This updates the `list_sockets` classmethod and the `generator` method to return the process name in addition to the PID. Because this is changing the public interface, this commit includes a major version bump for `linux.sockstat.Sockstat`. --- volatility3/framework/plugins/linux/sockstat.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index b0503b105..d3efc78dd 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -22,7 +22,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) def __init__(self, vmlinux, task): self._vmlinux = vmlinux @@ -507,7 +507,7 @@ class Sockstat(plugins.PluginInterface): dfop_addr = vmlinux.object_from_symbol("sockfs_dentry_operations").vol.offset fd_generator = lsof.Lsof.list_fds(context, vmlinux.name, filter_func) - for _pid, _task_comm, task, fd_fields in fd_generator: + for _pid, task_comm, task, fd_fields in fd_generator: fd_num, filp, _full_path = fd_fields if filp.f_op not in (sfop_addr, dfop_addr): @@ -548,7 +548,7 @@ class Sockstat(plugins.PluginInterface): except AttributeError: netns_id = NotAvailableValue() - yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields + yield task_comm, task, netns_id, fd_num, family, sock_type, protocol, sock_fields def _format_fields(self, sock_stat, protocol): """Prepare the socket fields to be rendered @@ -595,6 +595,7 @@ class Sockstat(plugins.PluginInterface): ) for ( + task_comm, task, netns_id, fd_num, @@ -617,6 +618,7 @@ class Sockstat(plugins.PluginInterface): fields = ( netns_id, + task_comm, task.pid, fd_num, format_hints.Hex(sock.vol.offset), @@ -636,6 +638,7 @@ class Sockstat(plugins.PluginInterface): tree_grid_args = [ ("NetNS", int), + ("Process Name", str), ("Pid", int), ("FD", int), ("Sock Offset", format_hints.Hex),