diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index e29ab6f29..d755d9402 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,4 +1,4 @@ -name: Black python linter +name: Black python formatter on: [push, pull_request] diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml new file mode 100644 index 000000000..77e3aa864 --- /dev/null +++ b/.github/workflows/ruff.yaml @@ -0,0 +1,15 @@ +--- +name: Ruff + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/ruff-action@v1 + with: + args: check + src: "." diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dfc42499d..ce2722457 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -42,8 +42,13 @@ jobs: - name: Testing... run: | - pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_windows -v - pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_linux -v + # VolShell + pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_windows_volshell -v + pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_linux_volshell -v + + # Volatility + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k "test_windows and not test_windows_volshell" -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k "test_linux and not test_linux_volshell" -v - name: Clean up post-test run: | diff --git a/development/banner_server.py b/development/banner_server.py index 3aea41c82..b62477a26 100644 --- a/development/banner_server.py +++ b/development/banner_server.py @@ -28,10 +28,10 @@ class BannerCacheGenerator: def run(self): context = contexts.Context() - json_output = {'version': 1} + json_output = {"version": 1} path = self._path - filename = '*' + filename = "*" for banner_cache in [linux.LinuxBannerCache, mac.MacBannerCache]: sub_path = banner_cache.os @@ -39,37 +39,54 @@ class BannerCacheGenerator: for extension in constants.ISF_EXTENSIONS: # Hopefully these will not be large lists, otherwise this might be slow try: - for found in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + extension): + for found in ( + pathlib.Path(path) + .joinpath(sub_path) + .resolve() + .rglob(filename + extension) + ): potentials.append(found.as_uri()) except FileNotFoundError: # If there's no linux symbols, don't cry about it pass - new_banners = banner_cache.read_new_banners(context, 'BannerServer', potentials, banner_cache.symbol_name, - banner_cache.os, progress_callback = PrintedProgress()) + new_banners = banner_cache.read_new_banners( + context, + "BannerServer", + potentials, + banner_cache.symbol_name, + banner_cache.os, + progress_callback=PrintedProgress(), + ) result_banners = {} for new_banner in new_banners: # Only accept file schemes - value = [self.convert_url(url) for url in new_banners[new_banner] if - urllib.parse.urlparse(url).scheme == 'file'] + value = [ + self.convert_url(url) + for url in new_banners[new_banner] + if urllib.parse.urlparse(url).scheme == "file" + ] if value and new_banner: # Convert files into URLs - result_banners[str(base64.b64encode(new_banner), 'latin-1')] = value + result_banners[str(base64.b64encode(new_banner), "latin-1")] = value json_output[banner_cache.os] = result_banners - output_path = os.path.join(self._path, 'banners.json') - with open(output_path, 'w') as fp: + output_path = os.path.join(self._path, "banners.json") + with open(output_path, "w") as fp: vollog.warning(f"Banners file written to {output_path}") json.dump(json_output, fp) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--path', default = os.path.dirname(__file__)) - parser.add_argument('--urlprefix', help = 'Web prefix that will eventually serve the ISF files', - default = 'http://localhost/symbols') + parser.add_argument("--path", default=os.path.dirname(__file__)) + parser.add_argument( + "--urlprefix", + help="Web prefix that will eventually serve the ISF files", + default="http://localhost/symbols", + ) args = parser.parse_args() diff --git a/development/compare-vol.py b/development/compare-vol.py index d0d834038..a01d8e93c 100644 --- a/development/compare-vol.py +++ b/development/compare-vol.py @@ -15,17 +15,17 @@ class VolatilityImage: filepath: str = "" vol2_profile: str = "" vol2_imageinfo_time: float = None - vol2_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) - vol3_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) - rekall_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) + vol2_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) + vol3_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) + rekall_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) @dataclass class VolatilityPlugin: name: str = "" - vol2_plugin_parameters: List[str] = field(default_factory = list) - vol3_plugin_parameters: List[str] = field(default_factory = list) - rekall_plugin_parameters: List[str] = field(default_factory = list) + vol2_plugin_parameters: List[str] = field(default_factory=list) + vol3_plugin_parameters: List[str] = field(default_factory=list) + rekall_plugin_parameters: List[str] = field(default_factory=list) class VolatilityTest: @@ -39,32 +39,50 @@ class VolatilityTest: def result_titles(self) -> List[str]: return [self.long_name] - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> None: + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> None: pass - def create_results(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> List[float]: + def create_results( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> List[float]: self.create_prerequisites(plugin, image, image_hash) # Volatility 2 Test - print(f"[*] Testing {self.short_name} {plugin.name} with image {image.filepath}") + print( + f"[*] Testing {self.short_name} {plugin.name} with image {image.filepath}" + ) os.chdir(self.path) cmd = self.plugin_cmd(plugin, image) start_time = time.perf_counter() try: - completed = subprocess.run(cmd, cwd = self.path, capture_output = True, timeout = 420) + completed = subprocess.run( + cmd, cwd=self.path, capture_output=True, timeout=420 + ) except subprocess.TimeoutExpired as excp: completed = excp end_time = time.perf_counter() total_time = end_time - start_time - print(f" Tested {self.short_name} {plugin.name} with image {image.filepath}: {total_time}") + print( + f" Tested {self.short_name} {plugin.name} with image {image.filepath}: {total_time}" + ) with open( - os.path.join(self.output_directory, f'{self.short_name}_{plugin.name}_{image_hash}_stdout'), - "wb") as f: + os.path.join( + self.output_directory, + f"{self.short_name}_{plugin.name}_{image_hash}_stdout", + ), + "wb", + ) as f: f.write(completed.stdout) if completed.stderr: with open( - os.path.join(self.output_directory, f'{self.short_name}_{plugin.name}_{image_hash}_stderr'), - "wb") as f: + os.path.join( + self.output_directory, + f"{self.short_name}_{plugin.name}_{image_hash}_stderr", + ), + "wb", + ) as f: f.write(completed.stderr) return [total_time] @@ -77,31 +95,57 @@ class Volatility2Test(VolatilityTest): long_name = "Volatility 2" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage): - return ["python2", "-u", "vol.py", "-f", image.filepath, "--profile", image.vol2_profile - ] + plugin.vol2_plugin_parameters + image.vol2_plugin_parameters.get(plugin.name, []) + return ( + [ + "python2", + "-u", + "vol.py", + "-f", + image.filepath, + "--profile", + image.vol2_profile, + ] + + plugin.vol2_plugin_parameters + + image.vol2_plugin_parameters.get(plugin.name, []) + ) def result_titles(self): return [self.long_name, "Imageinfo", f"{self.long_name} + Imageinfo"] - def create_results(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash) -> List[float]: + def create_results( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash + ) -> List[float]: result = super().create_results(plugin, image, image_hash) result += [image.vol2_imageinfo_time, result[0] + image.vol2_imageinfo_time] return result - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash): + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash + ): # Volatility 2 image info if not image.vol2_profile: - print(f"[*] Testing {self.short_name} imageinfo with image {image.filepath}") + print( + f"[*] Testing {self.short_name} imageinfo with image {image.filepath}" + ) os.chdir(self.path) cmd = ["python2", "-u", "vol.py", "-f", image.filepath, "imageinfo"] start_time = time.perf_counter() - vol2_completed = subprocess.run(cmd, cwd = self.path, capture_output = True) + vol2_completed = subprocess.run(cmd, cwd=self.path, capture_output=True) end_time = time.perf_counter() image.vol2_imageinfo_time = end_time - start_time - print(f" Tested volatility2 imageinfo with image {image.filepath}: {end_time - start_time}") - with open(os.path.join(self.output_directory, f'vol2_imageinfo_{image_hash}_stdout'), "wb") as f: + print( + f" Tested volatility2 imageinfo with image {image.filepath}: {end_time - start_time}" + ) + with open( + os.path.join( + self.output_directory, f"vol2_imageinfo_{image_hash}_stdout" + ), + "wb", + ) as f: f.write(vol2_completed.stdout) - image.vol2_profile = re.search(b"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout)[1] + image.vol2_profile = re.search( + rb"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout + )[1] class RekallTest(VolatilityTest): @@ -113,11 +157,16 @@ class RekallTest(VolatilityTest): plugin.rekall_plugin_parameters = plugin.vol2_plugin_parameters if not image.rekall_plugin_parameters: image.rekall_plugin_parameters = image.vol2_plugin_parameters - return ["rekall", "-f", image.filepath] + plugin.rekall_plugin_parameters + image.rekall_plugin_parameters.get( - plugin.name, []) + return ( + ["rekall", "-f", image.filepath] + + plugin.rekall_plugin_parameters + + image.rekall_plugin_parameters.get(plugin.name, []) + ) - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> None: - shutil.rmtree('/home/mike/.rekall_cache/sessions') + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> None: + shutil.rmtree("/home/mike/.rekall_cache/sessions") class Volatility3Test(VolatilityTest): @@ -125,14 +174,18 @@ class Volatility3Test(VolatilityTest): long_name = "Volatility 3" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage) -> List[str]: - return [ - "python", - "-u", - "vol.py", - "-q", - "-f", - image.filepath, - ] + plugin.vol3_plugin_parameters + image.vol3_plugin_parameters.get(plugin.name, []) + return ( + [ + "python", + "-u", + "vol.py", + "-q", + "-f", + image.filepath, + ] + + plugin.vol3_plugin_parameters + + image.vol3_plugin_parameters.get(plugin.name, []) + ) class Volatility3PyPyTest(VolatilityTest): @@ -140,26 +193,32 @@ class Volatility3PyPyTest(VolatilityTest): long_name = "Volatility 3 (PyPy)" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage) -> List[str]: - return [ - "pypy3", - "-u", - "vol.py", - "-q", - "-f", - image.filepath, - ] + plugin.vol3_plugin_parameters + image.vol3_plugin_parameters.get(plugin.name, []) + return ( + [ + "pypy3", + "-u", + "vol.py", + "-q", + "-f", + image.filepath, + ] + + plugin.vol3_plugin_parameters + + image.vol3_plugin_parameters.get(plugin.name, []) + ) class VolatilityTester: - def __init__(self, - images: List[VolatilityImage], - plugins: List[VolatilityPlugin], - frameworks: List[str], - output_dir: str, - vol2_path: str = None, - vol3_path: str = None, - rekall_path = None): + def __init__( + self, + images: List[VolatilityImage], + plugins: List[VolatilityPlugin], + frameworks: List[str], + output_dir: str, + vol2_path: str = None, + vol3_path: str = None, + rekall_path=None, + ): self.images = images self.plugins = plugins if not vol2_path: @@ -172,7 +231,7 @@ class VolatilityTester: Volatility3Test(vol3_path, output_dir), Volatility3PyPyTest(vol3_path, output_dir), Volatility2Test(vol2_path, output_dir), - RekallTest(rekall_path, output_dir) + RekallTest(rekall_path, output_dir), ] self.tests = [x for x in available_tests if x.short_name.lower() in frameworks] self.csv_writer = None @@ -183,7 +242,7 @@ class VolatilityTester: print(f"[?] Frameworks: {[x.long_name for x in self.tests]}") def run_tests(self): - with open("volatility-timings.csv", 'w') as csvfile: + with open("volatility-timings.csv", "w") as csvfile: self.csv_writer = csv.writer(csvfile) titles = ["Image Hash", "Image Path", "Plugin Name"] for test in self.tests: @@ -203,72 +262,121 @@ class VolatilityTester: self.csv_writer.writerow([image_hash, image.filepath, plugin.name] + results) -if __name__ == '__main__': +if __name__ == "__main__": plugins = [ - VolatilityPlugin(name = "pslist", - vol2_plugin_parameters = ["pslist"], - vol3_plugin_parameters = ["windows.pslist"]), - VolatilityPlugin(name = "psscan", - vol2_plugin_parameters = ["psscan"], - vol3_plugin_parameters = ["windows.psscan"], - rekall_plugin_parameters = ["psscan", "--scan_kernel"]), - VolatilityPlugin(name = "driverscan", - vol2_plugin_parameters = ["driverscan"], - vol3_plugin_parameters = ["windows.driverscan"], - rekall_plugin_parameters = ["driverscan", "--scan_kernel"]), - VolatilityPlugin(name = "handles", - vol2_plugin_parameters = ["handles"], - vol3_plugin_parameters = ["windows.handles"]), - VolatilityPlugin(name = "modules", - vol2_plugin_parameters = ["modules"], - vol3_plugin_parameters = ["windows.modules"]), - VolatilityPlugin(name = "hivelist", - vol2_plugin_parameters = ["hivelist"], - vol3_plugin_parameters = ["registry.hivelist"], - rekall_plugin_parameters = ["hives"]), - VolatilityPlugin(name = "vadinfo", - vol2_plugin_parameters = ["vadinfo"], - vol3_plugin_parameters = ["windows.vadinfo"], - rekall_plugin_parameters = ["vad"]), - VolatilityPlugin(name = "modscan", - vol2_plugin_parameters = ["modscan"], - vol3_plugin_parameters = ["windows.modscan"], - rekall_plugin_parameters = ["modscan", "--scan_kernel"]), - VolatilityPlugin(name = "svcscan", - vol2_plugin_parameters = ["svcscan"], - vol3_plugin_parameters = ["windows.svcscan"], - rekall_plugin_parameters = ["svcscan"]), - VolatilityPlugin(name = "ssdt", vol2_plugin_parameters = ["ssdt"], vol3_plugin_parameters = ["windows.ssdt"]), - VolatilityPlugin(name = "printkey", - vol2_plugin_parameters = ["printkey", "-K", "Classes"], - vol3_plugin_parameters = ["registry.printkey", "--key", "Classes"], - rekall_plugin_parameters = ["printkey", "--key", "Classes"]) + VolatilityPlugin( + name="pslist", + vol2_plugin_parameters=["pslist"], + vol3_plugin_parameters=["windows.pslist"], + ), + VolatilityPlugin( + name="psscan", + vol2_plugin_parameters=["psscan"], + vol3_plugin_parameters=["windows.psscan"], + rekall_plugin_parameters=["psscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="driverscan", + vol2_plugin_parameters=["driverscan"], + vol3_plugin_parameters=["windows.driverscan"], + rekall_plugin_parameters=["driverscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="handles", + vol2_plugin_parameters=["handles"], + vol3_plugin_parameters=["windows.handles"], + ), + VolatilityPlugin( + name="modules", + vol2_plugin_parameters=["modules"], + vol3_plugin_parameters=["windows.modules"], + ), + VolatilityPlugin( + name="hivelist", + vol2_plugin_parameters=["hivelist"], + vol3_plugin_parameters=["registry.hivelist"], + rekall_plugin_parameters=["hives"], + ), + VolatilityPlugin( + name="vadinfo", + vol2_plugin_parameters=["vadinfo"], + vol3_plugin_parameters=["windows.vadinfo"], + rekall_plugin_parameters=["vad"], + ), + VolatilityPlugin( + name="modscan", + vol2_plugin_parameters=["modscan"], + vol3_plugin_parameters=["windows.modscan"], + rekall_plugin_parameters=["modscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="svcscan", + vol2_plugin_parameters=["svcscan"], + vol3_plugin_parameters=["windows.svcscan"], + rekall_plugin_parameters=["svcscan"], + ), + VolatilityPlugin( + name="ssdt", + vol2_plugin_parameters=["ssdt"], + vol3_plugin_parameters=["windows.ssdt"], + ), + VolatilityPlugin( + name="printkey", + vol2_plugin_parameters=["printkey", "-K", "Classes"], + vol3_plugin_parameters=["registry.printkey", "--key", "Classes"], + rekall_plugin_parameters=["printkey", "--key", "Classes"], + ), ] parser = argparse.ArgumentParser() - parser.add_argument("--output-dir", type = str, default = os.getcwd(), help = "Directory to store all results") - parser.add_argument("--vol3path", - type = str, - default = os.path.join(os.getcwd(), 'volatility3'), - help = "Path ot the volatility 3 directory") - parser.add_argument("--vol2path", - type = str, - default = os.path.join(os.getcwd(), 'volatility'), - help = "Path to the volatility 2 directory") - parser.add_argument("--rekallpath", - type = str, - default = os.path.join(os.getcwd(), 'rekall'), - help = "Path to the rekall directory") - parser.add_argument("--frameworks", - nargs = "+", - type = str, - choices = [x.short_name.lower() for x in VolatilityTest.__subclasses__()], - default = [x.short_name.lower() for x in VolatilityTest.__subclasses__()], - help = "A comma separated list of frameworks to test") - parser.add_argument('images', metavar = 'IMAGE', type = str, nargs = '+', help = 'The list of images to compare') + parser.add_argument( + "--output-dir", + type=str, + default=os.getcwd(), + help="Directory to store all results", + ) + parser.add_argument( + "--vol3path", + type=str, + default=os.path.join(os.getcwd(), "volatility3"), + help="Path ot the volatility 3 directory", + ) + parser.add_argument( + "--vol2path", + type=str, + default=os.path.join(os.getcwd(), "volatility"), + help="Path to the volatility 2 directory", + ) + parser.add_argument( + "--rekallpath", + type=str, + default=os.path.join(os.getcwd(), "rekall"), + help="Path to the rekall directory", + ) + parser.add_argument( + "--frameworks", + nargs="+", + type=str, + choices=[x.short_name.lower() for x in VolatilityTest.__subclasses__()], + default=[x.short_name.lower() for x in VolatilityTest.__subclasses__()], + help="A comma separated list of frameworks to test", + ) + parser.add_argument( + "images", + metavar="IMAGE", + type=str, + nargs="+", + help="The list of images to compare", + ) args = parser.parse_args() - vt = VolatilityTester([VolatilityImage(filepath = x) for x in args.images], plugins, - [x.lower() for x in args.frameworks], args.output_dir, args.vol2path, args.vol3path, - args.rekallpath) + vt = VolatilityTester( + [VolatilityImage(filepath=x) for x in args.images], + plugins, + [x.lower() for x in args.frameworks], + args.output_dir, + args.vol2path, + args.vol3path, + args.rekallpath, + ) vt.run_tests() diff --git a/development/mac-kdk/parse_pbzx2.py b/development/mac-kdk/parse_pbzx2.py index 173a4d648..b175539b3 100644 --- a/development/mac-kdk/parse_pbzx2.py +++ b/development/mac-kdk/parse_pbzx2.py @@ -7,11 +7,12 @@ # Cleaned up C version (as the basis for my code) here, thanks to Pepijn Bruienne / @bruienne # https://gist.github.com/bruienne/029494bbcfb358098b41 +import os import struct import sys -def seekread(f, offset = None, length = 0, relative = True): +def seekread(f, offset=None, length=0, relative=True): if offset is not None: # offset provided, let's seek f.seek(offset, [0, 1, 2][relative]) @@ -22,55 +23,57 @@ def seekread(f, offset = None, length = 0, relative = True): def parse_pbzx(pbzx_path): section = 0 - xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) - with open(pbzx_path, 'rb') as f: + xar_out_path = f"{pbzx_path}.part{section:02d}.cpio.xz" + with open(pbzx_path, "rb") as f: # pbzx = f.read() # f.close() - magic = seekread(f, length = 4) - if magic != 'pbzx': + magic = seekread(f, length=4) + if magic != "pbzx": raise RuntimeError("Error: Not a pbzx file") # Read 8 bytes for initial flags - flags = seekread(f, length = 8) + flags = seekread(f, length=8) # Interpret the flags as a 64-bit big-endian unsigned int - flags = struct.unpack('>Q', flags)[0] + flags = struct.unpack(">Q", flags)[0] while flags & (1 << 24): - with open(xar_out_path, 'wb') as xar_f: + with open(xar_out_path, "wb") as xar_f: xar_f.seek(0, os.SEEK_END) # Read in more flags - flags = seekread(f, length = 8) - flags = struct.unpack('>Q', flags)[0] + flags = seekread(f, length=8) + flags = struct.unpack(">Q", flags)[0] # Read in length - f_length = seekread(f, length = 8) - f_length = struct.unpack('>Q', f_length)[0] - xzmagic = seekread(f, length = 6) - if xzmagic != '\xfd7zXZ\x00': + f_length = seekread(f, length=8) + f_length = struct.unpack(">Q", f_length)[0] + xzmagic = seekread(f, length=6) + if xzmagic != "\xfd7zXZ\x00": # This isn't xz content, this is actually _raw decompressed cpio_ chunk of 16MB in size... # Let's back up ... - seekread(f, offset = -6, length = 0) + seekread(f, offset=-6, length=0) # ... and split it out ... - f_content = seekread(f, length = f_length) + f_content = seekread(f, length=f_length) section += 1 - decomp_out = '%s.part%02d.cpio' % (pbzx_path, section) - with open(decomp_out, 'wb') as g: + decomp_out = f"{pbzx_path}.part{section:02d}.cpio" + with open(decomp_out, "wb") as g: g.write(f_content) # Now to start the next section, which should hopefully be .xz (we'll just assume it is ...) section += 1 - xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) + xar_out_path = f"{pbzx_path}.part{section:02d}.cpio.xz" else: f_length -= 6 # This part needs buffering - f_content = seekread(f, length = f_length) - tail = seekread(f, offset = -2, length = 2) + f_content = seekread(f, length=f_length) + tail = seekread(f, offset=-2, length=2) xar_f.write(xzmagic) xar_f.write(f_content) - if tail != 'YZ': + if tail != "YZ": raise RuntimeError("Error: Footer is not xar file footer") def main(): parse_pbzx(sys.argv[1]) - print("Now xz decompress the .xz chunks, then 'cat' them all together in order into a single new.cpio file") + print( + "Now xz decompress the .xz chunks, then 'cat' them all together in order into a single new.cpio file" + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/development/pdbparse-to-json.py b/development/pdbparse-to-json.py index 819e44e15..49b4da009 100644 --- a/development/pdbparse-to-json.py +++ b/development/pdbparse-to-json.py @@ -13,10 +13,10 @@ import pdbparse.undecorate logger = logging.getLogger(__name__) logger.setLevel(1) -if __name__ == '__main__': +if __name__ == "__main__": console = logging.StreamHandler() console.setLevel(1) - formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') + formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) logger.addHandler(console) @@ -25,19 +25,19 @@ class PDBRetreiver: def retreive_pdb(self, guid: str, file_name: str) -> Optional[str]: logger.info("Download PDB file...") - file_name = ".".join(file_name.split(".")[:-1] + ['pdb']) - for sym_url in ['http://msdl.microsoft.com/download/symbols']: + file_name = ".".join(file_name.split(".")[:-1] + ["pdb"]) + for sym_url in ["http://msdl.microsoft.com/download/symbols"]: url = sym_url + f"/{file_name}/{guid}/" result = None - for suffix in [file_name[:-1] + '_', file_name]: + for suffix in [file_name[:-1] + "_", file_name]: try: - logger.debug(f"Attempting to retrieve {url + suffix}") + logger.debug("Attempting to retrieve %s", url + suffix) result, _ = request.urlretrieve(url + suffix) except request.HTTPError as excp: - logger.debug(f"Failed with {excp}") + logger.debug("Failed with %s", excp) if result: - logger.debug(f"Successfully written to {result}") + logger.debug("Successfully written to %s", result) break return result @@ -69,7 +69,7 @@ class PDBConvertor: "float": "float", "double": "float", "long double": "float", - "void": "void" + "void": "void", } base_type_size = { @@ -122,13 +122,18 @@ class PDBConvertor: self._seen_ctypes.add(ctype) return self.ctype[ctype] - def lookup_ctype_pointers(self, ctype_pointer: str) -> Dict[str, Union[str, Dict[str, str]]]: - base_type = ctype_pointer.replace('32P', '').replace('64P', '') + def lookup_ctype_pointers( + self, ctype_pointer: str + ) -> Dict[str, Union[str, Dict[str, str]]]: + base_type = ctype_pointer.replace("32P", "").replace("64P", "") if base_type == ctype_pointer: # We raise a KeyError, because we've been asked about a type that isn't a pointer raise KeyError self._seen_ctypes.add(base_type) - return {"kind": "pointer", "subtype": {"kind": "base", "name": self.ctype[base_type]}} + return { + "kind": "pointer", + "subtype": {"kind": "base", "name": self.ctype[base_type]}, + } def read_pdb(self) -> Dict: """Reads in the PDB file and forms essentially a python dictionary of necessary data""" @@ -137,32 +142,31 @@ class PDBConvertor: "enums": self.read_enums(), "metadata": self.generate_metadata(), "symbols": self.read_symbols(), - "base_types": self.read_basetypes() + "base_types": self.read_basetypes(), } return output def generate_metadata(self) -> Dict[str, Any]: """Generates the metadata necessary for this object""" dbg = self._pdb.STREAM_DBI - last_bytes = str(binascii.hexlify(self._pdb.STREAM_PDB.GUID.Data4), 'ascii')[-16:] - guidstr = u'{:08x}{:04x}{:04x}{}'.format(self._pdb.STREAM_PDB.GUID.Data1, self._pdb.STREAM_PDB.GUID.Data2, - self._pdb.STREAM_PDB.GUID.Data3, last_bytes) + last_bytes = str(binascii.hexlify(self._pdb.STREAM_PDB.GUID.Data4), "ascii")[ + -16: + ] + guidstr = f"{self._pdb.STREAM_PDB.GUID.Data1:08x}{self._pdb.STREAM_PDB.GUID.Data2:04x}{self._pdb.STREAM_PDB.GUID.Data3:04x}{last_bytes}" pdb_data = { "GUID": guidstr.upper(), "age": self._pdb.STREAM_PDB.Age, "database": "ntkrnlmp.pdb", - "machine_type": int(dbg.machine) + "machine_type": int(dbg.machine), } result = { "format": "6.0.0", "producer": { "datetime": datetime.datetime.now().isoformat(), "name": "pdbconv", - "version": "0.1.0" + "version": "0.1.0", }, - "windows": { - "pdb": pdb_data - } + "windows": {"pdb": pdb_data}, } return result @@ -173,16 +177,21 @@ class PDBConvertor: stream = self._pdb.STREAM_TPI for type_index in stream.types: user_type = stream.types[type_index] - if (user_type.leaf_type == "LF_ENUM" and not user_type.prop.fwdref): + if user_type.leaf_type == "LF_ENUM" and not user_type.prop.fwdref: output.update(self._format_enum(user_type)) return output def _format_enum(self, user_enum): output = { user_enum.name: { - 'base': self.lookup_ctype(user_enum.utype), - 'size': self._determine_size(user_enum.utype), - 'constants': dict([(enum.name, enum.enum_value) for enum in user_enum.fieldlist.substructs]) + "base": self.lookup_ctype(user_enum.utype), + "size": self._determine_size(user_enum.utype), + "constants": dict( + [ + (enum.name, enum.enum_value) + for enum in user_enum.fieldlist.substructs + ] + ), } } return output @@ -195,14 +204,14 @@ class PDBConvertor: try: sects = self._pdb.STREAM_SECT_HDR_ORIG.sections omap = self._pdb.STREAM_OMAP_FROM_SRC - except AttributeError as e: + except AttributeError: # In this case there is no OMAP, so we use the given section # headers and use the identity function for omap.remap sects = self._pdb.STREAM_SECT_HDR.sections omap = None for sym in self._pdb.STREAM_GSYM.globals: - if not hasattr(sym, 'offset'): + if not hasattr(sym, "offset"): continue try: virt_base = sects[sym.segment - 1].VirtualAddress @@ -223,9 +232,9 @@ class PDBConvertor: stream = self._pdb.STREAM_TPI for type_index in stream.types: user_type = stream.types[type_index] - if (user_type.leaf_type == "LF_STRUCTURE" and not user_type.prop.fwdref): + if user_type.leaf_type == "LF_STRUCTURE" and not user_type.prop.fwdref: output.update(self._format_usertype(user_type, "struct")) - elif (user_type.leaf_type == "LF_UNION" and not user_type.prop.fwdref): + elif user_type.leaf_type == "LF_UNION" and not user_type.prop.fwdref: output.update(self._format_usertype(user_type, "union")) return output @@ -233,16 +242,22 @@ class PDBConvertor: """Produces a single usertype""" fields: Dict[str, Dict[str, Any]] = {} [fields.update(self._format_field(s)) for s in usertype.fieldlist.substructs] - return {usertype.name: {'fields': fields, 'kind': kind, 'size': usertype.size}} + return {usertype.name: {"fields": fields, "kind": kind, "size": usertype.size}} def _format_field(self, field) -> Dict[str, Dict[str, Any]]: - return {field.name: {"offset": field.offset, "type": self._format_kind(field.index)}} + return { + field.name: {"offset": field.offset, "type": self._format_kind(field.index)} + } def _determine_size(self, field): output = None if isinstance(field, str): output = self.base_type_size[field] - elif (field.leaf_type == "LF_STRUCTURE" or field.leaf_type == "LF_ARRAY" or field.leaf_type == "LF_UNION"): + elif ( + field.leaf_type == "LF_STRUCTURE" + or field.leaf_type == "LF_ARRAY" + or field.leaf_type == "LF_UNION" + ): output = field.size elif field.leaf_type == "LF_POINTER": output = self.base_type_size[field.ptr_attr.type] @@ -256,6 +271,7 @@ class PDBConvertor: output = self._determine_size(field.index) if output is None: import pdb + pdb.set_trace() raise ValueError(f"Unknown size for field: {field.name}") return output @@ -267,36 +283,37 @@ class PDBConvertor: output = self.lookup_ctype_pointers(kind) except KeyError: try: - output = {'kind': 'base', 'name': self.lookup_ctype(kind)} + output = {"kind": "base", "name": self.lookup_ctype(kind)} except KeyError: - output = {'kind': 'base', 'name': kind} - elif kind.leaf_type == 'LF_MODIFIER': + output = {"kind": "base", "name": kind} + elif kind.leaf_type == "LF_MODIFIER": output = self._format_kind(kind.modified_type) - elif kind.leaf_type == 'LF_STRUCTURE': - output = {'kind': 'struct', 'name': kind.name} - elif kind.leaf_type == 'LF_UNION': - output = {'kind': 'union', 'name': kind.name} - elif kind.leaf_type == 'LF_BITFIELD': + elif kind.leaf_type == "LF_STRUCTURE": + output = {"kind": "struct", "name": kind.name} + elif kind.leaf_type == "LF_UNION": + output = {"kind": "union", "name": kind.name} + elif kind.leaf_type == "LF_BITFIELD": output = { - 'kind': 'bitfield', - 'type': self._format_kind(kind.base_type), - 'bit_length': kind.length, - 'bit_position': kind.position + "kind": "bitfield", + "type": self._format_kind(kind.base_type), + "bit_length": kind.length, + "bit_position": kind.position, } - elif kind.leaf_type == 'LF_POINTER': - output = {'kind': 'pointer', 'subtype': self._format_kind(kind.utype)} - elif kind.leaf_type == 'LF_ARRAY': + elif kind.leaf_type == "LF_POINTER": + output = {"kind": "pointer", "subtype": self._format_kind(kind.utype)} + elif kind.leaf_type == "LF_ARRAY": output = { - 'kind': 'array', - 'count': kind.size // self._determine_size(kind.element_type), - 'subtype': self._format_kind(kind.element_type) + "kind": "array", + "count": kind.size // self._determine_size(kind.element_type), + "subtype": self._format_kind(kind.element_type), } - elif kind.leaf_type == 'LF_ENUM': - output = {'kind': 'enum', 'name': kind.name} - elif kind.leaf_type == 'LF_PROCEDURE': - output = {'kind': "function"} + elif kind.leaf_type == "LF_ENUM": + output = {"kind": "enum", "name": kind.name} + elif kind.leaf_type == "LF_PROCEDURE": + output = {"kind": "function"} else: import pdb + pdb.set_trace() return output @@ -306,40 +323,70 @@ class PDBConvertor: if "64" in self._pdb.STREAM_DBI.machine: ptr_size = 8 - output = {"pointer": {"endian": "little", "kind": "int", "signed": False, "size": ptr_size}} + output = { + "pointer": { + "endian": "little", + "kind": "int", + "signed": False, + "size": ptr_size, + } + } for index in self._seen_ctypes: output[self.ctype[index]] = { "endian": "little", "kind": self.ctype_python_types.get(self.ctype[index], "int"), "signed": False if "_U" in index else True, - "size": self.base_type_size[index] + "size": self.base_type_size[index], } return output -if __name__ == '__main__': - parser = argparse.ArgumentParser(description = "Convertor for PDB files to Volatility 3 Intermediate Symbol Format") - parser.add_argument("-o", "--output", metavar = "OUTPUT", help = "Filename for data output", required = True) - file_group = parser.add_argument_group("file", description = "File-based conversion of PDB to ISF") - file_group.add_argument("-f", "--file", metavar = "FILE", help = "PDB file to translate to ISF") - data_group = parser.add_argument_group("data", description = "Convert based on a GUID and filename pattern") - data_group.add_argument("-p", "--pattern", metavar = "PATTERN", help = "Filename pattern to recover PDB file") - data_group.add_argument("-g", - "--guid", - metavar = "GUID", - help = "GUID + Age string for the required PDB file", - default = None) - data_group.add_argument("-k", - "--keep", - action = "store_true", - default = False, - help = "Keep the downloaded PDB file") +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Convertor for PDB files to Volatility 3 Intermediate Symbol Format" + ) + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT", + help="Filename for data output", + required=True, + ) + file_group = parser.add_argument_group( + "file", description="File-based conversion of PDB to ISF" + ) + file_group.add_argument( + "-f", "--file", metavar="FILE", help="PDB file to translate to ISF" + ) + data_group = parser.add_argument_group( + "data", description="Convert based on a GUID and filename pattern" + ) + data_group.add_argument( + "-p", + "--pattern", + metavar="PATTERN", + help="Filename pattern to recover PDB file", + ) + data_group.add_argument( + "-g", + "--guid", + metavar="GUID", + help="GUID + Age string for the required PDB file", + default=None, + ) + data_group.add_argument( + "-k", + "--keep", + action="store_true", + default=False, + help="Keep the downloaded PDB file", + ) args = parser.parse_args() delfile = False filename = None if args.guid is not None and args.pattern is not None: - filename = PDBRetreiver().retreive_pdb(guid = args.guid, file_name = args.pattern) + filename = PDBRetreiver().retreive_pdb(guid=args.guid, file_name=args.pattern) delfile = True elif args.file: filename = args.file @@ -352,7 +399,7 @@ if __name__ == '__main__': convertor = PDBConvertor(filename) with open(args.output, "w") as f: - json.dump(convertor.read_pdb(), f, indent = 2, sort_keys = True) + json.dump(convertor.read_pdb(), f, indent=2, sort_keys=True) if args.keep: print(f"Temporary PDB file: {filename}") diff --git a/development/schema_validate.py b/development/schema_validate.py index 0908e934f..cf9565d68 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -1,34 +1,33 @@ import argparse import json +import logging import os import sys # TODO: Rather nasty hack, when volatility's actually installed this would be unnecessary sys.path += ".." -import logging - console = logging.StreamHandler() console.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') +formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) logger = logging.getLogger("") logger.addHandler(console) logger.setLevel(logging.DEBUG) -from volatility3 import schemas +from volatility3 import schemas # noqa: E402 -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser("Validates ") - parser.add_argument("-s", "--schema", dest = "schema", default = None) - parser.add_argument("filenames", metavar = "FILE", nargs = '+') + parser.add_argument("-s", "--schema", dest="schema", default=None) + parser.add_argument("filenames", metavar="FILE", nargs="+") args = parser.parse_args() schema = None if args.schema: - with open(os.path.abspath(args.schema), 'r') as s: + with open(os.path.abspath(args.schema)) as s: schema = json.load(s) failures = [] @@ -36,7 +35,7 @@ if __name__ == '__main__': try: if os.path.exists(filename): print(f"[?] Validating file: {filename}") - with open(filename, 'r') as t: + with open(filename) as t: test = json.load(t) if args.schema: diff --git a/development/stock-linux-json.py b/development/stock-linux-json.py index 877f78e1c..967cc7e18 100644 --- a/development/stock-linux-json.py +++ b/development/stock-linux-json.py @@ -9,7 +9,7 @@ import requests import rpmfile from debian import debfile -DWARF2JSON = './dwarf2json' +DWARF2JSON = "./dwarf2json" class Downloader: @@ -17,7 +17,7 @@ class Downloader: def __init__(self, url_lists: List[List[str]]) -> None: self.url_lists = url_lists - def download_lists(self, keep = False): + def download_lists(self, keep=False): for url_list in self.url_lists: print("Downloading files...") files_for_processing = self.download_list(url_list) @@ -35,43 +35,45 @@ class Downloader: with tempfile.NamedTemporaryFile() as archivedata: archivedata.write(data.content) archivedata.seek(0) - if url.endswith('.rpm'): + if url.endswith(".rpm"): processed_files[url] = self.process_rpm(archivedata) - elif url.endswith('.deb'): + elif url.endswith(".deb"): processed_files[url] = self.process_deb(archivedata) return processed_files def process_rpm(self, archivedata) -> Optional[str]: - rpm = rpmfile.RPMFile(fileobj = archivedata) + rpm = rpmfile.RPMFile(fileobj=archivedata) member = None extracted = None for member in rpm.getmembers(): - if 'vmlinux' in member.name or 'System.map' in member.name: + if "vmlinux" in member.name or "System.map" in member.name: print(f" - Extracting {member.name}") extracted = rpm.extractfile(member) break if not member or not extracted: return None - with tempfile.NamedTemporaryFile(delete = False, - prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: + with tempfile.NamedTemporaryFile( + delete=False, prefix="vmlinux" if "vmlinux" in member.name else "System.map" + ) as output: print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name def process_deb(self, archivedata) -> Optional[str]: - deb = debfile.DebFile(fileobj = archivedata) + deb = debfile.DebFile(fileobj=archivedata) member = None extracted = None for member in deb.data.tgz().getmembers(): - if member.name.endswith('vmlinux') or 'System.map' in member.name: + if member.name.endswith("vmlinux") or "System.map" in member.name: print(f" - Extracting {member.name}") extracted = deb.data.get_file(member.name) break if not member or not extracted: return None - with tempfile.NamedTemporaryFile(delete = False, - prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: + with tempfile.NamedTemporaryFile( + delete=False, prefix="vmlinux" if "vmlinux" in member.name else "System.map" + ) as output: print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name @@ -83,43 +85,55 @@ class Downloader: if named_files[i] is None: print(f"FAILURE: None encountered for {i}") return - args = [DWARF2JSON, 'linux'] - output_filename = 'unknown-kernel.json' + args = [DWARF2JSON, "linux"] + output_filename = "unknown-kernel.json" for named_file in named_files: - prefix = '--system-map' - if 'System' not in named_files[named_file]: - prefix = '--elf' - output_filename = './' + '-'.join((named_file.split('/')[-1]).split('-')[2:])[:-4] + '.json.xz' + prefix = "--system-map" + if "System" not in named_files[named_file]: + prefix = "--elf" + output_filename = ( + "./" + + "-".join((named_file.split("/")[-1]).split("-")[2:])[:-4] + + ".json.xz" + ) args += [prefix, named_files[named_file]] print(f" - Running {args}") - proc = subprocess.run(args, capture_output = True) + proc = subprocess.run(args, capture_output=True) print(f" - Writing to {output_filename}") - with lzma.open(output_filename, 'w') as f: + with lzma.open(output_filename, "w") as f: f.write(proc.stdout) -if __name__ == '__main__': - parser = argparse.ArgumentParser(description = "Takes a list of URLs for Centos and downloads them") - parser.add_argument("-f", - "--file", - dest = 'filename', - metavar = "FILENAME", - help = "Filename to be read", - required = True) - parser.add_argument("-d", - "--dwarf2json", - dest = 'dwarfpath', - metavar = "PATH", - default = DWARF2JSON, - help = "Path to the dwarf2json binary", - required = True) - parser.add_argument("-k", - "--keep", - dest = 'keep', - action = 'store_true', - help = 'Keep extracted temporary files after completion', - default = False) +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Takes a list of URLs for Centos and downloads them" + ) + parser.add_argument( + "-f", + "--file", + dest="filename", + metavar="FILENAME", + help="Filename to be read", + required=True, + ) + parser.add_argument( + "-d", + "--dwarf2json", + dest="dwarfpath", + metavar="PATH", + default=DWARF2JSON, + help="Path to the dwarf2json binary", + required=True, + ) + parser.add_argument( + "-k", + "--keep", + dest="keep", + action="store_true", + help="Keep extracted temporary files after completion", + default=False, + ) args = parser.parse_args() DWARF2JSON = args.dwarfpath @@ -132,4 +146,4 @@ if __name__ == '__main__': urls += [[lines[2 * i].strip(), lines[(2 * i) + 1].strip()]] d = Downloader(urls) - d.download_lists(keep = args.keep) + d.download_lists(keep=args.keep) diff --git a/doc/source/conf.py b/doc/source/conf.py index cabfdc327..7a9a72891 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -19,6 +19,8 @@ import sys import sphinx.ext.apidoc +from importlib.util import find_spec + def setup(app): volatility_directory = os.path.abspath( @@ -124,7 +126,7 @@ def setup(app): # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath("../..")) -from volatility3.framework import constants +from volatility3.framework import constants # noqa: E402 # -- General configuration ------------------------------------------------ @@ -147,13 +149,9 @@ extensions = [ autosectionlabel_prefix_document = True -try: - import sphinx_autodoc_typehints - +if find_spec("sphinx_autodoc_typehints") is not None: extensions.append("sphinx_autodoc_typehints") -except ImportError: - # If the autodoc typehints extension isn't available, carry on regardless - pass +# If the autodoc typehints extension isn't available, carry on regardless # Add any paths that contain templates here, relative to this directory. # templates_path = ['tools/templates'] diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 66dabfafe..d3bc9613a 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -23,7 +23,7 @@ Alignment .. _Array: Array - This represents a list of items, which can be access by an index, which is zero-based (meaning the first + This represents a list of items, which can be accessed by an index, which is zero-based (meaning the first element has index 0). Items in arrays are almost always the same size (it is not a generic list, as in python) even if they are :ref:`pointers` to different sized objects. @@ -43,7 +43,14 @@ Dereference .. _Domain: Domain - This the grouping for input values for a mapping or mathematical function. + The set of input values for a mapping or mathematical function. + +I +- +.. _Intermediate Symbol File (ISF): + +Intermediate Symbol File (ISF) + They contain kernel structures and specific offsets formatted as JSON. For macOS and Linux analysis, the kernel needs to be added as an ISF file to the volatility 3 symbols directory. For Windows, the required ISF file can often be generated from PDB files automatically downloaded from Microsoft servers, and therefore does not require manual intervention. M - @@ -54,9 +61,7 @@ Map, mapping of the :ref:`Range`). Mappings can be seen as a mathematical function, and therefore volatility 3 attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). - For further information, please see - `Function (mathematics) in wikipedia https://en.wikipedia.org/wiki/Function_(mathematics)` - + For further information, please see `Function (mathematics) in Wikipedia_`. .. _Member: @@ -69,7 +74,7 @@ O .. _Object: Object - This has a specific meaning within computer programming (as in Object Oriented Programming), but within the world + This has a specific meaning within computer programming (as in object-oriented programming), but within the world of Volatility it is used to refer to a type that has been associated with a chunk of data, or a specific instance of a type. See also :ref:`Type`. @@ -116,6 +121,11 @@ Page Table possible to use them as a way to map a particular address within a (potentially larger, but sparsely populated) virtual space to a concrete (and usually contiguous) physical space, through the process of :ref:`mapping`. +.. _Plugin: + +Plugin + Plugins are the "functions" of the volatility framework. They carry out algorithms on data stored in layers using objects constructed from symbols. Broadly, plugins take in a number of TranslationLayers (the data, which is a representation of part of an image, in a specified type described by templates) and outputs a TreeGrid. + .. _Pointer: Pointer @@ -145,9 +155,9 @@ Struct, Structure Symbol This is used in many different contexts, as a short term for many things. Within Volatility, a symbol is a - construct that usually encompasses a specific type :ref:`type` at a specific :ref:`offset`, + construct that usually encompasses a specific :ref:`type` at a specific :ref:`offset`, representing a particular instance of that type within the memory of a compiled and running program. An example - would be the location in memory of a list of active tcp endpoints maintained by the networking stack + would be the location in memory of a list of active TCP endpoints maintained by the networking stack within an operating system. T diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 39670a62d..07d9e1467 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -41,24 +41,36 @@ to be able to run properly. Any that are defined as optional need not necessari @classmethod def get_requirements(cls): - return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True), - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (2, 0, 0))] + return [ + requirements.ModuleRequirement( + name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"] + ), + requirements.ListRequirement( + name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True + ), + requirements.PluginRequirement( + name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0) + ), + ] -This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how +This is a classmethod, so it can be called before the specific plugin object has been instantiated (in order to know how to instantiate the plugin). At the moment these requirements are fairly straightforward: :: - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"] + ), This requirement specifies the need for a particular submodule. Each module requires a :py:class:`TranslationLayer ` and a @@ -85,9 +97,11 @@ not be requested directly from the user. :: - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.TranslationLayerRequirement( + name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"] + ), This requirement indicates that the plugin will operate on a single :py:class:`TranslationLayer `. The name of the @@ -110,8 +124,10 @@ not be requested directly from the user. :: - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + requirements.SymbolTableRequirement( + name = "nt_symbols", + description = "Windows kernel symbols" + ), This requirement specifies the need for a particular :py:class:`SymbolTable ` @@ -127,10 +143,12 @@ not be requested directly from the user. :: - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True), + requirements.ListRequirement( + name = 'pid', + description = 'Filter on specific process IDs', + element_type = int, + optional = True + ), The next requirement is a List Requirement, populated by integers. The description will be presented to the user to describe what the value represents. The optional flag indicates that the plugin can function without the ``pid`` value @@ -138,9 +156,11 @@ being defined within the configuration tree at all. :: - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (2, 0, 0))] + requirements.PluginRequirement( + name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0) + ) This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements on that plugin. The version is specified in terms of Semantic Versioning meaning that, to be compatible, the major @@ -180,16 +200,24 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) kernel = self.context.modules[self.config['kernel']] - return renderers.TreeGrid([("PID", int), - ("Process", str), - ("Base", format_hints.Hex), - ("Size", format_hints.Hex), - ("Name", str), - ("Path", str)], - self._generator(pslist.PsList.list_processes(self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Base", format_hints.Hex), + ("Size", format_hints.Hex), + ("Name", str), + ("Path", str), + ], + self._generator( + pslist.PsList.list_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func = filter_func + ) + ) + ) In this instance, the plugin constructs a filter (using the PsList plugin's *classmethod* for creating filters). It checks the plugin's configuration for the ``pid`` value, and passes it in as a list if it finds it, or None if @@ -281,5 +309,3 @@ such as ``!_UNICODE``) and the parameters to that type. Since the cast value must populate a string typed column, it had to be a Python string (such as being cast to the native type string) and could not have been a special Structure such as ``_UNICODE``. For the format hint columns, the format hint type must be used to ensure the error checking does not fail. - - diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index 4acf35f98..144cae644 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -3,7 +3,7 @@ Using Volatility 3 as a Library This portion of the documentation discusses how to access the Volatility 3 framework from an external application. -The general process of using volatility as a library is to as follows: +The general process of using volatility as a library is as follows: 1. :ref:`create_context` 2. (Optional) :ref:`available_plugins` @@ -21,7 +21,7 @@ Creating a context First we make sure the volatility framework works the way we expect it (and is the version we expect). The versioning used is semantic versioning, meaning any version with the same major number and a higher or equal minor number will satisfy the requirement. An example is below since the CLI doesn't need any of the features -from versions 1.1 or 1.2: +from version 1.1 or later: :: @@ -86,7 +86,7 @@ List requirements are a list of simple types (integers, booleans, floats and str options, multiple requirements needs all their subrequirements fulfilled and the other types require the names of valid translation layers or symbol tables within the context, respectively. Luckily, each of these requirements can tell you whether they've been fulfilled or not later in the process. For now, they can be used to ask the user to -fill in any parameters they made need to. Some requirements are optional, others are not. +fill in any parameters they may need to. Some requirements are optional, others are not. The plugin is essentially a multiple requirement. It should also be noted that automagic classes can have requirements (as can translation layers). @@ -100,7 +100,7 @@ Once you know what requirements the plugin will need, you can populate them with The configuration is essentially a hierarchical tree of values, much like the windows registry. Each plugin is instantiated at a particular branch within the hierarchy and will look for its configuration options under that hierarchy (if it holds any configurable items, it will likely instantiate those at a point -underneaths its own branch). To set the hierarchy, you'll need to know where the configurables will be constructed. +underneath its own branch). To set the hierarchy, you'll need to know where the configurables will be constructed. For this example, we'll assume plugins' base_config_path is set as `plugins`, and that automagics are configured under the `automagic` tree. We'll see later how to ensure this matches up with the plugins and automagic when they're @@ -139,7 +139,7 @@ A suitable list of automagics for a particular plugin (based on operating system This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just the automagics which apply to the operating system. Each automagic can exclude itself from being used for specific -operating systems, so that an automagic designed for linux is not used for windows or mac plugins. +operating systems, such that an automagic designed for linux is not used for windows or mac plugins. These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that the plugin will be constructed on ('plugins' by default) and a progress_callback. This is a callable which takes @@ -157,8 +157,8 @@ Any exceptions that occur during the execution of the automagic will be returned Run the plugin -------------- -Firstly, we should check whether the plugin will be able to run (ie, whether the configuration options it needs -have been successfully set). We do this as follow (where plugin_config_path is the base_config_path (which defaults +Firstly, we should check whether the plugin will be able to run (i.e., whether the configuration options it needs +have been successfully set). We do this as follows, where plugin_config_path is the base_config_path (which defaults to `plugins` and then the name of the class itself): :: @@ -166,7 +166,7 @@ to `plugins` and then the name of the class itself): unsatisfied = plugin.unsatisfied(context, plugin_config_path) If unsatisfied is an empty list, then the plugin has been given everything it requires. If not, it will be a -Dictionary of the hierarchy paths and their associated requirements that weren't satisfied. +dict of the hierarchy paths and their associated requirements that weren't satisfied. The plugin can then be instantiated with the context (containing the plugin's configuration) and the path that the plugin can find its configuration at. This configuration path only needs to be a unique value to identify where the diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 3c4f4ce5d..47ea2e905 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -36,7 +36,7 @@ operating system mode for volshell, and the current layer available for use. (primary) >>> -Volshell itself in essentially a plugin, but an interactive one. As such, most values are accessed through `self` +Volshell itself is essentially a plugin, but an interactive one. As such, most values are accessed through `self` although there is also a `context` object whenever a context must be provided. The prompt for the tool will indicate the name of the current layer (which can be accessed as `self.current_layer` @@ -92,7 +92,7 @@ It can also be provided with an object and will interpret the data for each in t 0x2e8 : UniqueProcessId symbol_table_name1!pointer 4 ... -These values can be accessed directory as attributes +These values can be accessed directly as attributes :: @@ -180,15 +180,68 @@ used: layer = cc(mynewlayer.MyNewLayer, on_top_of = 'primary', other_parameter = 'important') with open('output.dmp', 'wb') as fp: - for i in range(0, 1073741824, 0x1000): + for i in range(0, 0x4000000, 0x1000): data = layer.read(i, 0x1000, pad = True) fp.write(data) As this demonstrates, all of the python is accessible, as are the volshell built in functions (such as `cc` which creates a constructable, like a layer or a symbol table). +User Convenience +---------------- + +There are functions available that make often-done tasks easier, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is advertised when volshell starts. + Loading files -------------- +^^^^^^^^^^^^^ Files can be loaded as physical layers using the `load_file` or `lf` command, which takes a filename or a URI. This will be added to `context.layers` and can be accessed by the name returned by `lf`. + +Regex +^^^^^ + +It is easy to scan for some bytes or a pattern using `regex_scan` or `rx`. + +:: + + (layer_name) >>> rx(rb"(Linux version|Darwin Kernel Version) [0-9]+\.[0-9]+\.[0-9]+") + 0x880001400070 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0x880001400080 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0x880001400090 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0x8800014000a0 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0x8800014000b0 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0x8800014000c0 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0x8800014000d0 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0x8800014000e0 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0x880001769027 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0x880001769037 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0x880001769047 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0x880001769057 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0x880001769067 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0x880001769077 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0x880001769087 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0x880001769097 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0xffff81400070 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0xffff81400080 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0xffff81400090 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0xffff814000a0 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0xffff814000b0 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0xffff814000c0 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0xffff814000d0 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0xffff814000e0 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0xffff81769027 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0xffff81769037 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0xffff81769047 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0xffff81769057 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0xffff81769067 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0xffff81769077 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0xffff81769087 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0xffff81769097 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + +An optional size can be given for the displayed results as with the other fuctions (db, dw, dd, dq, etc). + +You can, of course, specify a different layer name as well. diff --git a/pyproject.toml b/pyproject.toml index 9b4b8d485..86e3921d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,10 @@ full = [ "capstone>=5.0.3,<6", "pycryptodome>=3.21.0,<4", "leechcorepyc>=2.19.2,<3; sys_platform != 'darwin'", + # https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst + # 10.0.0 dropped support for Python3.7 + # 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 + "pillow>=10.0.0,<11.0.0", ] cloud = [ @@ -32,6 +36,7 @@ dev = [ "jsonschema>=4.23.0,<5", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", + "types-jsonschema>=4.23.0,<5", ] test = [ @@ -68,8 +73,23 @@ include = ["volatility3*"] mypy_path = "./stubs" show_traceback = true -[tool.mypy.overrides] -ignore_missing_imports = true +[tool.ruff] +line-length = 88 +target-version = "py38" + +[tool.ruff.lint] +select = [ + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade +] + +ignore = [ + "E501", # ignore due to conflict with formatter +] [build-system] requires = ["setuptools>=68"] diff --git a/test/plugins/windows/test_scheduled_tasks.py b/test/plugins/windows/test_scheduled_tasks.py index 8f771b323..15d7f79a6 100644 --- a/test/plugins/windows/test_scheduled_tasks.py +++ b/test/plugins/windows/test_scheduled_tasks.py @@ -2,9 +2,11 @@ import sys import struct import traceback import unittest + sys.path.insert(0, "../../volatility3") from volatility3.plugins.windows import scheduled_tasks + class TestActionsDecoding(unittest.TestCase): def test_decode_exe_action(self): # fmt: off @@ -84,8 +86,7 @@ class TestActionsDecoding(unittest.TestCase): self.assertEqual(actions[0].action_type, scheduled_tasks.ActionType.Exe) except Exception: self.fail( - "ActionDecoder.decode should not raise exception:\n%s" - % traceback.format_exc() + f"ActionDecoder.decode should not raise exception:\n{traceback.format_exc()}" ) diff --git a/test/test_volatility.py b/test/test_volatility.py index b5910e1c8..bb7c9a851 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -39,7 +39,9 @@ def runvol(args, volatility, python): return p.returncode, stdout, stderr -def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): +def runvol_plugin(plugin, img, volatility, python, pluginargs=None, globalargs=None): + pluginargs = pluginargs or [] + globalargs = globalargs or [] args = ( globalargs + [ @@ -54,13 +56,68 @@ def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]) return runvol(args, volatility, python) +def runvolshell(img, volshell, python, volshellargs=None, globalargs=None): + volshellargs = volshellargs or [] + globalargs = globalargs or [] + args = ( + globalargs + + [ + "--single-location", + img, + "-q", + ] + + volshellargs + ) + + return runvol(args, volshell, python) + + # # TESTS # + +def basic_volshell_test(image, volatility, python, globalargs): + # Basic VolShell test to verify requirements and ensure VolShell runs without crashing + + volshell_commands = [ + "print(ps())", + "exit()", + ] + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".txt") + try: + volshell_script = "\n".join(volshell_commands) + with os.fdopen(fd, "w") as f: + f.write(volshell_script) + + rc, out, _err = runvolshell( + img=image, + volshell=volatility, + python=python, + volshellargs=["--script", filename], + globalargs=globalargs, + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + assert rc == 0 + assert out.count(b"\n") >= 4 + + return out + + # WINDOWS +def test_windows_volshell(image, volatility, python): + out = basic_volshell_test(image, volatility, python, globalargs=["-w"]) + assert out.count(b" 40 + + def test_windows_pslist(image, volatility, python): rc, out, _err = runvol_plugin("windows.pslist.PsList", image, volatility, python) out = out.lower() @@ -332,86 +389,91 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): # LINUX +def test_linux_volshell(image, volatility, python): + out = basic_volshell_test(image, volatility, python, globalargs=["-l"]) + assert out.count(b" 100 + + def test_linux_pslist(image, volatility, python): rc, out, _err = runvol_plugin("linux.pslist.PsList", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.find(b"watchdog") != -1 assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_check_idt(image, volatility, python): rc, out, _err = runvol_plugin( "linux.check_idt.Check_idt", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.count(b"__kernel__") >= 10 assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_check_syscall(image, volatility, python): rc, out, _err = runvol_plugin( "linux.check_syscall.Check_syscall", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.find(b"sys_close") != -1 assert out.find(b"sys_open") != -1 assert out.count(b"\n") > 100 - assert rc == 0 def test_linux_lsmod(image, volatility, python): rc, out, _err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + out = out.lower() + assert out.count(b"\n") > 10 def test_linux_lsof(image, volatility, python): rc, out, _err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.count(b"socket:") >= 10 assert out.count(b"\n") > 35 - assert rc == 0 def test_linux_proc_maps(image, volatility, python): rc, out, _err = runvol_plugin("linux.proc.Maps", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.count(b"anonymous mapping") >= 10 assert out.count(b"\n") > 100 - assert rc == 0 def test_linux_tty_check(image, volatility, python): rc, out, _err = runvol_plugin( "linux.tty_check.tty_check", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.find(b"__kernel__") != -1 assert out.count(b"\n") >= 5 - assert rc == 0 def test_linux_sockstat(image, volatility, python): rc, out, _err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) + assert rc == 0 assert out.count(b"AF_UNIX") >= 354 assert out.count(b"AF_BLUETOOTH") >= 5 assert out.count(b"AF_INET") >= 32 assert out.count(b"AF_INET6") >= 20 assert out.count(b"AF_PACKET") >= 1 assert out.count(b"AF_NETLINK") >= 43 - assert rc == 0 def test_linux_library_list(image, volatility, python): @@ -423,49 +485,48 @@ def test_linux_library_list(image, volatility, python): pluginargs=["--pids", "2363"], ) + assert rc == 0 assert re.search( rb"NetworkManager\s2363\s0x7f52cdda0000\s/lib/x86_64-linux-gnu/libnss_files.so.2", out, ) assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_pstree(image, volatility, python): rc, out, _err = runvol_plugin("linux.pstree.PsTree", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_pidhashtable(image, volatility, python): rc, out, _err = runvol_plugin( "linux.pidhashtable.PIDHashTable", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_bash(image, volatility, python): rc, out, _err = runvol_plugin("linux.bash.Bash", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_boottime(image, volatility, python): rc, out, _err = runvol_plugin("linux.boottime.Boottime", image, volatility, python) - out = out.lower() - assert out.count(b"utc") >= 1 assert rc == 0 + out = out.lower() + assert out.count(b"utc") >= 1 def test_linux_capabilities(image, volatility, python): @@ -482,36 +543,33 @@ def test_linux_capabilities(image, volatility, python): # However, we can still check that the plugin requirements are met. return None - out = out.lower() - - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_check_creds(image, volatility, python): - rc, _out, _err = runvol_plugin( + rc, out, _err = runvol_plugin( "linux.check_creds.Check_creds", image, volatility, python ) # linux-sample-1.bin has no processes sharing credentials. # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_elfs(image, volatility, python): rc, out, _err = runvol_plugin("linux.elfs.Elfs", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_envars(image, volatility, python): rc, out, _err = runvol_plugin("linux.envars.Envars", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_kthreads(image, volatility, python): @@ -528,44 +586,42 @@ def test_linux_kthreads(image, volatility, python): # However, we can still check that the plugin requirements are met. return None - out = out.lower() - - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_malfind(image, volatility, python): - rc, _out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) + rc, out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) # linux-sample-1.bin has no process memory ranges with potential injected code. # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_mountinfo(image, volatility, python): rc, out, _err = runvol_plugin( "linux.mountinfo.MountInfo", image, volatility, python ) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_psaux(image, volatility, python): rc, out, _err = runvol_plugin("linux.psaux.PsAux", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 50 assert rc == 0 + assert out.count(b"\n") > 50 def test_linux_ptrace(image, volatility, python): - rc, _out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) + rc, out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) - # linux-sample-1.bin has no processes being ptreaced. + # linux-sample-1.bin has no processes being ptraced. # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_vmaregexscan(image, volatility, python): @@ -576,10 +632,9 @@ def test_linux_vmaregexscan(image, volatility, python): python, pluginargs=["--pid", "1", "--pattern", "\\x7fELF"], ) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_vmayarascan_yara_rule(image, volatility, python): @@ -613,9 +668,8 @@ def test_linux_vmayarascan_yara_rule(image, volatility, python): with contextlib.suppress(FileNotFoundError): os.remove(filename) - out = out.lower() - assert out.count(b"\n") > 4 assert rc == 0 + assert out.count(b"\n") > 4 def test_linux_vmayarascan_yara_string(image, volatility, python): @@ -626,10 +680,9 @@ def test_linux_vmayarascan_yara_string(image, volatility, python): python, pluginargs=["--pid", "1", "--yara-string", "ELF"], ) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_page_cache_files(image, volatility, python): @@ -640,8 +693,8 @@ def test_linux_page_cache_files(image, volatility, python): python, pluginargs=["--find", "/etc/passwd"], ) - out = out.lower() + assert rc == 0 assert out.count(b"\n") > 4 # inode_num inode_addr ... file_path @@ -649,12 +702,140 @@ def test_linux_page_cache_files(image, volatility, python): rb"146829\s0x88001ab5c270.*?/etc/passwd", out, ) + + +def test_linux_page_cache_inodepages(image, volatility, python): + + inode_address = hex(0x88001AB5C270) + inode_dump_filename = f"inode_{inode_address}.dmp" + try: + rc, out, _err = runvol_plugin( + "linux.pagecache.InodePages", + image, + volatility, + python, + pluginargs=["--inode", inode_address, "--dump"], + ) + + assert rc == 0 + assert out.count(b"\n") > 4 + + # PageVAddr PagePAddr MappingAddr .. DumpSafe + assert re.search( + rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", + out, + ) + assert os.path.exists(inode_dump_filename) + with open(inode_dump_filename, "rb") as fp: + inode_contents = fp.read() + assert inode_contents.count(b"\n") > 30 + assert inode_contents.count(b"root:x:0:0:root:/root:/bin/bash") > 0 + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(inode_dump_filename) + + +def test_linux_check_afinfo(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_afinfo.Check_afinfo", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results. + # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_check_modules(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_modules.Check_modules", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_ebpf_progs(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.ebpf.EBPF", + image, + volatility, + python, + globalargs=["-vvv"], + ) + + if rc != 0 and err.count(b"Unsupported kernel") > 0: + # The linux-sample-1.bin kernel implementation isn't supported. + # However, we can still check that the plugin requirements are met. + return None + + assert rc == 0 + assert out.count(b"\n") > 4 + + +def test_linux_iomem(image, volatility, python): + rc, out, _err = runvol_plugin("linux.iomem.IOMem", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_keyboard_notifiers(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.keyboard_notifiers.Keyboard_notifiers", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results for this plugin. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_kmesg(image, volatility, python): + rc, out, _err = runvol_plugin("linux.kmsg.Kmsg", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_netfilter(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.netfilter.Netfilter", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results for this plugin. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_psscan(image, volatility, python): + rc, out, _err = runvol_plugin("linux.psscan.PsScan", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_hidden_modules(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.hidden_modules.Hidden_modules", image, volatility, python + ) + + # linux-sample-1.bin has no hidden modules. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 # MAC +def test_mac_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python, globalargs=["-m"]) + + def test_mac_pslist(image, volatility, python): rc, out, _err = runvol_plugin("mac.pslist.PsList", image, volatility, python) out = out.lower() diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 901f299a8..6172a17f3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,7 @@ import os import sys import tempfile import traceback -from typing import Any, Dict, List, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Tuple, Type, Union from urllib import parse, request try: @@ -57,14 +57,14 @@ formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) -class PrintedProgress(object): +class PrintedProgress: """A progress handler that prints the progress value and the description onto the command line.""" def __init__(self): self._max_message_len = 0 - def __call__(self, progress: Union[int, float], description: str = None): + def __call__(self, progress: Union[int, float], description: Optional[str] = None): """A simple function for providing text-based feedback. .. warning:: Only for development use. @@ -81,7 +81,7 @@ class PrintedProgress(object): class MuteProgress(PrintedProgress): """A dummy progress handler that produces no output when called.""" - def __call__(self, progress: Union[int, float], description: str = None): + def __call__(self, progress: Union[int, float], description: Optional[str] = None): pass @@ -126,9 +126,7 @@ class CommandLine: "--help", action="help", default=argparse.SUPPRESS, - help="Show this help message and exit, for specific plugin options use '{} --help'".format( - parser.prog - ), + help=f"Show this help message and exit, for specific plugin options use '{parser.prog} --help'", ) parser.add_argument( "-c", @@ -360,9 +358,7 @@ class CommandLine: subparser = parser.add_subparsers( title="Plugins", dest="plugin", - description="For plugin specific options, run '{} --help'".format( - self.CLI_NAME - ), + description=f"For plugin specific options, run '{self.CLI_NAME} --help'", action=volargparse.HelpfulSubparserAction, metavar="PLUGIN", ) @@ -416,7 +412,7 @@ class CommandLine: # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: - with open(args.config, "r") as f: + with open(args.config) as f: json_val = json.load(f) ctx.config.splice( plugin_config_path, @@ -722,9 +718,7 @@ class CommandLine: if isinstance(requirement, requirements.ListRequirement): if not isinstance(value, list): raise TypeError( - "Configuration for ListRequirement was not a list: {}".format( - requirement.name - ) + f"Configuration for ListRequirement was not a list: {requirement.name}" ) value = [requirement.element_type(x) for x in value] if not inspect.isclass(configurables_list[configurable]): @@ -797,7 +791,7 @@ class CommandLine: fd, self._name = tempfile.mkstemp( suffix=".vol3", prefix="tmp_", dir=output_dir ) - self._file = io.open(fd, mode="w+b") + self._file = open(fd, mode="w+b") CLIFileHandler.__init__(self, filename) for item in dir(self._file): if not item.startswith("_") and item not in ( @@ -870,9 +864,7 @@ class CommandLine: requirement, interfaces.configuration.RequirementInterface ): raise TypeError( - "Plugin contains requirements that are not RequirementInterfaces: {}".format( - configurable.__name__ - ) + f"Plugin contains requirements that are not RequirementInterfaces: {configurable.__name__}" ) if isinstance(requirement, interfaces.configuration.SimpleTypeRequirement): additional["type"] = requirement.instance_type @@ -887,7 +879,7 @@ class CommandLine: volatility3.framework.configuration.requirements.ListRequirement, ): # Allow a list of integers, specified with the convenient 0x hexadecimal format - if requirement.element_type == int: + if requirement.element_type is int: additional["type"] = lambda x: int(x, 0) else: additional["type"] = requirement.element_type diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 3d69934e9..b6f019da9 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -1,7 +1,8 @@ import logging -from typing import Any, List, Optional -from volatility3.framework import constants, interfaces import re +from typing import Any, List, Optional + +from volatility3.framework import constants, interfaces vollog = logging.getLogger(__name__) @@ -67,16 +68,16 @@ class ColumnFilter: ) -> None: self.column_num = column_num self.pattern = pattern - self.exclude = exclude self.regex = regex + self.exclude = exclude def find(self, item) -> bool: """Identifies whether an item is found in the appropriate column""" try: if self.regex: - return re.search(self.pattern, f"{item}") + return bool(re.search(self.pattern, f"{item}")) return self.pattern in f"{item}" - except IOError: + except OSError: return False def found(self, row: List[Any]) -> bool: diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 408a562d8..b1944ae5a 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -176,7 +176,7 @@ class QuickTextRenderer(CLIRenderer): format_hints.HexBytes: optional(hex_bytes_as_text), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), interfaces.renderers.Disassembly: optional(display_disassembly), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), "default": optional(lambda x: f"{x}"), } @@ -256,7 +256,7 @@ class CSVRenderer(CLIRenderer): format_hints.HexBytes: optional(hex_bytes_as_text), format_hints.MultiTypeData: optional(multitypedata_as_text), interfaces.renderers.Disassembly: optional(display_disassembly), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), "default": optional(lambda x: f"{x}"), } @@ -450,7 +450,7 @@ class JsonRenderer(CLIRenderer): format_hints.HexBytes: quoted_optional(hex_bytes_as_text), interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: lambda x: ( x.isoformat() if not isinstance(x, interfaces.renderers.BaseAbsentValue) @@ -467,7 +467,7 @@ class JsonRenderer(CLIRenderer): def output_result(self, outfd, result): """Outputs the JSON data to a file in a particular format""" - outfd.write("{}\n".format(json.dumps(result, indent=2, sort_keys=True))) + outfd.write(f"{json.dumps(result, indent=2, sort_keys=True)}\n") def render(self, grid: interfaces.renderers.TreeGrid): outfd = sys.stdout diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index fd61ddce0..dce9cafa6 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -5,7 +5,7 @@ import argparse import gettext import re -from typing import List, Optional, Sequence, Any, Union +from typing import Optional, Sequence, Any, Union # This effectively overrides/monkeypatches the core argparse module to provide more helpful output around choices diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index e9d3fda08..0affe5d59 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -282,9 +282,7 @@ class VolShell(cli.CommandLine): for plugin in volshell_plugin_list: subparser = parser.add_argument_group( title=plugin.capitalize(), - description="Configuration options based on {} options".format( - plugin.capitalize() - ), + description=f"Configuration options based on {plugin.capitalize()} options", ) self.populate_requirements_argparse(subparser, volshell_plugin_list[plugin]) configurables_list[plugin] = volshell_plugin_list[plugin] @@ -331,7 +329,7 @@ class VolShell(cli.CommandLine): # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: - with open(args.config, "r") as f: + with open(args.config) as f: json_val = json.load(f) ctx.config.splice( plugin_config_path, diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 82c470e1a..2321408fe 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -203,7 +203,7 @@ class Volshell(interfaces.plugins.PluginInterface): connector = " " if chunk_size < 2: connector = "" - ascii_data = connector.join([self._ascii_bytes(x) for x in valid_data]) + ascii_data = connector.join(self._ascii_bytes(x) for x in valid_data) print(hex(offset), " ", hex_data, " ", ascii_data) offset += 16 @@ -240,7 +240,7 @@ class Volshell(interfaces.plugins.PluginInterface): return None return self.context.modules[self.current_kernel_name] - def change_layer(self, layer_name: str = None): + def change_layer(self, layer_name: Optional[str] = None): """Changes the current default layer""" if not layer_name: layer_name = self.current_layer @@ -250,7 +250,7 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_layer = layer_name sys.ps1 = f"({self.current_layer}) >>> " - def change_symbol_table(self, symbol_table_name: str = None): + def change_symbol_table(self, symbol_table_name: Optional[str] = None): """Changes the current_symbol_table""" if not symbol_table_name: print("No symbol table provided, not changing current symbol table") @@ -262,7 +262,7 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_symbol_table = symbol_table_name print(f"Current Symbol Table: {self.current_symbol_table}") - def change_kernel(self, kernel_name: str = None): + def change_kernel(self, kernel_name: Optional[str] = None): if not kernel_name: print("No kernel module name provided, not changing current kernel") if kernel_name not in self.context.modules: @@ -347,7 +347,7 @@ class Volshell(interfaces.plugins.PluginInterface): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if not isinstance( @@ -479,7 +479,7 @@ class Volshell(interfaces.plugins.PluginInterface): if treegrid is not None: self.render_treegrid(treegrid) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: print("No symbol table provided") @@ -553,17 +553,16 @@ class Volshell(interfaces.plugins.PluginInterface): if argname in kwargs: del kwargs[argname] - for keyword in kwargs: - val = kwargs[keyword] - if not isinstance( - val, interfaces.configuration.BasicTypes - ) and not isinstance(val, list): - if not isinstance(val, list) or all( - isinstance(x, interfaces.configuration.BasicTypes) for x in val - ): - raise TypeError( - "Configurable values must be simple types (int, bool, str, bytes)" - ) + for keyword, val in kwargs.items(): + BasicType_or_list_of_BasicType = False # excludes list of lists + if isinstance(val, interfaces.configuration.BasicTypes): + BasicType_or_list_of_BasicType = True + if all(isinstance(x, interfaces.configuration.BasicTypes) for x in val): + BasicType_or_list_of_BasicType = True + if not BasicType_or_list_of_BasicType: + raise TypeError( + "Configurable values must be simple types (int, bool, str, bytes)" + ) self.context.config[config_path + "." + keyword] = val constructed = clazz(self.context, config_path, **constructor_args) @@ -585,7 +584,6 @@ class NullFileHandler(io.BytesIO, interfaces.plugins.FileHandlerInterface): def writelines(self, lines: Iterable[bytes]): """Dummy method""" - pass def write(self, b: bytes): """Dummy method""" diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index c5e555ec7..cc58fa1c2 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -20,7 +20,7 @@ class Volshell(generic.Volshell): name="kernel", description="Linux kernel module" ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.IntRequirement( name="pid", description="Process ID", optional=True @@ -61,7 +61,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -69,7 +69,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 2b32ad677..0ed35eb27 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -63,7 +63,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -71,7 +71,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 5c2190c02..303d4d5c3 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -60,7 +60,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -68,7 +68,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 23ea745de..a1925faef 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -6,28 +6,12 @@ import glob import sys import zipfile - -required_python_version = (3, 8, 0) -if ( - sys.version_info.major != required_python_version[0] - or sys.version_info.minor < required_python_version[1] - or ( - sys.version_info.minor == required_python_version[1] - and sys.version_info.micro < required_python_version[2] - ) -): - raise RuntimeError( - "Volatility framework requires python version {}.{}.{} or greater".format( - *required_python_version - ) - ) - import importlib import inspect import logging import os import traceback -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar +from typing import Any, Dict, Generator, List, Optional, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces @@ -56,27 +40,25 @@ def require_interface_version(*args) -> None: if len(args): if args[0] != interface_version()[0]: raise RuntimeError( - "Framework interface version {} is incompatible with required version {}".format( - interface_version()[0], args[0] - ) + f"Framework interface version {interface_version()[0]} is incompatible with required version {args[0]}" ) if len(args) > 1: if args[1] > interface_version()[1]: raise RuntimeError( "Framework interface version {} is an older revision than the required version {}".format( - ".".join([str(x) for x in interface_version()[0:2]]), - ".".join([str(x) for x in args[0:2]]), + ".".join(str(x) for x in interface_version()[0:2]), + ".".join(str(x) for x in args[0:2]), ) ) -class NonInheritable(object): +class NonInheritable: def __init__(self, value: Any, cls: Type) -> None: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Type = None) -> Any: - if type == self.cls: + def __get__(self, obj: Any, get_type: Optional[Type] = None) -> Any: + if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) return self.default_value @@ -99,8 +81,7 @@ def class_subclasses(cls: Type[T]) -> Generator[Type[T], None, None]: # The typing system is not clever enough to realize that clazz has a hidden attr after the hasattr check if not hasattr(clazz, "hidden") or not clazz.hidden: # type: ignore yield clazz - for return_value in class_subclasses(clazz): - yield return_value + yield from class_subclasses(clazz) def import_files(base_module, ignore_errors: bool = False) -> List[str]: @@ -161,9 +142,7 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]: def _filter_files(filename: str): """Ensures that a filename traversed is an importable python file""" - return ( - filename.endswith(".py") or filename.endswith(".pyc") - ) and not filename.startswith("__") + return (filename.endswith((".py", ".pyc"))) and not filename.startswith("__") def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str]: @@ -187,9 +166,7 @@ def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str traceback.TracebackException.from_exception(e).format(chain=True) ) ) - vollog.debug( - "Failed to import module {} based on file: {}".format(module, path) - ) + vollog.debug(f"Failed to import module {module} based on file: {path}") failures.append(module) if not ignore_errors: raise @@ -207,8 +184,7 @@ def _zipwalk(path: str): zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( dirlist ) - for value in zip_results: - yield value, zip_results[value] + yield from zip_results.items() def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index ef54a0aa5..f22cae012 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,12 +3,10 @@ # import logging -import os -from typing import Optional, Tuple, Type +from typing import Optional, Tuple from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder -from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import linux @@ -173,9 +171,7 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if aslr_shift & 0xFFF != 0 or kaslr_shift & 0xFFF != 0: continue vollog.debug( - "Linux ASLR shift values determined: physical {:0x} virtual {:0x}".format( - kaslr_shift, aslr_shift - ) + f"Linux ASLR shift values determined: physical {kaslr_shift:0x} virtual {aslr_shift:0x}" ) return kaslr_shift, aslr_shift @@ -198,5 +194,8 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder): banner_config_key = "kernel_banner" operating_system = "linux" symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" - find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] exclusion_list = ["mac", "windows"] + + @classmethod + def find_aslr(cls, *args): + return LinuxIntelStacker.find_aslr(*args)[1] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 7c478b521..f3679d160 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -3,13 +3,11 @@ # import logging -import os import struct from typing import Optional from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.automagic import symbol_cache, symbol_finder -from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import mac @@ -184,7 +182,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = [int(x) for x in banner[22:].split(b".")[0:2]] + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) tmp_aslr_shift = offset - cls.virtual_to_physical_address( version_json_address diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 0b4f6c73a..729c48063 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -215,9 +215,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): return (virtual_layer_name, kvo, kernel) else: vollog.debug( - "Potential kernel_virtual_offset did not map to expected location: {}".format( - hex(kvo) - ) + f"Potential kernel_virtual_offset did not map to expected location: {hex(kvo)}" ) except exceptions.InvalidAddressException: vollog.debug( diff --git a/volatility3/framework/automagic/stacker.py b/volatility3/framework/automagic/stacker.py index c251d3c46..596864264 100644 --- a/volatility3/framework/automagic/stacker.py +++ b/volatility3/framework/automagic/stacker.py @@ -166,7 +166,9 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): cls, context: interfaces.context.ContextInterface, initial_layer: str, - stack_set: List[Type[interfaces.automagic.StackerLayerInterface]] = None, + stack_set: Optional[ + List[Type[interfaces.automagic.StackerLayerInterface]] + ] = None, progress_callback: constants.ProgressCallback = None, ): """Stacks as many possible layers on top of the initial layer as can be done. diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index e38771f79..065eb6d43 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -104,10 +104,11 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): for subclazz in framework.class_subclasses(IdentifierProcessor): self._classifiers[subclazz.operating_system] = subclazz + @abstractmethod def add_identifier(self, location: str, operating_system: str, identifier: str): """Adds an identifier to the store""" - pass + @abstractmethod def find_location( self, identifier: bytes, operating_system: Optional[str] ) -> Optional[str]: @@ -120,19 +121,19 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: The location of the symbols file that matches the identifier """ - pass + @abstractmethod def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" - pass + @abstractmethod def update(self): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. """ - pass + @abstractmethod def get_identifier_dictionary( self, operating_system: Optional[str] = None, local_only: bool = False ) -> Dict[bytes, str]: @@ -145,16 +146,16 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A dictionary of identifiers mapped to a location """ - pass + @abstractmethod def get_identifier(self, location: str) -> Optional[bytes]: """Returns an identifier based on a specific location or None""" - pass + @abstractmethod def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" - pass + @abstractmethod def get_location_statistics( self, location: str ) -> Optional[Tuple[int, int, int, int]]: @@ -164,6 +165,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): A tuple of base_types, types, enums, symbols, or None is location not found """ + @abstractmethod def get_hash(self, location: str) -> Optional[str]: """Returns the hash of the JSON from within a location ISF""" @@ -572,6 +574,6 @@ class RemoteIdentifierFormat: try: subrbf = RemoteIdentifierFormat(location) yield from subrbf.process(identifiers, operating_system) - except IOError: + except OSError: vollog.debug(f"Remote file not found: {location}") return identifiers diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 6d689e194..1d30f3f51 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -4,7 +4,7 @@ import logging import os -from typing import Any, Callable, Iterable, List, Optional, Tuple +from typing import Callable, List, Optional, Tuple from volatility3.framework import constants, interfaces, layers from volatility3.framework.automagic import symbol_cache diff --git a/volatility3/framework/check_python_version.py b/volatility3/framework/check_python_version.py new file mode 100644 index 000000000..f2d284f2a --- /dev/null +++ b/volatility3/framework/check_python_version.py @@ -0,0 +1,14 @@ +import sys + +required_python_version = (3, 8, 0) +if ( + sys.version_info.major != required_python_version[0] + or sys.version_info.minor < required_python_version[1] + or ( + sys.version_info.minor == required_python_version[1] + and sys.version_info.micro < required_python_version[2] + ) +): + raise RuntimeError( + f"Volatility framework requires python version {required_python_version[0]}.{required_python_version[1]}.{required_python_version[2]} or greater" + ) diff --git a/volatility3/framework/configuration/__init__.py b/volatility3/framework/configuration/__init__.py index 7a84ee455..7b914cf16 100644 --- a/volatility3/framework/configuration/__init__.py +++ b/volatility3/framework/configuration/__init__.py @@ -2,4 +2,4 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework.configuration import requirements +from volatility3.framework.configuration import requirements as requirements diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index f130f9544..3e3608000 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -11,7 +11,7 @@ expect to be in the context (such as particular layers or symboltables). import abc import logging import os -from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type +from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Type from urllib import parse, request from volatility3.framework import constants, interfaces @@ -111,7 +111,7 @@ class ListRequirement(interfaces.configuration.RequirementInterface): Args: element_type: The (requirement) type of each element within the list - max_elements; The maximum number of acceptable elements this list can contain + max_elements: The maximum number of acceptable elements this list can contain min_elements: The minimum number of acceptable elements this list can contain """ super().__init__(*args, **kwargs) @@ -314,11 +314,11 @@ class TranslationLayerRequirement( def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: interfaces.configuration.ConfigSimpleType = None, optional: bool = False, - oses: List = None, - architectures: List = None, + oses: Optional[List] = None, + architectures: Optional[List[str]] = None, ) -> None: """Constructs a Translation Layer Requirement. @@ -526,18 +526,18 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): description: Optional[str] = None, default: bool = False, optional: bool = False, - component: Type[interfaces.configuration.VersionableInterface] = None, + component: Optional[Type[interfaces.configuration.VersionableInterface]] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: if version is None: raise TypeError("Version cannot be None") + if component is None: + raise TypeError("Component cannot be None") if description is None: - description = f"Version {'.'.join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" + description = f"Version {'.'.join(str(x) for x in version)} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) - if component is None: - raise TypeError("Component cannot be None") self._component: Type[interfaces.configuration.VersionableInterface] = component self._version = version @@ -546,7 +546,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): context: interfaces.context.ContextInterface, config_path: str, accumulator: Optional[ - List[interfaces.configuration.VersionableInterface] + Set[interfaces.configuration.VersionableInterface] ] = None, ) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type @@ -580,7 +580,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): ) if result: - result.update({config_path: self}) + result[config_path] = self return result context.config[interfaces.configuration.path_join(config_path, self.name)] = ( @@ -604,10 +604,10 @@ class PluginRequirement(VersionRequirement): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, optional: bool = False, - plugin: Type[interfaces.plugins.PluginInterface] = None, + plugin: Optional[Type[interfaces.plugins.PluginInterface]] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: super().__init__( @@ -627,7 +627,7 @@ class ModuleRequirement( def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, architectures: Optional[List[str]] = None, optional: bool = False, @@ -664,9 +664,7 @@ class ModuleRequirement( if value is not None: vollog.log( constants.LOGLEVEL_V, - "TypeError - Module Requirement only accepts string labels: {}".format( - repr(value) - ), + f"TypeError - Module Requirement only accepts string labels: {repr(value)}", ) return {config_path: self} diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 8bdf84730..23cc2dde5 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -13,14 +13,14 @@ import sys import warnings from typing import Callable, Optional -import volatility3.framework.constants.linux -import volatility3.framework.constants.windows +from volatility3.framework.constants import linux as linux +from volatility3.framework.constants import windows as windows from volatility3.framework.constants._version import ( - PACKAGE_VERSION, - VERSION_MAJOR, - VERSION_MINOR, - VERSION_PATCH, - VERSION_SUFFIX, + PACKAGE_VERSION as PACKAGE_VERSION, + VERSION_MAJOR as VERSION_MAJOR, + VERSION_MINOR as VERSION_MINOR, + VERSION_PATCH as VERSION_PATCH, + VERSION_SUFFIX as VERSION_SUFFIX, ) PLUGINS_PATH = [ diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 2ea034176..02402c5c9 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,11 +1,11 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 12 # Number of changes that only add to the interface +VERSION_MINOR = 15 # Number of changes that only add to the interface VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( - ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + ".".join(str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]) + VERSION_SUFFIX ) """The canonical version of the volatility3 package""" diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 1a55656b3..a9ec4ac69 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -229,7 +229,7 @@ class Module(interfaces.context.ModuleInterface): def object( self, object_type: str, - offset: int = None, + offset: Optional[int] = None, native_layer_name: Optional[str] = None, absolute: bool = False, **kwargs, @@ -356,7 +356,7 @@ class SizedModule(Module): return size or 0 @property # type: ignore # FIXME: mypy #5107 - @functools.lru_cache() + @functools.lru_cache def hash(self) -> str: """Hashes the module for equality checks. diff --git a/volatility3/framework/interfaces/__init__.py b/volatility3/framework/interfaces/__init__.py index 51d81d63a..fd6b1e062 100644 --- a/volatility3/framework/interfaces/__init__.py +++ b/volatility3/framework/interfaces/__init__.py @@ -13,12 +13,12 @@ components of volatility to write plugins. # This will also avoid namespace issues, because people can use interfaces.layers to # avoid clashing with the layers package from volatility3.framework.interfaces import ( - renderers, - configuration, - context, - layers, - objects, - plugins, - symbols, - automagic, + renderers as renderers, + configuration as configuration, + context as context, + layers as layers, + objects as objects, + plugins as plugins, + symbols as symbols, + automagic as automagic, ) diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index 0867b1608..4ac386fc0 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -42,7 +42,7 @@ class AutomagicInterface( priority = 10 """An ordering to indicate how soon this automagic should be run""" - exclusion_list = [] + exclusion_list: List[str] = [] """A list of plugin categories (typically operating systems) which the plugin will not operate on""" def __init__( diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index da0a4556c..b6f4f889c 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -53,7 +53,7 @@ ConfigSimpleType = Optional[Union[SimpleTypes, List[SimpleTypes]]] def path_join(*args) -> str: """Joins configuration paths together.""" # If a path element (particularly the first) is empty, then remove it from the list - args = tuple([arg for arg in args if arg]) + args = tuple(arg for arg in args if arg) return CONFIG_SEPARATOR.join(args) @@ -82,7 +82,7 @@ class HierarchicalDict(collections.abc.Mapping): def __init__( self, - initial_dict: Dict[str, "SimpleTypeRequirement"] = None, + initial_dict: Optional[Dict[str, "SimpleTypeRequirement"]] = None, separator: str = CONFIG_SEPARATOR, ) -> None: """ @@ -94,7 +94,7 @@ class HierarchicalDict(collections.abc.Mapping): raise TypeError(f"Separator must be a one character string: {separator}") self._separator = separator self._data: Dict[str, ConfigSimpleType] = {} - self._subdict: Dict[str, "HierarchicalDict"] = {} + self._subdict: Dict[str, HierarchicalDict] = {} if isinstance(initial_dict, str): initial_dict = json.loads(initial_dict) if isinstance(initial_dict, dict): @@ -182,9 +182,7 @@ class HierarchicalDict(collections.abc.Mapping): else: if not isinstance(value, HierarchicalDict): raise TypeError( - "HierarchicalDicts can only store HierarchicalDicts within their structure: {}".format( - type(value) - ) + f"HierarchicalDicts can only store HierarchicalDicts within their structure: {type(value)}" ) self._subdict[key] = value @@ -330,7 +328,7 @@ class RequirementInterface(metaclass=ABCMeta): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: ConfigSimpleType = None, optional: bool = False, ) -> None: @@ -498,9 +496,7 @@ class SimpleTypeRequirement(RequirementInterface): if not isinstance(value, self.instance_type): vollog.log( constants.LOGLEVEL_V, - "TypeError - {} requirements only accept {} type: {}".format( - self.name, self.instance_type.__name__, repr(value) - ), + f"TypeError - {self.name} requirements only accept {self.instance_type.__name__} type: {repr(value)}", ) return {config_path: self} return {} @@ -622,7 +618,7 @@ class ConstructableRequirementInterface(RequirementInterface): self, context: "interfaces.context.ContextInterface", config_path: str, - requirement_dict: Dict[str, object] = None, + requirement_dict: Optional[Dict[str, object]] = None, ) -> Optional["interfaces.objects.ObjectInterface"]: """Constructs the class, handing args and the subrequirements as parameters to __init__""" @@ -656,6 +652,7 @@ class ConstructableRequirementInterface(RequirementInterface): class ConfigurableRequirementInterface(RequirementInterface): """Simple Abstract class to provide build_required_config.""" + @abstractmethod def build_configuration( self, context: "interfaces.context.ContextInterface", @@ -775,17 +772,16 @@ class ConfigurableInterface(metaclass=ABCMeta): str: The newly generated full configuration path """ random_config_dict = "".join( - random.SystemRandom().choice(string.ascii_uppercase + string.digits) - for _ in range(8) + random.SystemRandom().choices(string.ascii_uppercase + string.digits, k=8) ) new_config_path = path_join(base_config_path, random_config_dict) # TODO: Check that the new_config_path is empty, although it's not critical if it's not since the values are merged in # This should check that each k corresponds to a requirement and each v is of the appropriate type # This would require knowledge of the new configurable itself to verify, and they should do validation in the - # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a simple type + # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a basic type for k, v in kwargs.items(): - if not isinstance(v, (int, str, bool, float, bytes)): + if not isinstance(v, BasicTypes): raise TypeError( "Config values passed to make_subconfig can only be simple types" ) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index e85429732..30840a5b9 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -85,7 +85,7 @@ class ContextInterface(metaclass=ABCMeta): object_type: Union[str, "interfaces.objects.Template"], layer_name: str, offset: int, - native_layer_name: str = None, + native_layer_name: Optional[str] = None, **arguments, ) -> "interfaces.objects.ObjectInterface": """Object factory, takes a context, symbol, offset and optional @@ -114,6 +114,7 @@ class ContextInterface(metaclass=ABCMeta): """ return copy.deepcopy(self) + @abstractmethod def module( self, module_name: str, @@ -232,7 +233,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): def object( self, object_type: str, - offset: int = None, + offset: Optional[int] = None, native_layer_name: Optional[str] = None, absolute: bool = False, **kwargs, @@ -277,28 +278,36 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol = self.get_symbol(name) return self.offset + symbol.address + @abstractmethod def get_type(self, name: str) -> "interfaces.objects.Template": """Returns a type from the module's symbol table.""" + @abstractmethod def get_symbol(self, name: str) -> "interfaces.symbols.SymbolInterface": """Returns a symbol object from the module's symbol table.""" + @abstractmethod def get_enumeration(self, name: str) -> "interfaces.objects.Template": """Returns an enumeration from the module's symbol table.""" + @abstractmethod def has_type(self, name: str) -> bool: """Determines whether a type is present in the module's symbol table.""" + @abstractmethod def has_symbol(self, name: str) -> bool: """Determines whether a symbol is present in the module's symbol table.""" + @abstractmethod def has_enumeration(self, name: str) -> bool: """Determines whether an enumeration is present in the module's symbol table.""" @property + @abstractmethod def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" + @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: """Returns the symbols within table_name (or this module if not specified) that live at the specified absolute offset provided.""" @@ -344,6 +353,7 @@ class ModuleContainer(collections.abc.Mapping): def __iter__(self): return iter(self._modules) + @abstractmethod def free_module_name(self, prefix: str = "module") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table.""" diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 78687d8d5..a90a78667 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -188,7 +188,6 @@ class DataLayerInterface( the object unreadable (exceptions will be thrown using a DataLayer after destruction) """ - pass @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -211,7 +210,7 @@ class DataLayerInterface( context: interfaces.context.ContextInterface, scanner: ScannerInterface, progress_callback: constants.ProgressCallback = None, - sections: Iterable[Tuple[int, int]] = None, + sections: Optional[Iterable[Tuple[int, int]]] = None, ) -> Iterable[Any]: """Scans a Translation layer by chunk. @@ -361,9 +360,7 @@ class DataLayerInterface( data += self.context.layers[layer_name].read(address, chunk_size) except exceptions.InvalidAddressException: vollog.debug( - "Invalid address in layer {} found scanning {} at address {:x}".format( - layer_name, self.name, address - ) + f"Invalid address in layer {layer_name} found scanning {self.name} at address {address:x}" ) if len(data) > scanner.chunk_size + scanner.overlap: @@ -721,7 +718,7 @@ class LayerContainer(collections.abc.Mapping): raise NotImplementedError("Cycle checking has not yet been implemented") -class DummyProgress(object): +class DummyProgress: """A class to emulate Multiprocessing/threading Value objects.""" def __init__(self) -> None: diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 51d25510d..23c90b13b 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -374,6 +374,7 @@ class Template: f"{self.__class__.__name__} object has no attribute {attr}" ) + @abc.abstractmethod def __call__( self, context: "interfaces.context.ContextInterface", diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 74902636e..f763815a6 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -46,7 +46,7 @@ class FileHandlerInterface(io.RawIOBase): def preferred_filename(self, filename: str): """Sets the preferred filename""" if self.closed: - raise IOError("FileHandler name cannot be changed once closed") + raise OSError("FileHandler name cannot be changed once closed") if not isinstance(filename, str): raise TypeError("FileHandler preferred filenames must be strings") if os.path.sep in filename: diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index b13de1834..e26164ee7 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -26,7 +26,11 @@ from typing import ( Union, ) -Column = NamedTuple("Column", [("name", str), ("type", Any)]) + +class Column(NamedTuple): + name: str + type: Any + RenderOption = Any @@ -98,11 +102,11 @@ class TreeNode(abc.Sequence, metaclass=ABCMeta): """ -class BaseAbsentValue(object): +class BaseAbsentValue: """Class that represents values which are not present for some reason.""" -class Disassembly(object): +class Disassembly: """A class to indicate that the bytes provided should be disassembled (based on the architecture)""" @@ -137,7 +141,7 @@ ColumnsType = List[Tuple[str, BaseTypes]] VisitorSignature = Callable[[TreeNode, _Type], _Type] -class TreeGrid(object, metaclass=ABCMeta): +class TreeGrid(metaclass=ABCMeta): """Class providing the interface for a TreeGrid (which contains TreeNodes) The structure of a TreeGrid is designed to maintain the structure of the tree in a single object. @@ -179,7 +183,7 @@ class TreeGrid(object, metaclass=ABCMeta): @abstractmethod def populate( self, - function: VisitorSignature = None, + function: Optional[VisitorSignature] = None, initial_accumulator: Any = None, fail_on_errors: bool = True, ) -> Optional[Exception]: @@ -231,7 +235,7 @@ class TreeGrid(object, metaclass=ABCMeta): node: Optional[TreeNode], function: VisitorSignature, initial_accumulator: _Type, - sort_key: ColumnSortKey = None, + sort_key: Optional[ColumnSortKey] = None, ) -> None: """Visits all the nodes in a tree, calling function on each one. diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index b645f5cd1..b8712e38d 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -250,13 +250,13 @@ class BaseSymbolTableInterface: def clear_symbol_cache(self) -> None: """Clears the symbol cache of this symbol table.""" - pass class SymbolSpaceInterface(collections.abc.Mapping): """An interface for the container that holds all the symbol-containing tables for use within a context.""" + @abstractmethod def free_table_name(self, prefix: str = "layer") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table.""" @@ -378,7 +378,7 @@ class NativeTableInterface(BaseSymbolTableInterface): return [] -class MetadataInterface(object): +class MetadataInterface: """Interface for accessing metadata stored within a symbol table.""" def __init__(self, json_data: Dict) -> None: diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 3cfc0a25b..a5b25d178 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -1,7 +1,6 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import contextlib import logging import struct from typing import Tuple, Optional @@ -138,7 +137,7 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): ulong_bitmap_array = summary_header.get_buffer_long() # outer_index points to a 32 bits array inside a list of arrays, # each bit indicating a page mapping state - for outer_index in range(0, ulong_bitmap_array.vol.count): + for outer_index in range(ulong_bitmap_array.vol.count): ulong_bitmap = ulong_bitmap_array[outer_index] # All pages in this 32 bits array are mapped (speedup iteration process) if ulong_bitmap == 0xFFFFFFFF: @@ -166,7 +165,7 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): seg_first_bit = None # Some pages in this 32 bits array are mapped and some aren't else: - for inner_bit_position in range(0, 32): + for inner_bit_position in range(32): current_bit = outer_index * 32 + inner_bit_position page_mapped = ulong_bitmap & (1 << inner_bit_position) if page_mapped: @@ -220,9 +219,7 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): for idx, (start_position, mapped_offset, length, _) in enumerate(segments): vollog.log( constants.LOGLEVEL_VVVV, - "Segment {}: Position {:#x} Offset {:#x} Length {:#x}".format( - idx, start_position, mapped_offset, length - ), + f"Segment {idx}: Position {start_position:#x} Offset {mapped_offset:#x} Length {length:#x}", ) self._segments = segments diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 7918ebed4..7c2c72ac1 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -76,13 +76,13 @@ class Intel(linear.LinearlyMappedLayer): self._index_shift = math.ceil(math.log2(struct.calcsize(self._entry_format))) @classproperty - @functools.lru_cache() + @functools.lru_cache def page_shift(cls) -> int: """Page shift for the intel memory layers.""" return cls._page_size_in_bits @classproperty - @functools.lru_cache() + @functools.lru_cache def page_size(cls) -> int: """Page size for the intel memory layers. @@ -91,25 +91,25 @@ class Intel(linear.LinearlyMappedLayer): return 1 << cls._page_size_in_bits @classproperty - @functools.lru_cache() + @functools.lru_cache def page_mask(cls) -> int: """Page mask for the intel memory layers.""" return ~(cls.page_size - 1) @classproperty - @functools.lru_cache() + @functools.lru_cache def bits_per_register(cls) -> int: """Returns the bits_per_register to determine the range of an IntelTranslationLayer.""" return cls._bits_per_register @classproperty - @functools.lru_cache() + @functools.lru_cache def minimum_address(cls) -> int: return 0 @classproperty - @functools.lru_cache() + @functools.lru_cache def maximum_address(cls) -> int: return (1 << cls._maxvirtaddr) - 1 @@ -251,12 +251,7 @@ class Intel(linear.LinearlyMappedLayer): if INTEL_TRANSLATION_DEBUGGING: vollog.log( constants.LOGLEVEL_VVVV, - "Entry {} at index {} gives data {} as {}".format( - hex(entry), - hex(index), - hex(struct.unpack(self._entry_format, entry_data)[0]), - name, - ), + f"Entry {hex(entry)} at index {hex(index)} gives data {hex(struct.unpack(self._entry_format, entry_data)[0])} as {name}", ) # Read out the new entry from memory diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index 542fd6ca2..eeede1673 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -48,7 +48,7 @@ if HAS_LEECHCORE: try: self._handle = leechcorepyc.LeechCore(self._device) except TypeError: - raise IOError(f"Unable to open LeechCore device {self._device}") + raise OSError(f"Unable to open LeechCore device {self._device}") return self._handle def fileno(self): diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 8d84a774b..2b4fae963 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -194,7 +194,7 @@ class PdbMSFStream(linear.LinearlyMappedLayer): ) -> None: super().__init__(context, config_path, name, metadata) self._base_layer = self.config["base_layer"] - self._pages = self.config.get("pages", None) + self._pages = self.config.get("pages", []) self._pages_len = len(self._pages) if not self._pages: raise PDBFormatException(name, "Invalid/no pages specified") @@ -225,7 +225,7 @@ class PdbMSFStream(linear.LinearlyMappedLayer): returned = 0 page_size = self._pdb_layer.page_size while length > 0: - page = math.floor((offset + returned) / page_size) + page = (offset + returned) // page_size page_position = (offset + returned) % page_size chunk_size = min(page_size - page_position, length) if page >= self._pages_len: diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index ff483291c..a8127e954 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -236,7 +236,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if self._architecture is None: vollog.log( constants.LOGLEVEL_VV, - f"QEVM architecture could not be determined", + "QEVM architecture could not be determined", ) # Once all segments have been read, determine the PCI hole if any diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 609832886..c684ccd40 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -140,7 +140,13 @@ class RegistryHive(linear.LinearlyMappedLayer): """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast("string", max_length=2, encoding="latin-1") + try: + signature = cell.cast("string", max_length=2, encoding="latin-1") + except (RegistryInvalidIndex, exceptions.InvalidAddressException): + vollog.debug( + f"Failed to get cell signature for cell (0x{cell.vol.offset:x})" + ) + return cell if signature == "nk": return cell.u.KeyNode elif signature == "sk": @@ -156,9 +162,7 @@ class RegistryHive(linear.LinearlyMappedLayer): else: # It doesn't matter that we use KeyNode, we're just after the first two bytes vollog.debug( - "Unknown Signature {} (0x{:x}) at offset {}".format( - signature, cell.u.KeyNode.Signature, cell_offset - ) + f"Unknown Signature {signature} (0x{cell.u.KeyNode.Signature:x}) at offset {cell_offset}" ) return cell @@ -178,9 +182,7 @@ class RegistryHive(linear.LinearlyMappedLayer): if not root_node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"): raise RegistryFormatException( self.name, - "Encountered {} instead of _CM_KEY_NODE".format( - root_node.vol.type_name - ), + f"Encountered {root_node.vol.type_name} instead of _CM_KEY_NODE", ) node_key = [root_node] if key.endswith("\\"): diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 2dba7caa8..236d256f1 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -29,7 +29,7 @@ except ImportError: try: # Import so that the handler is found by the framework.class_subclasses callc - import smb.SMBHandler # lgtm [py/unused-import] + from smb import SMBHandler as SMBHandler # lgtm [py/unused-import] except ImportError: # If we fail to import this, it means that SMB handling won't be available pass @@ -57,7 +57,7 @@ def cascadeCloseFile(new_fp: IO[bytes], original_fp: IO[bytes]) -> IO[bytes]: return new_fp -class ResourceAccessor(object): +class ResourceAccessor: """Object for opening URLs as files (downloading locally first if necessary)""" diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index dd8dc46be..be9f1c39a 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -5,7 +5,7 @@ import re from typing import Generator, List, Tuple, Dict, Optional from volatility3.framework.interfaces import layers -from volatility3.framework.layers.scanners import multiregexp +from volatility3.framework.layers.scanners import multiregexp as multiregexp class BytesScanner(layers.ScannerInterface): @@ -72,7 +72,7 @@ class MultiStringScanner(layers.ScannerInterface): return None for char in value: - trie[char] = trie.get(char, {}) + trie.setdefault(char, {}) trie = trie[char] # Mark the end of a string diff --git a/volatility3/framework/layers/scanners/multiregexp.py b/volatility3/framework/layers/scanners/multiregexp.py index be3581f05..9831a9d8e 100644 --- a/volatility3/framework/layers/scanners/multiregexp.py +++ b/volatility3/framework/layers/scanners/multiregexp.py @@ -6,7 +6,7 @@ import re from typing import Generator, List, Tuple -class MultiRegexp(object): +class MultiRegexp: """Algorithm for multi-string matching.""" def __init__(self) -> None: diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 622ff0250..39fb21b63 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -57,6 +57,10 @@ class VmwareLayer(segmented.SegmentedLayer): ) meta_layer = self.context.layers.get(self._meta_layer, None) + if meta_layer is None: + raise exceptions.LayerException( + self._meta_layer, "VMware: Meta layer not found" + ) header_size = struct.calcsize(self.header_structure) data = meta_layer.read(0, header_size) magic, unknown, groupCount = struct.unpack(self.header_structure, data) diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index e7aa0ccec..c0a5e1a7d 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -54,6 +54,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): segments = [] self._segment_headers = [] + segment_names = None for sindex in range(ehdr.e_shnum): shdr = self.context.object( diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index b65277067..869d4dae6 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -35,13 +35,13 @@ def convert_data_to_value( data_format: DataFormatInfo, ) -> TUnion[int, float, bytes, str, bool]: """Converts a series of bytes to a particular type of value.""" - if struct_type == int: + if struct_type is int: return int.from_bytes( data, byteorder=data_format.byteorder, signed=data_format.signed ) - if struct_type == bool: + if struct_type is bool: struct_format = "?" - elif struct_type == float: + elif struct_type is float: float_vals = "zzezfzzzd" if ( data_format.length > len(float_vals) @@ -70,7 +70,7 @@ def convert_value_to_data( f"Written value is not of the correct type for {struct_type.__name__}" ) - if struct_type == int and isinstance(value, int): + if struct_type is int and isinstance(value, int): # Doubling up on the isinstance is for mypy return int.to_bytes( value, @@ -78,9 +78,9 @@ def convert_value_to_data( byteorder=data_format.byteorder, signed=data_format.signed, ) - if struct_type == bool: + if struct_type is bool: struct_format = "?" - elif struct_type == float: + elif struct_type is float: float_vals = "zzezfzzzd" if ( data_format.length > len(float_vals) @@ -152,7 +152,7 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): type_name: str, object_info: interfaces.objects.ObjectInformation, data_format: DataFormatInfo, - new_value: TUnion[int, float, bool, bytes, str] = None, + new_value: Optional[TUnion[int, float, bool, bytes, str]] = None, **kwargs, ) -> "PrimitiveObject": """Creates the appropriate class and returns it so that the native type @@ -601,7 +601,7 @@ class Enumeration(interfaces.objects.ObjectInterface, int): inverse_choices[v] = k return inverse_choices - def lookup(self, value: int = None) -> str: + def lookup(self, value: Optional[int] = None) -> str: """Looks up an individual value and returns the associated name. If multiple identifiers map to the same value, the first matching identifier will be returned @@ -690,7 +690,7 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): type_name: str, object_info: interfaces.objects.ObjectInformation, count: int = 0, - subtype: templates.ObjectTemplate = None, + subtype: Optional[templates.ObjectTemplate] = None, ) -> None: super().__init__(context=context, type_name=type_name, object_info=object_info) self._vol["count"] = count diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index 8aa527cdb..b241ed56a 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -22,8 +22,8 @@ def bswap_32(value: int) -> int: def bswap_64(value: int) -> int: - low = bswap_32((value >> 32)) - high = bswap_32((value & 0xFFFFFFFF)) + low = bswap_32(value >> 32) + high = bswap_32(value & 0xFFFFFFFF) return ((high << 32) | low) & 0xFFFFFFFFFFFFFFFF diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 4f07bd5a8..1c2ac52e9 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -7,6 +7,7 @@ import os import pathlib import zipfile from typing import Generator, List +from importlib.util import find_spec from volatility3 import schemas, symbols from volatility3.framework import constants, interfaces, renderers @@ -96,16 +97,12 @@ class IsfInfo(plugins.PluginInterface): if filter_item in isf_file: filtered_list.append(isf_file) - try: - import jsonschema - - if not self.config["validate"]: - raise ImportError # Act as if we couldn't import if validation is turned off + if find_spec("jsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" - except ImportError: + else: def check_valid(data): return "Unknown" @@ -135,6 +132,7 @@ class IsfInfo(plugins.PluginInterface): valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {entry}") + continue yield ( 0, ( diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 24149a390..10e7a7a72 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -119,7 +119,7 @@ class LayerWriter(plugins.PluginInterface): # Update the filename, which may have changed if a file # with the same name already existed. output_name = file_handle.preferred_filename - except IOError as excp: + except OSError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) yield 0, (f"Layer has been written to {output_name}",) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index 77a433a3b..056e3cd51 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -22,7 +22,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): """Recovers bash command history from memory.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -33,7 +33,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py index 56de52883..c57bdd65a 100644 --- a/volatility3/framework/plugins/linux/boottime.py +++ b/volatility3/framework/plugins/linux/boottime.py @@ -15,7 +15,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """Shows the time the system was started""" _required_framework_version = (2, 11, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -26,7 +26,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index a8a8fb1fa..1d0c60c11 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -49,8 +49,8 @@ class CapabilitiesData: class Capabilities(plugins.PluginInterface): """Lists process capabilities""" - _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _required_framework_version = (2, 13, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -61,7 +61,7 @@ class Capabilities(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pids", @@ -136,7 +136,7 @@ class Capabilities(plugins.PluginInterface): comm=utility.array_to_string(task.comm), pid=int(task.pid), tgid=int(task.tgid), - ppid=int(task.parent.pid), + ppid=int(task.get_parent_pid()), euid=int(task.cred.euid), ) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 0857576d5..96f77ce4d 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -12,7 +12,7 @@ class Check_creds(interfaces.plugins.PluginInterface): """Checks if any processes are sharing credential structures""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 2) @classmethod def get_requirements(cls): @@ -23,7 +23,7 @@ class Check_creds(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] @@ -55,7 +55,7 @@ class Check_creds(interfaces.plugins.PluginInterface): for cred_addr, pids in creds.items(): if len(pids) > 1: - pid_str = ", ".join([str(pid) for pid in pids]) + pid_str = ", ".join(str(pid) for pid in pids) fields = [ format_hints.Hex(cred_addr), diff --git a/volatility3/framework/plugins/linux/check_idt.py b/volatility3/framework/plugins/linux/check_idt.py index cc3a08933..07582e2c1 100644 --- a/volatility3/framework/plugins/linux/check_idt.py +++ b/volatility3/framework/plugins/linux/check_idt.py @@ -53,7 +53,7 @@ class Check_idt(interfaces.plugins.PluginInterface): address_mask = self.context.layers[vmlinux.layer_name].address_mask # hw handlers + system call - check_idxs = list(range(0, 20)) + [128] + check_idxs = list(range(20)) + [128] if is_32bit: if vmlinux.has_type("gate_struct"): diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index b6634d612..3537a9fa1 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -103,7 +103,7 @@ class Check_syscall(plugins.PluginInterface): try: func_addr = vmlinux.get_symbol(syscall_entry_func).address - except exceptions.SymbolError as e: + except exceptions.SymbolError: # if we can't find the disassemble function then bail and rely on a different method return 0 diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 9f3bd274b..2fd740941 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -25,7 +25,7 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 2) + _version = (2, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -36,7 +36,7 @@ class Elfs(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 22aba6408..8cdbfe493 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -3,8 +3,9 @@ # import logging +from typing import Iterable, Tuple -from volatility3.framework import exceptions, renderers +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -16,8 +17,8 @@ vollog = logging.getLogger(__name__) class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" - _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _required_framework_version = (2, 13, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -29,7 +30,7 @@ class Envars(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", @@ -39,84 +40,98 @@ class Envars(plugins.PluginInterface): ), ] + @staticmethod + def get_task_env_variables( + context: interfaces.context.ContextInterface, + task: interfaces.objects.ObjectInterface, + env_area_max_size: int = 8192, + ) -> Iterable[Tuple[str, str]]: + """Yields environment variables for a given task. + + Args: + context: The plugin's operational context. + task: The task object from which to extract environment variables. + env_area_max_size: Maximum allowable size for the environment variables area. + Tasks exceeding this size will be skipped. Default is 8192. + + Yields: + Tuples of (key, value) representing each environment variable. + """ + + task_name = utility.array_to_string(task.comm) + task_pid = task.pid + env_start = task.mm.env_start + env_end = task.mm.env_end + env_area_size = env_end - env_start + if not (0 < env_area_size <= env_area_max_size): + vollog.debug( + f"Task {task_pid} {task_name} appears to have environment variables of size " + f"{env_area_size} bytes which fails the sanity checking, will not extract " + "any envars." + ) + return None + + # Get process layer to read envars from + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + return None + proc_layer = context.layers[proc_layer_name] + + # Ensure the entire buffer is readable to prevent relying on exception handling + if not proc_layer.is_valid(env_start, env_area_size): + # Not mapped / swapped out + vollog.debug( + f"Unable to read environment variables for {task_pid} {task_name} starting at " + f" virtual address 0x{env_start:x} for {env_area_size} bytes, will not " + "extract any envars." + ) + return None + + # Read the full task environment variable buffer. + envar_data = proc_layer.read(env_start, env_area_size) + + # Parse envar data, envars are null terminated, keys and values are separated by '=' + envar_data = envar_data.rstrip(b"\x00") + for envar_pair in envar_data.split(b"\x00"): + try: + env_key, env_value = envar_pair.decode().split("=", 1) + except ValueError: + # Some legitimate programs, like 'avahi-daemon', avoid reallocating the args + # and instead exploit the fact that the environment variables area is contiguous + # to the args. This allows them to include a longer process name in the listing, + # causing overwrites and incorrect results. In such cases, it's better to abort + # the current task rather than displaying misleading or incorrect output. + break + + yield env_key, env_value + def _generator(self, tasks): """Generates a listing of processes along with environment variables""" # walk the process list and return the envars for task in tasks: - pid = task.pid - - # get process name as string - name = utility.array_to_string(task.comm) - - # try and get task parent - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." - ) - ppid = 0 - - # kernel threads never have an mm as they do not have userland mappings - try: - mm = task.mm - except exceptions.InvalidAddressException: - # no mm so cannot get envars - vollog.debug( - f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." - ) - mm = None + if task.is_kernel_thread: continue - # if mm exists attempt to get envars - if mm: - # get process layer to read envars from - proc_layer_name = task.add_process_layer() - if proc_layer_name is None: - vollog.debug( - f"Unable to construct process layer for task {pid} {name}, will not extract any envars." - ) - continue - proc_layer = self.context.layers[proc_layer_name] + task_pid = task.pid + task_name = utility.array_to_string(task.comm) + task_ppid = task.get_parent_pid() - # get the size of the envars with sanity checking - envars_size = task.mm.env_end - task.mm.env_start - if not (0 < envars_size <= 8192): - vollog.debug( - f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." - ) - continue - - # attempt to read all envars data - try: - envar_data = proc_layer.read(task.mm.env_start, envars_size) - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." - ) - continue - - # parse envar data, envars are null terminated, keys and values are separated by '=' - envar_data = envar_data.rstrip(b"\x00") - for envar_pair in envar_data.split(b"\x00"): - try: - key, value = envar_pair.decode().split("=", 1) - except ValueError: - vollog.debug( - f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" - ) - continue - yield (0, (pid, ppid, name, key, value)) + for env_key, env_value in self.get_task_env_variables(self.context, task): + yield (0, (task_pid, task_ppid, task_name, env_key, env_value)) def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - - return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], - self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ), + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func ) + + headers = [ + ("PID", int), + ("PPID", int), + ("COMM", str), + ("KEY", str), + ("VALUE", str), + ] + + return renderers.TreeGrid(headers, self._generator(tasks)) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py new file mode 100644 index 000000000..7b644eccf --- /dev/null +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -0,0 +1,334 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +import io + +from dataclasses import dataclass +from typing import Type, List, Dict, Tuple +from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import ( + format_hints, + TreeGrid, + NotAvailableValue, + UnreadableValue, +) +from volatility3.framework.objects import utility +from volatility3.framework.constants import architectures +from volatility3.framework.symbols import linux + +# Image manipulation functions are kept in the plugin, +# to prevent a general exit on missing PIL (pillow) dependency. +try: + from PIL import Image + + has_pil = True +except ImportError: + has_pil = False + +vollog = logging.getLogger(__name__) + + +@dataclass +class Framebuffer: + """Framebuffer object internal representation. This is useful to unify a framebuffer with precalculated + properties and pass it through functions conveniently.""" + + id: str + xres_virtual: int + yres_virtual: int + line_length: int + bpp: int + """Bits Per Pixel""" + size: int + color_fields: Dict[str, Tuple[int, int, int]] + fb_info: interfaces.objects.ObjectInterface + + +class Fbdev(interfaces.plugins.PluginInterface): + """Extract framebuffers from the fbdev graphics subsystem""" + + _version = (1, 0, 0) + _required_framework_version = (2, 11, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + ), + requirements.BooleanRequirement( + name="dump", + description="Dump framebuffers", + default=False, + optional=True, + ), + ] + + @classmethod + def parse_fb_pixel_bitfields( + cls, fb_var_screeninfo: interfaces.objects.ObjectInterface + ) -> Dict[str, Tuple[int, int, int]]: + """Organize a framebuffer pixel format into a dictionary. + This is needed to know the position and bitlength of a color inside + a pixel. + + Args: + fb_var_screeninfo: a fb_var_screeninfo kernel object instance + + Returns: + The color fields mappings + + Documentation: + include/uapi/linux/fb.h: + struct fb_bitfield { + __u32 offset; /* beginning of bitfield */ + __u32 length; /* length of bitfield */ + __u32 msb_right; /* != 0 : Most significant bit is right */ + }; + """ + # Naturally order by RGBA + color_mappings = [ + ("R", fb_var_screeninfo.red), + ("G", fb_var_screeninfo.green), + ("B", fb_var_screeninfo.blue), + ("A", fb_var_screeninfo.transp), + ] + color_fields = {} + for color_code, fb_bitfield in color_mappings: + color_fields[color_code] = ( + int(fb_bitfield.offset), + int(fb_bitfield.length), + int(fb_bitfield.msb_right), + ) + return color_fields + + @classmethod + def convert_fb_raw_buffer_to_image( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + fb: Framebuffer, + ): + """Convert raw framebuffer pixels to an image. + + Args: + fb: the relevant Framebuffer object + + Returns: + A PIL Image object + + Documentation: + include/uapi/linux/fb.h: + /* Interpretation of offset for color fields: All offsets are from the right, + * inside a "pixel" value, which is exactly 'bits_per_pixel' wide (means: you + * can use the offset as right argument to <<). A pixel afterwards is a bit + * stream and is written to video memory as that unmodified. + """ + kernel = context.modules[kernel_name] + kernel_layer = context.layers[kernel.layer_name] + + raw_pixels = io.BytesIO(kernel_layer.read(fb.fb_info.screen_base, fb.size)) + bytes_per_pixel = fb.bpp // 8 + image = Image.new("RGBA", (fb.xres_virtual, fb.yres_virtual)) + + # This is not designed to be extremely fast (numpy isn't available), + # but convenient and dynamic for any color field layout. + for y in range(fb.yres_virtual): + for x in range(fb.xres_virtual): + raw_pixel = int.from_bytes(raw_pixels.read(bytes_per_pixel), "little") + pixel = [0, 0, 0, 255] + # The framebuffer is expected to have been correctly constructed, + # especially by parse_fb_pixel_bitfields, to get the needed RGBA mappings. + for i, color_code in enumerate(["R", "G", "B", "A"]): + offset, length, msb_right = fb.color_fields[color_code] + if length == 0: + continue + color_value = (raw_pixel >> offset) & (2**length - 1) + if msb_right: + # Reverse bit order + color_value = int( + "{:0{length}b}".format(color_value, length=length)[::-1], 2 + ) + pixel[i] = color_value + image.putpixel((x, y), tuple(pixel)) + + return image + + @classmethod + def dump_fb( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + fb: Framebuffer, + convert_to_png_image: bool, + ) -> str: + """Dump a Framebuffer buffer to disk. + + Args: + fb: the relevant Framebuffer object + convert_to_image: a boolean specifying if the buffer should be converted to an image + + Returns: + The filename of the dumped buffer. + """ + kernel = context.modules[kernel_name] + kernel_layer = context.layers[kernel.layer_name] + id = "N-A" if isinstance(fb.id, NotAvailableValue) else fb.id + base_filename = f"{id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" + if convert_to_png_image: + image_object = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) + raw_io_output = io.BytesIO() + image_object.save(raw_io_output, "PNG") + final_fb_buffer = raw_io_output.getvalue() + filename = f"{base_filename}.png" + else: + final_fb_buffer = kernel_layer.read(fb.fb_info.screen_base, fb.size) + filename = f"{base_filename}.raw" + + with open_method(filename) as f: + f.write(final_fb_buffer) + return f.preferred_filename + + @classmethod + def parse_fb_info( + cls, + fb_info: interfaces.objects.ObjectInterface, + ) -> Framebuffer: + """Parse an fb_info struct + Args: + fb_info: an fb_info kernel object live instance + + Returns: + A Framebuffer object + + Documentation: + https://docs.kernel.org/fb/api.html: + - struct fb_fix_screeninfo stores device independent unchangeable information about the frame buffer device and the current format. + Those information can't be directly modified by applications, but can be changed by the driver when an application modifies the format. + - struct fb_var_screeninfo stores device independent changeable information about a frame buffer device, its current format and video mode, + as well as other miscellaneous parameters. + """ + id = utility.array_to_string(fb_info.fix.id) or NotAvailableValue() + color_fields = None + + # 0 = color, 1 = grayscale, >1 = FOURCC + if fb_info.var.grayscale in [0, 1]: + color_fields = cls.parse_fb_pixel_bitfields(fb_info.var) + + # There a lot of tricky pixel formats used by drivers and vendors in include/uapi/linux/videodev2.h. + # As Volatility3 is not a video format converter, it is best to play it safe and let the user parse + # the raw data manually (with ffmpeg for example). + elif fb_info.var.grayscale > 1: + fourcc = linux.LinuxUtilities.convert_fourcc_code(fb_info.var.grayscale) + warn_msg = f"""Framebuffer "{id}" uses a FOURCC pixel format "{fourcc}" that isn't natively supported. +You can try using ffmpeg to decode the raw buffer. Example usage: +"ffmpeg -pix_fmts" to list supported formats, then +"ffmpeg -f rawvideo -video_size {fb_info.var.xres_virtual}x{fb_info.var.yres_virtual} -i .raw -pix_fmt output.png".""" + vollog.warning(warn_msg) + + # Prefer using the virtual resolution, instead of the visible one. + # This prevents missing non-visible data stored in the framebuffer. + fb = Framebuffer( + id, + xres_virtual=fb_info.var.xres_virtual, + yres_virtual=fb_info.var.yres_virtual, + line_length=fb_info.fix.line_length, + bpp=fb_info.var.bits_per_pixel, + size=fb_info.var.yres_virtual * fb_info.fix.line_length, + color_fields=color_fields, + fb_info=fb_info, + ) + + return fb + + def _generator(self): + + if not has_pil: + vollog.error( + "PIL (pillow) module is required to use this plugin. Please install it manually or through pyproject.toml." + ) + return None + + kernel_name = self.config["kernel"] + kernel = self.context.modules[kernel_name] + + if not kernel.has_symbol("num_registered_fb"): + raise exceptions.SymbolError( + "num_registered_fb", + kernel.symbol_table_name, + "The provided symbol does not exist in the symbol table. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.", + ) + + num_registered_fb = kernel.object_from_symbol("num_registered_fb") + if num_registered_fb < 1: + vollog.info("No registered framebuffer in the fbdev API.") + return None + + registered_fb = kernel.object_from_symbol("registered_fb") + fb_info_list = utility.array_of_pointers( + registered_fb, + num_registered_fb, + kernel.symbol_table_name + constants.BANG + "fb_info", + self.context, + ) + + for fb_info in fb_info_list: + fb = self.parse_fb_info(fb_info) + file_output = "Disabled" + if self.config["dump"]: + try: + file_output = self.dump_fb( + self.context, kernel_name, self.open, fb, bool(fb.color_fields) + ) + file_output = str(file_output) + except exceptions.InvalidAddressException as excp: + vollog.error( + f'Layer {excp.layer_name} failed to read address {hex(excp.invalid_address)} when dumping framebuffer "{fb.id}".' + ) + file_output = UnreadableValue() + + try: + fb_device_name = utility.pointer_to_string( + fb.fb_info.dev.kobj.name, 256 + ) + except exceptions.InvalidAddressException: + fb_device_name = NotAvailableValue() + + yield ( + 0, + ( + format_hints.Hex(fb.fb_info.screen_base), + fb_device_name, + fb.id, + fb.size, + f"{fb.xres_virtual}x{fb.yres_virtual}", + fb.bpp, + "RUNNING" if fb.fb_info.state == 0 else "SUSPENDED", + file_output, + ), + ) + + def run(self): + columns = [ + ("Address", format_hints.Hex), + ("Device", str), + ("ID", str), + ("Size", int), + ("Virtual resolution", str), + ("BPP", int), + ("State", str), + ("Filename", str), + ] + + return TreeGrid( + columns, + self._generator(), + ) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index e26d69543..d66e3b9ca 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return "%lu.%06lu" % (nsec / 1000000000, (nsec % 1000000000) / 1000) + return f"{nsec / 1000000000:lu}.{(nsec % 1000000000) / 1000:06lu}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = "%s(%u)" % (caller_name, caller_id & ~0x80000000) + caller = f"{caller_name}({caller_id & ~0x80000000:u})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index b9ced73f3..40e992069 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -20,7 +20,7 @@ class Kthreads(plugins.PluginInterface): """Enumerates kthread functions""" _required_framework_version = (2, 11, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -34,7 +34,7 @@ class Kthreads(plugins.PluginInterface): name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py index 7ec1f7f7f..e251b5689 100644 --- a/volatility3/framework/plugins/linux/library_list.py +++ b/volatility3/framework/plugins/linux/library_list.py @@ -21,7 +21,7 @@ class LibraryList(interfaces.plugins.PluginInterface): """Enumerate libraries loaded into processes""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls): @@ -32,7 +32,7 @@ class LibraryList(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pids", diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index a65b0d00b..49e990e93 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -54,8 +54,7 @@ class Lsmod(plugins.PluginInterface): table_name = modules.vol.type_name.split(constants.BANG)[0] - for module in modules.to_list(table_name + constants.BANG + "module", "list"): - yield module + yield from modules.to_list(table_name + constants.BANG + "module", "list") def _generator(self): try: diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 802954f43..daa8e5a3d 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -110,7 +110,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -121,7 +121,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 0b10e60c6..e45688e97 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -18,7 +18,7 @@ class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -29,7 +29,7 @@ class Malfind(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 65775c4aa..b4f80e4f5 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -36,7 +36,7 @@ class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 2, 0) - _version = (1, 2, 2) + _version = (1, 2, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -47,7 +47,7 @@ class MountInfo(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 46b24b27a..382268515 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -462,7 +462,7 @@ class InodePages(plugins.PluginInterface): f.seek(current_fp) f.write(page_bytes) - except IOError as e: + except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) def _generator(self): @@ -483,6 +483,9 @@ class InodePages(plugins.PluginInterface): if inode_in.path == self.config["find"]: inode = inode_in.inode break # Only the first match + else: + vollog.error("Unable to find inode with path %s", self.config["find"]) + return None elif self.config["inode"]: inode = vmlinux.object("inode", self.config["inode"], absolute=True) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 2d210c233..060b3928e 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -3,7 +3,7 @@ # import logging -from typing import List +from typing import List, Iterable from volatility3.framework import renderers, interfaces, constants from volatility3.framework.symbols import linux @@ -19,7 +19,7 @@ class PIDHashTable(plugins.PluginInterface): """Enumerates processes through the PID hash table""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -30,7 +30,7 @@ class PIDHashTable(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) @@ -218,7 +218,7 @@ class PIDHashTable(plugins.PluginInterface): return None - def get_tasks(self) -> interfaces.objects.ObjectInterface: + def get_tasks(self) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerates processes through the PID hash table Yields: @@ -231,14 +231,16 @@ class PIDHashTable(plugins.PluginInterface): yield from sorted(pid_func(), key=lambda t: (t.tgid, t.pid)) - def _generator( - self, decorate_comm: bool = False - ) -> interfaces.objects.ObjectInterface: + def _generator(self, decorate_comm: bool = False): for task in self.get_tasks(): - offset, pid, tid, ppid, name, _creation_time = ( - pslist.PsList.get_task_fields(task, decorate_comm) + task_fields = pslist.PsList.get_task_fields(task, decorate_comm) + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, ) - fields = format_hints.Hex(offset), pid, tid, ppid, name yield 0, fields def run(self): diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 00832140a..441c6bc93 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,7 +21,7 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @@ -35,7 +35,7 @@ class Maps(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", @@ -125,9 +125,7 @@ class Maps(plugins.PluginInterface): proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None vm_size = vm_end - vm_start @@ -165,7 +163,9 @@ class Maps(plugins.PluginInterface): address_list = self.config.get("address", None) if not address_list: # do not filter as no address_list was supplied - vma_filter_func = lambda _: True + def vma_filter_func(_): + return True + else: # filter for any vm_start that matches the supplied address config def vma_filter_function(x: interfaces.objects.ObjectInterface) -> bool: diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 5467c3b4c..a544c9d67 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -14,8 +14,8 @@ from volatility3.plugins.linux import pslist class PsAux(plugins.PluginInterface): """Lists processes with their command line arguments""" - _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _required_framework_version = (2, 13, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls): @@ -27,7 +27,7 @@ class PsAux(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", @@ -98,14 +98,8 @@ class PsAux(plugins.PluginInterface): # walk the process list and report the arguments for task in tasks: pid = task.pid - - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - ppid = 0 - + ppid = task.get_parent_pid() name = utility.array_to_string(task.comm) - args = self._get_command_line_args(task, name) yield (0, (pid, ppid, name, args)) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 6460462a7..37cf000fc 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -2,7 +2,9 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import datetime -from typing import Any, Callable, Iterable, List, Tuple +import dataclasses +import contextlib +from typing import Any, Callable, Iterable, List, Optional from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -14,11 +16,25 @@ from volatility3.plugins import timeliner from volatility3.plugins.linux import elfs +@dataclasses.dataclass +class TaskFields: + offset: int + user_pid: int + user_tid: int + user_ppid: int + name: str + uid: Optional[int] + gid: Optional[int] + euid: Optional[int] + egid: Optional[int] + creation_time: Optional[datetime.datetime] + + class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" - _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) + _required_framework_version = (2, 13, 0) + _version = (4, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -58,7 +74,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ] @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[Any], bool]: + def create_pid_filter( + cls, pid_list: Optional[List[int]] = None + ) -> Callable[[Any], bool]: """Constructs a filter function for process IDs. Args: @@ -82,7 +100,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def get_task_fields( cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False - ) -> Tuple[int, int, int, int, str, datetime.datetime]: + ) -> TaskFields: """Extract the fields needed for the final output Args: @@ -91,21 +109,34 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): and of Kernel threads in square brackets. Defaults to False. Returns: - A tuple with the fields to show in the plugin output. + A TaskFields object with the fields to show in the plugin output. """ - pid = task.tgid - tid = task.pid - ppid = task.parent.tgid if task.parent else 0 name = utility.array_to_string(task.comm) - start_time = task.get_create_time() if decorate_comm: if task.is_kernel_thread: name = f"[{name}]" elif task.is_user_thread: name = f"{{{name}}}" - task_fields = (task.vol.offset, pid, tid, ppid, name, start_time) - return task_fields + # This function may be called with a partially initialized/uninitialized task. + # Ensure it always returns a valid TaskFields object, ready for use in a plugin. + valid_cred = task.cred and task.cred.is_readable() + creation_time = None + with contextlib.suppress(Exception): + creation_time = task.get_create_time() + + return TaskFields( + offset=task.vol.offset, + user_pid=task.tgid, + user_tid=task.pid, + user_ppid=task.get_parent_pid(), + name=name, + uid=task.cred.uid if valid_cred else None, + gid=task.cred.gid if valid_cred else None, + euid=task.cred.euid if valid_cred else None, + egid=task.cred.egid if valid_cred else None, + creation_time=creation_time, + ) def _get_file_output(self, task: interfaces.objects.ObjectInterface) -> str: """Extract the elf for the process if requested @@ -179,17 +210,19 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): else: file_output = "Disabled" - offset, pid, tid, ppid, name, creation_time = self.get_task_fields( - task, decorate_comm - ) + task_fields = self.get_task_fields(task, decorate_comm) yield 0, ( - format_hints.Hex(offset), - pid, - tid, - ppid, - name, - creation_time or renderers.NotAvailableValue(), + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, + task_fields.uid or renderers.NotAvailableValue(), + task_fields.gid or renderers.NotAvailableValue(), + task_fields.euid or renderers.NotAvailableValue(), + task_fields.egid or renderers.NotAvailableValue(), + task_fields.creation_time or renderers.NotAvailableValue(), file_output, ) @@ -238,6 +271,10 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ("TID", int), ("PPID", int), ("COMM", str), + ("UID", int), + ("GID", int), + ("EUID", int), + ("EGID", int), ("CREATION TIME", datetime.datetime), ("File output", str), ] @@ -251,10 +288,11 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for task in self.list_tasks( self.context, self.config["kernel"], filter_func, include_threads=True ): - offset, user_pid, user_tid, _user_ppid, name, creation_time = ( - self.get_task_fields(task) + task_fields = self.get_task_fields(task) + description = f"Process {task_fields.user_pid}/{task_fields.user_tid} {task_fields.name} ({task_fields.offset})" + + yield ( + description, + timeliner.TimeLinerType.CREATED, + task_fields.creation_time, ) - - description = f"Process {user_pid}/{user_tid} {name} ({offset})" - - yield (description, timeliner.TimeLinerType.CREATED, creation_time) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 40784a647..ba68c4856 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -2,15 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import Iterable, List, Tuple +from typing import Iterable, List import struct from enum import Enum from volatility3.framework import renderers, interfaces, symbols, constants, exceptions from volatility3.framework.configuration import requirements -from volatility3.framework.objects import utility from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints +from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) @@ -27,8 +27,8 @@ class DescExitStateEnum(Enum): class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" - _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _required_framework_version = (2, 13, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -38,37 +38,11 @@ class PsScan(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) + ), ] - def _get_task_fields( - self, task: interfaces.objects.ObjectInterface - ) -> Tuple[int, int, int, str, str]: - """Extract the fields needed for the final output - - Args: - task: A task object from where to get the fields. - Returns: - A tuple with the fields to show in the plugin output. - """ - pid = task.tgid - tid = task.pid - ppid = 0 - - if task.parent.is_readable(): - ppid = task.parent.tgid - name = utility.array_to_string(task.comm) - exit_state = DescExitStateEnum(task.exit_state).name - - task_fields = ( - format_hints.Hex(task.vol.offset), - pid, - tid, - ppid, - name, - exit_state, - ) - return task_fields - def _generator(self): """Generates the tasks found from scanning.""" @@ -78,8 +52,18 @@ class PsScan(interfaces.plugins.PluginInterface): for task in self.scan_tasks( self.context, vmlinux_module_name, vmlinux.layer_name ): - row = self._get_task_fields(task) - yield (0, row) + task_fields = pslist.PsList.get_task_fields(task) + exit_state = DescExitStateEnum(task.exit_state).name + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, + exit_state, + ) + + yield (0, fields) @classmethod def scan_tasks( @@ -133,7 +117,7 @@ class PsScan(interfaces.plugins.PluginInterface): ) elif len(kernel_layer.dependencies) == 0: vollog.error( - f"Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." + "Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." ) raise exceptions.LayerException( kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 9dc5ea3cc..74e172139 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -12,8 +12,8 @@ class PsTree(interfaces.plugins.PluginInterface): """Plugin for listing processes in a tree based on their parent process ID.""" - _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _required_framework_version = (2, 13, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls): @@ -25,7 +25,7 @@ class PsTree(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", @@ -56,9 +56,9 @@ class PsTree(interfaces.plugins.PluginInterface): seen = set([pid]) level = 0 proc = self._tasks.get(pid) - while proc and proc.parent and proc.parent.pid not in seen: + while proc and proc.get_parent_pid() not in seen: if proc.is_thread_group_leader: - parent_pid = proc.parent.pid + parent_pid = proc.get_parent_pid() else: parent_pid = proc.tgid @@ -101,13 +101,17 @@ class PsTree(interfaces.plugins.PluginInterface): def yield_processes(pid): task = self._tasks[pid] - offset, pid, tid, ppid, name, _creation_time = ( - pslist.PsList.get_task_fields(task, decorate_comm) + task_fields = pslist.PsList.get_task_fields(task, decorate_comm) + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, ) - fields = format_hints.Hex(offset), pid, tid, ppid, name - yield (self._levels[tid] - 1, fields) + yield (self._levels[task_fields.user_tid] - 1, fields) - for child_pid in sorted(self._children.get(tid, [])): + for child_pid in sorted(self._children.get(task_fields.user_tid, [])): yield from yield_processes(child_pid) for pid, level in self._levels.items(): diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index 271c0e75e..6493f22b9 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -19,7 +19,7 @@ class Ptrace(plugins.PluginInterface): """Enumerates ptrace's tracer and tracee tasks""" _required_framework_version = (2, 10, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -30,7 +30,7 @@ class Ptrace(plugins.PluginInterface): architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index e5cf48d16..7376bcbee 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -372,7 +372,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): bt_sock = sock.cast("bt_sock") def bt_addr(addr): - return ":".join(reversed(["%02x" % x for x in addr.b])) + return ":".join(reversed([f"{x:02x}" for x in addr.b])) src_addr = src_port = dst_addr = dst_port = None bt_protocol = bt_sock.get_protocol() @@ -438,7 +438,7 @@ class Sockstat(plugins.PluginInterface): """Lists all network connections for all processes.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 1) + _version = (3, 0, 2) @classmethod def get_requirements(cls): @@ -455,7 +455,7 @@ class Sockstat(plugins.PluginInterface): name="lsof", plugin=lsof.Lsof, version=(2, 0, 0) ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/vmaregexscan.py b/volatility3/framework/plugins/linux/vmaregexscan.py index 4446fc550..8fb96da1e 100644 --- a/volatility3/framework/plugins/linux/vmaregexscan.py +++ b/volatility3/framework/plugins/linux/vmaregexscan.py @@ -21,7 +21,7 @@ class VmaRegExScan(plugins.PluginInterface): """Scans all virtual memory areas for tasks using RegEx.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) MAXSIZE_DEFAULT = 128 @@ -35,7 +35,7 @@ class VmaRegExScan(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 650fcf078..4db23e50b 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -18,7 +18,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -31,7 +31,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): optional=True, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/mac/check_sysctl.py b/volatility3/framework/plugins/mac/check_sysctl.py index 4f64eaed8..ed3e34aea 100644 --- a/volatility3/framework/plugins/mac/check_sysctl.py +++ b/volatility3/framework/plugins/mac/check_sysctl.py @@ -60,7 +60,7 @@ class Check_sysctl(plugins.PluginInterface): return var_str def _process_sysctl_list(self, kernel, sysctl_list, recursive=0): - if type(sysctl_list) == volatility3.framework.objects.Pointer: + if type(sysctl_list) is volatility3.framework.objects.Pointer: sysctl_list = sysctl_list.dereference().cast("sysctl_oid_list") sysctl = sysctl_list.slh_first @@ -93,10 +93,9 @@ class Check_sysctl(plugins.PluginInterface): val = self._parse_global_variable_sysctls(kernel, name) elif ctltype == "CTLTYPE_NODE": if sysctl.oid_handler == 0: - for info in self._process_sysctl_list( + yield from self._process_sysctl_list( kernel, sysctl.oid_arg1, recursive=1 - ): - yield info + ) val = "Node" diff --git a/volatility3/framework/plugins/mac/kauth_scopes.py b/volatility3/framework/plugins/mac/kauth_scopes.py index afb320a07..c2c473eac 100644 --- a/volatility3/framework/plugins/mac/kauth_scopes.py +++ b/volatility3/framework/plugins/mac/kauth_scopes.py @@ -80,7 +80,7 @@ class Kauth_scopes(interfaces.plugins.PluginInterface): ( identifier, format_hints.Hex(scope.ks_idata), - len([l for l in scope.get_listeners()]), + len([listener for listener in scope.get_listeners()]), format_hints.Hex(callback), module_name, symbol_name, diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 2a8692b77..41fde31ca 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -119,8 +119,7 @@ class Kevents(interfaces.plugins.PluginInterface): return None for klist in klist_array: - for kn in mac.MacUtilities.walk_slist(klist, "kn_link"): - yield kn + yield from mac.MacUtilities.walk_slist(klist, "kn_link") @classmethod def _get_task_kevents(cls, kernel, task): diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index ff654e1a7..1a1e33571 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -49,8 +49,7 @@ class Mount(plugins.PluginInterface): list_head = kernel.object_from_symbol(symbol_name="mountlist") - for mount in mac.MacUtilities.walk_tailq(list_head, "mnt_list"): - yield mount + yield from mac.MacUtilities.walk_tailq(list_head, "mnt_list") def _generator(self): for mount in self.list_mounts(self.context, self.config["kernel"]): diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index fe5179dfa..bd905615d 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -115,9 +115,7 @@ class Maps(interfaces.plugins.PluginInterface): proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None vm_size = vm_end - vm_start @@ -154,7 +152,9 @@ class Maps(interfaces.plugins.PluginInterface): address_list = self.config.get("address", None) if not address_list: # do not filter as no address_list was supplied - vma_filter_func = lambda _: True + def vma_filter_func(_): + return True + else: # filter for any vm_start that matches the supplied address config def vma_filter_function(task: interfaces.objects.ObjectInterface) -> bool: diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 9b570f3f9..8c5e5c1a5 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Dict, Iterable, List +from typing import Callable, Dict, Iterable, List, Optional from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -82,8 +82,12 @@ class PsList(interfaces.plugins.PluginInterface): return list_tasks @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[int], bool]: - filter_func = lambda _: False + def create_pid_filter( + cls, pid_list: Optional[List[int]] = None + ) -> Callable[[int], bool]: + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 pid_list = pid_list or [] filter_list = [x for x in pid_list if x is not None] diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index c754e43ef..0f4064d79 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -54,7 +54,9 @@ class Timeliner(interfaces.plugins.PluginInterface): self.automagics: Optional[List[interfaces.automagic.AutomagicInterface]] = None @classmethod - def get_usable_plugins(cls, selected_list: List[str] = None) -> List[Type]: + def get_usable_plugins( + cls, selected_list: Optional[List[str]] = None + ) -> List[Type]: # Initialize for the run plugin_list = list(framework.class_subclasses(TimeLinerInterface)) @@ -143,9 +145,7 @@ class Timeliner(interfaces.plugins.PluginInterface): times = self.timeline.get((plugin_name, item), {}) if times.get(timestamp_type, None) is not None: vollog.debug( - "Multiple timestamps for the same plugin/file combination found: {} {}".format( - plugin_name, item - ) + f"Multiple timestamps for the same plugin/file combination found: {plugin_name} {item}" ) times[timestamp_type] = timestamp self.timeline[(plugin_name, item)] = times @@ -206,8 +206,7 @@ class Timeliner(interfaces.plugins.PluginInterface): ) vollog.log(logging.DEBUG, traceback.format_exc()) - for data_item in sorted(data, key=self._sort_function): - yield data_item + yield from sorted(data, key=self._sort_function) # Write out a body file if necessary if self.config.get("create-bodyfile", True): diff --git a/volatility3/framework/plugins/windows/cmdline.py b/volatility3/framework/plugins/windows/cmdline.py index 8cfb5576c..9bd9eda0e 100644 --- a/volatility3/framework/plugins/windows/cmdline.py +++ b/volatility3/framework/plugins/windows/cmdline.py @@ -84,9 +84,7 @@ class CmdLine(interfaces.plugins.PluginInterface): result_text = f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)" except exceptions.InvalidAddressException as exp: - result_text = "Process {}: Required memory at {:#x} is not valid (incomplete layer {}?)".format( - proc_id, exp.invalid_address, exp.layer_name - ) + result_text = f"Process {proc_id}: Required memory at {exp.invalid_address:#x} is not valid (incomplete layer {exp.layer_name}?)" yield (0, (proc.UniqueProcessId, process_name, result_text)) diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index ad1c9d4bd..a448989c0 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -95,9 +95,7 @@ class Consoles(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) @classmethod @@ -176,12 +174,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) vollog.debug( - "Determined OS Version: {}.{} {}.{}".format( - kuser.NtMajorVersion, - kuser.NtMinorVersion, - vers.MajorVersion, - vers.MinorVersion, - ) + f"Determined OS Version: {kuser.NtMajorVersion}.{kuser.NtMinorVersion} {vers.MajorVersion}.{vers.MinorVersion}" ) if nt_major_version == 10 and arch == "x64": @@ -260,9 +253,7 @@ class Consoles(interfaces.plugins.PluginInterface): if ver: conhost_mod_version = ver[3] vollog.debug( - "Determined conhost.exe's FileVersion: {}".format( - conhost_mod_version - ) + f"Determined conhost.exe's FileVersion: {conhost_mod_version}" ) else: vollog.debug("Could not determine conhost.exe's FileVersion.") @@ -311,12 +302,7 @@ class Consoles(interfaces.plugins.PluginInterface): else: raise NotImplementedError( - "This version of Windows is not supported: {}.{} {}.{}!".format( - nt_major_version, - nt_minor_version, - vers.MajorVersion, - vers_minor_version, - ) + f"This version of Windows is not supported: {nt_major_version}.{nt_minor_version} {vers.MajorVersion}.{vers_minor_version}!" ) vollog.debug(f"Determined symbol filename: {filename}") diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index b0c162f46..183e4095c 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -433,6 +433,8 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] vads = self.get_vad_maps(proc) + if not vads: + continue # for each valid process, look for malicious syscall invocations for address, vad_path in self._get_rule_hits( diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 5a1b37fcf..1dafb6bf5 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -5,9 +5,9 @@ import contextlib import datetime import logging import re -from typing import List, Optional, Type +from typing import List -from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed @@ -19,7 +19,7 @@ vollog = logging.getLogger(__name__) class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): - """Lists the loaded modules in a particular windows memory image.""" + """Lists the loaded DLLs in a particular windows memory image.""" _required_framework_version = (2, 0, 0) _version = (3, 0, 0) @@ -39,6 +39,9 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="psscan", component=psscan.PsScan, version=(1, 1, 0) ), + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), requirements.VersionRequirement( name="info", component=info.Info, version=(1, 0, 0) ), @@ -53,16 +56,16 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): description="Process offset in the physical address space", optional=True, ), - requirements.StringRequirement( - name="name", - description="Specify a regular expression to match dll name(s)", - optional=True, - ), requirements.IntRequirement( name="base", description="Specify a base virtual address in process memory", optional=True, ), + requirements.StringRequirement( + name="name", + description="Specify a regular expression to match dll name(s)", + optional=True, + ), requirements.BooleanRequirement( name="ignore-case", description="Specify case insensitivity for the regular expression name matching", @@ -75,9 +78,6 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): default=False, optional=True, ), - requirements.VersionRequirement( - name="pedump", component=pedump.PEDump, version=(1, 0, 0) - ), ] def _generator(self, procs): @@ -90,12 +90,15 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kuser = info.Info.get_kuser_structure( self.context, kernel.layer_name, kernel.symbol_table_name ) + nt_major_version = int(kuser.NtMajorVersion) nt_minor_version = int(kuser.NtMinorVersion) + # LoadTime only applies to versions higher or equal to Window 7 (6.1 and higher) dll_load_time_field = (nt_major_version > 6) or ( nt_major_version == 6 and nt_minor_version >= 1 ) + for proc in procs: proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() @@ -135,7 +138,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute - # and 32bit version shouldn't have the Quadpart according to MSDN + # and 32-bit version shouldn't have the Quadpart according to MSDN try: DllLoadTime = conversion.wintime_to_datetime( entry.LoadTime.QuadPart @@ -199,16 +202,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _depth, row_data = row if not isinstance(row_data[6], datetime.datetime): continue - description = ( - "DLL Load: Process {} {} Loaded {} ({}) Size {} Offset {}".format( - row_data[0], - row_data[1], - row_data[4], - row_data[5], - row_data[3], - row_data[2], - ) - ) + description = f"DLL Load: Process {row_data[0]} {row_data[1]} Loaded {row_data[4]} ({row_data[5]}) Size {row_data[3]} Offset {row_data[2]}" yield (description, timeliner.TimeLinerType.CREATED, row_data[6]) def run(self): diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index bc554c0bf..64d9be4db 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -192,13 +192,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] - desired_file_name = "file.{0:#x}.{1:#x}.{2}.{3}.{4}".format( - file_obj.vol.offset, - memory_object.vol.offset, - cache_name, - ntpath.basename(obj_name), - extension, - ) + desired_file_name = f"file.{file_obj.vol.offset:#x}.{memory_object.vol.offset:#x}.{cache_name}.{ntpath.basename(obj_name)}.{extension}" file_handle = cls.dump_file_producer( file_obj, memory_object, open_method, layer, desired_file_name diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index 66db03c9c..cac4ecf40 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -92,7 +92,7 @@ class Envars(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing global environment variables keys (some keys might be excluded)", @@ -113,7 +113,7 @@ class Envars(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing user environment variables keys (some keys might be excluded)", @@ -134,7 +134,7 @@ class Envars(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing volatile environment variables keys (some keys might be excluded)", diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index 9b20ed2d0..eece7fb6c 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -26,7 +26,7 @@ def createservicesid(svc) -> str: ## The use of struct here is OK. It doesn't make much sense ## to leverage obj.Object inside this loop. dec.append(struct.unpack(" 0: - for x in self._make_handle_array(entry, level - 1, depth): - yield x + yield from self._make_handle_array(entry, level - 1, depth) depth += 1 else: handle_multiplier = 4 @@ -264,8 +289,7 @@ class Handles(interfaces.plugins.PluginInterface): ) return None - for handle_table_entry in self._make_handle_array(TableCode, table_levels): - yield handle_table_entry + yield from self._make_handle_array(TableCode, table_levels) def _generator(self, procs): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index 69fa94f06..30d4b602c 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -12,20 +12,15 @@ from volatility3.plugins.windows import pslist, vadinfo vollog = logging.getLogger(__name__) -VadData = NamedTuple( - "VadData", - [ - ("protection", str), - ("path", str), - ], -) -DLLData = NamedTuple( - "DLLData", - [ - ("path", str), - ], -) +class VadData(NamedTuple): + protection: str + path: str + + +class DLLData(NamedTuple): + path: str + ### Useful references on process hollowing # https://cysinfo.com/detecting-deceptive-hollowing-techniques/ @@ -146,9 +141,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): """ image_base = self._get_image_base(proc) if image_base is not None and image_base != proc.SectionBaseAddress: - yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format( - image_base, proc.SectionBaseAddress - ) + yield f"The ImageBaseAddress reported from the PEB ({image_base:#x}) does not match the process SectionBaseAddress ({proc.SectionBaseAddress:#x})" def _check_exe_protection( self, proc, vads: Dict[int, VadData], __ @@ -166,13 +159,9 @@ class HollowProcesses(interfaces.plugins.PluginInterface): base = proc.SectionBaseAddress if base not in vads: - yield "There is no VAD starting at the base address of the process executable ({:#x})".format( - base - ) + yield f"There is no VAD starting at the base address of the process executable ({base:#x})" elif vads[base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for VAD hosting the process executable ({:#x}) with path {}".format( - vads[base].protection, base, vads[base].path - ) + yield f"Unexpected protection ({vads[base].protection}) for VAD hosting the process executable ({base:#x}) with path {vads[base].path}" def _check_dlls_protection( self, _, vads: Dict[int, VadData], dlls: Dict[int, DLLData] @@ -184,9 +173,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): # PAGE_EXECUTE_WRITECOPY is the only valid permission for mapped DLLs and .exe files if vads[dll_base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format( - vads[dll_base].protection, dll_base, dlls[dll_base].path - ) + yield f"Unexpected protection ({vads[dll_base].protection}) for DLL in the PEB's load order list ({dll_base:#x}) with path {dlls[dll_base].path}" def _generator(self, procs): checks = [ diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index d2fdc0ad8..3bf7f57ed 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,7 +1,9 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -import logging, io, pefile +import logging +import io +import pefile from volatility3.framework.symbols import intermed from volatility3.framework import renderers, interfaces, exceptions, constants from volatility3.framework.configuration import requirements @@ -119,9 +121,7 @@ class IAT(interfaces.plugins.PluginInterface): ) except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index f09851b30..dac0f9c4a 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -13,12 +13,6 @@ from volatility3.plugins.windows import pslist, direct_system_calls vollog = logging.getLogger(__name__) -try: - import capstone -except ImportError: - # The generator of DirectSystemCalls will bail with a warning if capstone is not installed - pass - class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): _required_framework_version = (2, 4, 0) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 1df090b5b..14362776b 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -106,9 +106,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None @@ -122,8 +120,7 @@ class Malfind(interfaces.plugins.PluginInterface): vadinfo.winnt_protections, ) write_exec = "EXECUTE" in protection_string and "WRITE" in protection_string - dirty_page_check = False - + dirty_page = None if not write_exec: """ # Inspect "PAGE_EXECUTE_READ" VAD pages to detect @@ -137,12 +134,12 @@ class Malfind(interfaces.plugins.PluginInterface): try: # If we have a dirty page in a non writable "EXECUTE" region, it is suspicious. if proc_layer.is_dirty(page): - dirty_page_check = True + dirty_page = page break except exceptions.InvalidAddressException: # Abort as it is likely that other addresses in the same range will also fail. break - if not dirty_page_check: + if dirty_page is None: continue else: continue @@ -154,10 +151,10 @@ class Malfind(interfaces.plugins.PluginInterface): if cls.is_vad_empty(proc_layer, vad): continue - if dirty_page_check: + if dirty_page is not None: # Useful information to investigate the page content with volshell afterwards. vollog.warning( - f"[proc_id {proc_id}] Found suspicious DIRTY + {protection_string} page at {hex(page)}", + f"[proc_id {proc_id}] Found suspicious DIRTY + {protection_string} page at {hex(dirty_page)}", ) data = proc_layer.read(vad.get_start(), 64, pad=True) yield vad, data @@ -211,9 +208,7 @@ class Malfind(interfaces.plugins.PluginInterface): file_output = file_handle.preferred_filename except (exceptions.InvalidAddressException, OverflowError) as excp: vollog.debug( - "Unable to dump PE with pid {0}.{1:#x}: {2}".format( - proc.UniqueProcessId, vad.get_start(), excp - ) + f"Unable to dump PE with pid {proc.UniqueProcessId}.{vad.get_start():#x}: {excp}" ) yield ( diff --git a/volatility3/framework/plugins/windows/memmap.py b/volatility3/framework/plugins/windows/memmap.py index b5c9a211e..62ab3c510 100644 --- a/volatility3/framework/plugins/windows/memmap.py +++ b/volatility3/framework/plugins/windows/memmap.py @@ -53,9 +53,7 @@ class Memmap(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue @@ -80,11 +78,7 @@ class Memmap(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: file_output = "Error outputting to file" vollog.debug( - "Unable to write {}'s address {} to {}".format( - proc_layer_name, - offset, - file_handle.preferred_filename, - ) + f"Unable to write {proc_layer_name}'s address {offset} to {file_handle.preferred_filename}" ) yield ( diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index feea78ece..c4d05e634 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -275,19 +275,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): display_data = True if display_data: - for record in cls.parse_data_record( + yield from cls.parse_data_record( mft_record, attr, record_map, return_first_record - ): - yield record + ) def _generator(self): - for record in self.enumerate_mft_records( + yield from self.enumerate_mft_records( self.context, self.config_path, self.config["primary"], self.parse_mft_records, - ): - yield record + ) def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index ba45834d5..85eb474a8 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -2,14 +2,14 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import List, Iterable, Generator +from typing import Generator, Iterable, List, Optional -from volatility3.framework import exceptions, interfaces, constants, renderers +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, pedump +from volatility3.plugins.windows import pedump, pslist vollog = logging.getLogger(__name__) @@ -104,12 +104,11 @@ class Modules(interfaces.plugins.PluginInterface): try: BaseDllName = mod.BaseDllName.get_string() + if self.config["name"] and self.config["name"] not in BaseDllName: + continue except exceptions.InvalidAddressException: BaseDllName = interfaces.renderers.BaseAbsentValue() - if self.config["name"] and self.config["name"] not in BaseDllName: - continue - try: FullDllName = mod.FullDllName.get_string() except exceptions.InvalidAddressException: @@ -134,7 +133,7 @@ class Modules(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - pids: List[int] = None, + pids: Optional[List[int]] = None, ) -> Generator[str, None, None]: """Build a cache of possible virtual layers, in priority starting with the primary/kernel layer. Then keep one layer per session by cycling @@ -165,26 +164,42 @@ class Modules(interfaces.plugins.PluginInterface): # create the session space object in the process' own layer. # not all processes have a valid session pointer. - session_space = context.object( - symbol_table + constants.BANG + "_MM_SESSION_SPACE", - layer_name=layer_name, - offset=proc.Session, - ) + try: + session_space = context.object( + symbol_table + constants.BANG + "_MM_SESSION_SPACE", + layer_name=layer_name, + offset=proc.Session, + ) + session_id = session_space.SessionId - if session_space.SessionId in seen_ids: + except exceptions.SymbolError: + # In Windows 11 24H2, the _MM_SESSION_SPACE type was + # replaced with _PSP_SESSION_SPACE, and the kernel PDB + # doesn't contain information about its members (otherwise, + # we would just fall back to the new type). However, it + # appears to be, for our purposes, functionally identical + # to the _MM_SESSION_SPACE. Because _MM_SESSION_SPACE + # stores its session ID at offset 8 as an unsigned long, we + # create an unsigned long at that offset and use that + # instead. + session_id = context.object( + layer_name=layer_name, + object_type=symbol_table + constants.BANG + "unsigned long", + offset=proc.Session + 8, + ) + + if session_id in seen_ids: continue except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, - "Process {} does not have a valid Session or a layer could not be constructed for it".format( - proc_id - ), + f"Process {proc_id} does not have a valid Session or a layer could not be constructed for it", ) continue # save the layer if we haven't seen the session yet - seen_ids.append(session_space.SessionId) + seen_ids.append(session_id) yield proc_layer_name @classmethod @@ -250,8 +265,7 @@ class Modules(interfaces.plugins.PluginInterface): object_type=type_name, offset=list_entry.vol.offset - reloff, absolute=True ) - for mod in module.InLoadOrderLinks: - yield mod + yield from module.InLoadOrderLinks def run(self): return renderers.TreeGrid( diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 66a24da5a..162031104 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -161,20 +161,15 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): raise NotImplementedError( "Kernel Debug Structure version format not supported!" ) - except: - # unsure what to raise here. Also, it might be useful to add some kind of fallback, + except Exception: + # FIXME: unsure what to raise here. Also, it might be useful to add some kind of fallback, # either to a user-provided version or to another method to determine tcpip.sys's version raise exceptions.VolatilityException( "Kernel Debug Structure missing VERSION/KUSER structure, unable to determine Windows version!" ) vollog.debug( - "Determined OS Version: {}.{} {}.{}".format( - kuser.NtMajorVersion, - kuser.NtMinorVersion, - vers.MajorVersion, - vers.MinorVersion, - ) + f"Determined OS Version: {kuser.NtMajorVersion}.{kuser.NtMinorVersion} {vers.MajorVersion}.{vers.MinorVersion}" ) if nt_major_version == 10 and arch == "x64": @@ -272,9 +267,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if ver: tcpip_mod_version = ver[3] vollog.debug( - "Determined tcpip.sys's FileVersion: {}".format( - tcpip_mod_version - ) + f"Determined tcpip.sys's FileVersion: {tcpip_mod_version}" ) else: vollog.debug("Could not determine tcpip.sys's FileVersion.") @@ -316,12 +309,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): else: raise NotImplementedError( - "This version of Windows is not supported: {}.{} {}.{}!".format( - nt_major_version, - nt_minor_version, - vers.MajorVersion, - vers_minor_version, - ) + f"This version of Windows is not supported: {nt_major_version}.{nt_minor_version} {vers.MajorVersion}.{vers_minor_version}!" ) vollog.debug(f"Determined symbol filename: {filename}") @@ -510,17 +498,8 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for i in row_data ] description = ( - "Network connection: Process {} {} Local Address {}:{} " - "Remote Address {}:{} State {} Protocol {} ".format( - row_data[7], - row_data[8], - row_data[2], - row_data[3], - row_data[4], - row_data[5], - row_data[6], - row_data[1], - ) + f"Network connection: Process {row_data[7]} {row_data[8]} Local Address {row_data[2]}:{row_data[3]} " + f"Remote Address {row_data[4]}:{row_data[5]} State {row_data[6]} Protocol {row_data[1]} " ) yield (description, timeliner.TimeLinerType.CREATED, row_data[9]) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 0908767fc..902be5fc8 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -111,8 +111,21 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The list of indices at which a 1 was found. """ ret = [] + # This value is broken in many samples and was causing essentially infinite loops + # Testing showed that 8192 is the current size across all Windows versions + # We give some leeway in case it increases in later versions, while still keeping it sane + # The problematic samples had values that looked like addresses, so in the billions + if bitmap_size_in_byte > 8192 * 10: + return ret + for idx in range(bitmap_size_in_byte): - current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[0] + try: + current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[ + 0 + ] + except exceptions.InvalidAddressException: + continue + current_offs = idx * 8 for bit in range(8): if current_byte & (1 << bit) != 0: @@ -154,32 +167,37 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) else: # invalid argument. - return None + return vollog.debug(f"Current Port: {port}") # the given port serves as a shifted index into the port pool lists list_index = port >> 8 truncated_port = port & 0xFF - # constructing port_pool object here so callers don't have to - port_pool = context.object( - net_symbol_table + constants.BANG + "_INET_PORT_POOL", - layer_name=layer_name, - offset=port_pool_addr, - ) + try: + # constructing port_pool object here so callers don't have to + port_pool = context.object( + net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name=layer_name, + offset=port_pool_addr, + ) + # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) + inpa = port_pool.PortAssignments[list_index] - # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) - inpa = port_pool.PortAssignments[list_index] - - # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry - assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry + assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + except exceptions.InvalidAddressException: + return if not assignment: - return None + return # the value within assignment.Entry is a) masked and b) points inside of the network object # first decode the pointer - netw_inside = cls._decode_pointer(assignment.Entry) + try: + netw_inside = cls._decode_pointer(assignment.Entry) + except exceptions.InvalidAddressException: + return if netw_inside: # if the value is valid, calculate the actual object address by subtracting the offset @@ -188,16 +206,30 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) yield curr_obj + try: + next_obj_address = cls._decode_pointer(curr_obj.Next) + except exceptions.InvalidAddressException: + return + # if the same port is used on different interfaces multiple objects are created # those can be found by following the pointer within the object's `Next` field until it is empty - while curr_obj.Next: - curr_obj = context.object( - obj_name, - layer_name=layer_name, - offset=cls._decode_pointer(curr_obj.Next) - ptr_offset, - ) + while next_obj_address: + try: + curr_obj = context.object( + obj_name, + layer_name=layer_name, + offset=next_obj_address - ptr_offset, + ) + except exceptions.InvalidAddressException: + return + yield curr_obj + try: + next_obj_address = cls._decode_pointer(curr_obj.Next) + except exceptions.InvalidAddressException: + return + @classmethod def get_tcpip_module( cls, @@ -243,16 +275,25 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The hash table entries which are _not_ empty """ # we are looking for entries whose values are not their own address + # smear sanity check from mass testing + if ht_length > 4096: + return + for index in range(ht_length): current_addr = ht_offset + index * alignment - current_pointer = context.object( - net_symbol_table + constants.BANG + "pointer", - layer_name=layer_name, - offset=current_addr, - ) + try: + current_pointer = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=current_addr, + ) + except exceptions.InvalidAddressException: + continue + # check if addr of pointer is equal to the value pointed to if current_pointer.vol.offset == current_pointer: continue + yield current_pointer @classmethod @@ -292,11 +333,15 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): tcpip_symbol_table + constants.BANG + "PartitionCount" ).address - part_table_addr = context.object( - net_symbol_table + constants.BANG + "pointer", - layer_name=layer_name, - offset=tcpip_module_offset + part_table_symbol, - ) + try: + part_table_addr = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=tcpip_module_offset + part_table_symbol, + ) + except exceptions.InvalidAddressException: + vollog.debug("`PartitionTable` not present in memory.") + return # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects part_table = context.object( @@ -304,23 +349,41 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): layer_name=layer_name, offset=part_table_addr, ) - part_count = int.from_bytes( - context.layers[layer_name].read(tcpip_module_offset + part_count_symbol, 1), - "little", - ) + + try: + part_count = int.from_bytes( + context.layers[layer_name].read( + tcpip_module_offset + part_count_symbol, 1 + ), + "little", + ) + except exceptions.InvalidAddressException: + vollog.debug("`PartitionCount` not present in memory.") + return + part_table.Partitions.count = part_count vollog.debug( - "Found TCP connection PartitionTable @ 0x{:x} (partition count: {})".format( - part_table_addr, part_count - ) + f"Found TCP connection PartitionTable @ 0x{part_table_addr:x} (partition count: {part_count})" ) entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset( "ListEntry" ) - for ctr, partition in enumerate(part_table.Partitions): + + try: + partitions = part_table.Partitions + except exceptions.InvalidAddressException: + vollog.debug("Partitions member not present in memory") + return + + for ctr, partition in enumerate(partitions): vollog.debug(f"Parsing partition {ctr}") - if partition.Endpoints.NumEntries > 0: + try: + num_entries = partition.Endpoints.NumEntries + except exceptions.InvalidAddressException: + continue + + if num_entries > 0: for endpoint_entry in cls.parse_hashtable( context, layer_name, @@ -404,6 +467,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): upp_symbol = context.symbol_space.get_symbol( tcpip_symbol_table + constants.BANG + "UdpPortPool" ).address + upp_addr = context.object( net_symbol_table + constants.BANG + "pointer", layer_name=layer_name, @@ -490,18 +554,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ # first, TCP endpoints by parsing the partition table - for endpoint in cls.parse_partitions( - context, - layer_name, - net_symbol_table, - tcpip_symbol_table, - tcpip_module_offset, - ): - yield endpoint - - # then, towards the UDP and TCP port pools - # first, find their addresses - upp_addr, tpp_addr = cls.find_port_pools( + yield from cls.parse_partitions( context, layer_name, net_symbol_table, @@ -509,6 +562,19 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): tcpip_module_offset, ) + # then, towards the UDP and TCP port pools + # first, find their addresses + try: + upp_addr, tpp_addr = cls.find_port_pools( + context, + layer_name, + net_symbol_table, + tcpip_symbol_table, + tcpip_module_offset, + ) + except (exceptions.SymbolError, exceptions.InvalidAddressException): + vollog.debug("Unable to reconstruct port pools") + # create port pool objects at the detected address and parse the port bitmap upp_obj = context.object( net_symbol_table + constants.BANG + "_INET_PORT_POOL", @@ -624,9 +690,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): proto = "TCPv6" else: vollog.debug( - "TCP Endpoint @ 0x{:2x} has unknown address family 0x{:x}".format( - netw_obj.vol.offset, netw_obj.get_address_family() - ) + f"TCP Endpoint @ 0x{netw_obj.vol.offset:2x} has unknown address family 0x{netw_obj.get_address_family():x}" ) proto = "TCPv?" diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 955098d6b..21e657ab3 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -158,7 +158,7 @@ class PESymbolFinder: class PDBSymbolFinder(PESymbolFinder): """ - PESymbolFinder implementation for PDB modules + PESymbolFinder implementation for PDB modules """ def _do_get_address(self, name: str) -> Optional[int]: @@ -195,7 +195,7 @@ class PDBSymbolFinder(PESymbolFinder): class ExportSymbolFinder(PESymbolFinder): """ - PESymbolFinder implementation for PDB modules + PESymbolFinder implementation for PDB modules """ def _get_name(self, export: pefile.ExportData) -> Optional[str]: @@ -300,7 +300,7 @@ class PESymbols(interfaces.plugins.PluginInterface): base_address: int, ) -> Optional[pefile.PE]: """ - Attempts to pefile object from the bytes of the PE file + Attempts to create a pefile object from the bytes of the PE file Args: pe_table_name: name of the pe types table @@ -645,7 +645,7 @@ class PESymbols(interfaces.plugins.PluginInterface): and wanted_addresses_identifier not in wanted_symbols ): vollog.warning( - f"Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." + "Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." ) return diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 858d0615a..5107cb48b 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -64,30 +64,27 @@ class PEDump(interfaces.plugins.PluginInterface): """ Returns the filename of the dump file or None """ - try: - file_handle = open_method(file_name) + with open_method(file_name) as file_handle: + try: + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=base, - layer_name=layer_name, - ) + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + OSError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - IOError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") - return None - finally: - file_handle.close() - - return file_handle.preferred_filename + return file_handle.preferred_filename @classmethod def dump_ldr_entry( @@ -96,7 +93,7 @@ class PEDump(interfaces.plugins.PluginInterface): pe_table_name: str, ldr_entry: interfaces.objects.ObjectInterface, open_method: Type[interfaces.plugins.FileHandlerInterface], - layer_name: str = None, + layer_name: Optional[str] = None, prefix: str = "", ) -> Optional[str]: """Extracts the PE file referenced an LDR_DATA_TABLE_ENTRY (DLL, kernel module) instance @@ -119,12 +116,7 @@ class PEDump(interfaces.plugins.PluginInterface): if layer_name is None: layer_name = ldr_entry.vol.layer_name - file_name = "{}{}.{:#x}.{:#x}.dmp".format( - prefix, - ntpath.basename(name), - ldr_entry.vol.offset, - ldr_entry.DllBase, - ) + file_name = f"{prefix}{ntpath.basename(name)}.{ldr_entry.vol.offset:#x}.{ldr_entry.DllBase:#x}.dmp" return cls.dump_pe( context, @@ -146,11 +138,7 @@ class PEDump(interfaces.plugins.PluginInterface): pid: int, base: int, ) -> Optional[str]: - file_name = "PE.{:#x}.{:d}.{:#x}.dmp".format( - proc_offset, - pid, - base, - ) + file_name = f"PE.{proc_offset:#x}.{pid:d}.{base:#x}.dmp" return PEDump.dump_pe( context, pe_table_name, layer_name, open_method, file_name, base @@ -227,11 +215,11 @@ class PEDump(interfaces.plugins.PluginInterface): ) if self.config["kernel_module"] and self.config["pid"]: - vollog.error("Only --kernel_module or --pid should be set. Not both") + vollog.error("Only 'kernel-module' or 'pid' should be set, not both") return if not self.config["kernel_module"] and not self.config["pid"]: - vollog.error("--kernel_module or --pid must be set") + vollog.error("Either 'kernel-module' or 'pid' argument must be set") return if self.config["kernel_module"]: diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 8c56d202d..efde09638 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -183,7 +183,7 @@ class PoolScanner(plugins.PluginInterface): @staticmethod def builtin_constraints( - symbol_table: str, tags_filter: List[bytes] = None + symbol_table: str, tags_filter: Optional[List[bytes]] = None ) -> List[PoolConstraint]: """Get built-in PoolConstraints given a list of pool tags. diff --git a/volatility3/framework/plugins/windows/privileges.py b/volatility3/framework/plugins/windows/privileges.py index 0370dfc92..7b4d00205 100644 --- a/volatility3/framework/plugins/windows/privileges.py +++ b/volatility3/framework/plugins/windows/privileges.py @@ -39,7 +39,7 @@ class Privs(interfaces.plugins.PluginInterface): ) # Get service sids dictionary (we need only the service sids). - with open(sids_json_file_name, "r") as file_handle: + with open(sids_json_file_name) as file_handle: temp_json = json.load(file_handle)["privileges"] self.privilege_info = { int(priv_num): temp_json[priv_num] for priv_num in temp_json diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 478cc8b1b..579a235d8 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Iterator, List, Type +from typing import Callable, Iterator, List, Optional, Type from volatility3.framework import renderers, interfaces, layers, exceptions, constants from volatility3.framework.configuration import requirements @@ -114,7 +114,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def create_pid_filter( - cls, pid_list: List[int] = None, exclude: bool = False + cls, pid_list: Optional[List[int]] = None, exclude: bool = False ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process IDs. @@ -126,15 +126,24 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Returns: Filter function for passing to the `list_processes` method """ - filter_func = lambda _: False + + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 pid_list = pid_list or [] filter_list = [x for x in pid_list if x is not None] if filter_list: if exclude: - filter_func = lambda x: x.UniqueProcessId in filter_list + + def filter_func(x): + return x.UniqueProcessId in filter_list + else: - filter_func = lambda x: x.UniqueProcessId not in filter_list + + def filter_func(x): + return x.UniqueProcessId not in filter_list + return filter_func @classmethod @@ -162,7 +171,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def create_name_filter( - cls, name_list: List[str] = None, exclude: bool = False + cls, name_list: Optional[List[str]] = None, exclude: bool = False ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process names. @@ -173,20 +182,24 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Returns: Filter function for passing to the `list_processes` method """ - filter_func = lambda _: False + + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 name_list = name_list or [] filter_list = [x for x in name_list if x is not None] if filter_list: if exclude: - filter_func = ( - lambda x: utility.array_to_string(x.ImageFileName) in filter_list - ) + + def filter_func(x): + return utility.array_to_string(x.ImageFileName) in filter_list + else: - filter_func = ( - lambda x: utility.array_to_string(x.ImageFileName) - not in filter_list - ) + + def filter_func(x): + return utility.array_to_string(x.ImageFileName) not in filter_list + return filter_func @classmethod diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 5ce470cd8..cdf344ee6 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -89,7 +89,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, layer_name: str, - offset: int = None, + offset: Optional[int] = None, physical: bool = True, exclude: bool = False, ) -> Callable[[interfaces.objects.ObjectInterface], bool]: @@ -102,29 +102,38 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Returns: Filter function to be passed to the list of processes. """ - filter_func = lambda _: False + + def filter_func(_): + return False if offset: if physical: if exclude: - filter_func = ( - lambda proc: cls.physical_offset_from_virtual( - context, layer_name, proc + + def filter_func(proc): + return ( + cls.physical_offset_from_virtual(context, layer_name, proc) + == offset ) - == offset - ) + else: - filter_func = ( - lambda proc: cls.physical_offset_from_virtual( - context, layer_name, proc + + def filter_func(proc): + return ( + cls.physical_offset_from_virtual(context, layer_name, proc) + != offset ) - != offset - ) + else: if exclude: - filter_func = lambda proc: proc.vol.offset == offset + + def filter_func(proc): + return proc.vol.offset == offset + else: - filter_func = lambda proc: proc.vol.offset != offset + + def filter_func(proc): + return proc.vol.offset != offset return filter_func diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 6c845bf81..b5ddd2ee5 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -14,7 +14,6 @@ from volatility3.plugins.windows import ( info, pslist, psscan, - sessions, thrdscan, ) @@ -26,7 +25,7 @@ class PsXView(plugins.PluginInterface): identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this plugin's output in a terminal.""" - # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality + # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the functionality # which the original plugin used to do it. # The sessions method is omitted because it begins with the list of processes found by Pslist anyway. @@ -219,7 +218,7 @@ class PsXView(plugins.PluginInterface): name = self._proc_name_to_string(proc) exit_time = proc.get_exit_time() - if type(exit_time) != datetime.datetime: + if type(exit_time) is not datetime.datetime: exit_time = "" else: exit_time = str(exit_time) diff --git a/volatility3/framework/plugins/windows/registry/hivelist.py b/volatility3/framework/plugins/windows/registry/hivelist.py index ddc9c1855..91a99a9fb 100644 --- a/volatility3/framework/plugins/windows/registry/hivelist.py +++ b/volatility3/framework/plugins/windows/registry/hivelist.py @@ -232,10 +232,8 @@ class HiveList(interfaces.plugins.PluginInterface): for hive in hg: if hive.vol.offset in seen: vollog.debug( - "Hivelist found an already seen offset {} while " - "traversing forwards, this should not occur".format( - hex(hive.vol.offset) - ) + f"Hivelist found an already seen offset {hex(hive.vol.offset)} while " + "traversing forwards, this should not occur" ) break seen.add(hive.vol.offset) @@ -249,18 +247,14 @@ class HiveList(interfaces.plugins.PluginInterface): forward_invalid = hg.invalid if forward_invalid: vollog.debug( - "Hivelist failed traversing the list forwards at {}, traversing backwards".format( - hex(forward_invalid) - ) + f"Hivelist failed traversing the list forwards at {hex(forward_invalid)}, traversing backwards" ) hg = HiveGenerator(cmhive, forward=False) for hive in hg: if hive.vol.offset in seen: vollog.debug( - "Hivelist found an already seen offset {} while " - "traversing backwards, list walking met in the middle".format( - hex(hive.vol.offset) - ) + f"Hivelist found an already seen offset {hex(hive.vol.offset)} while " + "traversing backwards, list walking met in the middle" ) break seen.add(hive.vol.offset) @@ -281,10 +275,8 @@ class HiveList(interfaces.plugins.PluginInterface): # by walking the list, so revert to scanning, and walk the list forwards and backwards from each # found hive vollog.debug( - "Hivelist failed traversing backwards at {}, a different " - "location from forwards, revert to scanning".format( - hex(backward_invalid) - ) + f"Hivelist failed traversing backwards at {hex(backward_invalid)}, a different " + "location from forwards, revert to scanning" ) for hive in hivescan.HiveScan.scan_hives( context, layer_name, symbol_table @@ -320,9 +312,7 @@ class HiveList(interfaces.plugins.PluginInterface): yield linked_hive except exceptions.InvalidAddressException: vollog.debug( - "InvalidAddressException when traversing hive {} found from scan, skipping".format( - hex(hive.vol.offset) - ) + f"InvalidAddressException when traversing hive {hex(hive.vol.offset)} found from scan, skipping" ) def run(self) -> renderers.TreeGrid: diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 4fe3f97fb..ed926805b 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import List, Sequence, Iterable, Tuple, Union +from typing import List, Optional, Sequence, Iterable, Tuple, Union from volatility3.framework import objects, renderers, exceptions, interfaces, constants from volatility3.framework.configuration import requirements @@ -51,7 +51,7 @@ class PrintKey(interfaces.plugins.PluginInterface): def key_iterator( cls, hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, + node_path: Optional[Sequence[objects.StructType]] = None, recurse: bool = False, ) -> Iterable[ Tuple[ @@ -121,7 +121,7 @@ class PrintKey(interfaces.plugins.PluginInterface): def _printkey_iterator( self, hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, + node_path: Optional[Sequence[objects.StructType]] = None, recurse: bool = False, ): """Method that wraps the more generic key_iterator, to provide output @@ -242,8 +242,8 @@ class PrintKey(interfaces.plugins.PluginInterface): self, layer_name: str, symbol_table: str, - hive_offsets: List[int] = None, - key: str = None, + hive_offsets: Optional[List[int]] = None, + key: Optional[str] = None, recurse: bool = False, ): for hive in hivelist.HiveList.list_hives( diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index bd832b20c..932ee9d6f 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -39,7 +39,7 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac os.path.join(os.path.dirname(__file__), "userassist.json"), "rb" ) as fp: self._folder_guids = json.load(fp) - except IOError: + except OSError: vollog.error("Usersassist data file not found") @classmethod @@ -308,9 +308,7 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac ) except exceptions.InvalidAddressException as excp: vollog.debug( - "Invalid address identified in lower layer {}: {}".format( - excp.layer_name, excp.invalid_address - ) + f"Invalid address identified in lower layer {excp.layer_name}: {excp.invalid_address}" ) except KeyError: vollog.debug( diff --git a/volatility3/framework/plugins/windows/scheduled_tasks.py b/volatility3/framework/plugins/windows/scheduled_tasks.py index 277a0d856..6dd5613c4 100644 --- a/volatility3/framework/plugins/windows/scheduled_tasks.py +++ b/volatility3/framework/plugins/windows/scheduled_tasks.py @@ -270,7 +270,6 @@ class _ScheduledTasksReader(io.BytesIO): return val def read_aligned_bstring_expand_sz(self) -> Optional[str]: - # type: () -> Optional[str] sz = self.read_aligned_u4() if sz is None: return None diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 6afaf4356..b8e9b5bd7 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -17,8 +17,6 @@ from volatility3.framework.symbols.windows.extensions import pe, shimcache from volatility3.plugins import timeliner from volatility3.plugins.windows import modules, pslist, vadinfo -# from volatility3.plugins.windows import pslist, vadinfo, modules - vollog = logging.getLogger(__name__) @@ -146,7 +144,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf context, layer_name, kernel_symbol_table ): pid = process.UniqueProcessId - vollog.debug("checking process %d" % pid) + vollog.debug("checking process %d", pid) for vad in vadinfo.VadInfo.list_vads( process, lambda x: x.get_tag() == b"Vad " and x.Protection == 4 ): @@ -285,10 +283,9 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf if not shim_head: return - for shim_entry in shim_head.ListEntry.to_list( + yield from shim_head.ListEntry.to_list( shimcache_symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", "ListEntry" - ): - yield shim_entry + ) @classmethod def try_get_shim_head_at_offset( @@ -333,7 +330,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf eresource_rel_off = ersrc_size + ((offset - ersrc_size) % ersrc_alignment) eresource_offset = offset - eresource_rel_off - vollog.debug("Constructing ERESOURCE at %s" % hex(eresource_offset)) + vollog.debug(f"Constructing ERESOURCE at {hex(eresource_offset)}") eresource = context.object( kernel_symbol_table + constants.BANG + "_ERESOURCE", layer_name, diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index d321c2cc0..6ae07381a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -172,9 +172,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.debug( - "Unable to construct cSystems array at given offset: {:x}".format( - array_start - ) + f"Unable to construct cSystems array at given offset: {array_start:x}" ) array = None @@ -284,16 +282,13 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): for proc in proc_list: try: - proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() return proc, proc_layer_name except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None, None @@ -435,15 +430,20 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): # we do not want to fail just because the count is not in memory # 16 was the size on samples I tested, so I chose it as the default + count = 16 + if target_address: - count = int.from_bytes( - self.context.layers[proc_layer_name].read( - target_address, 4 - ), - "little", - ) - else: - count = 16 + try: + count = int.from_bytes( + self.context.layers[proc_layer_name].read( + target_address, 4 + ), + "little", + ) + except exceptions.InvalidAddressException: + vollog.debug( + "Unable to read `cCsystems`. Defaulting to 16." + ) found_count = True diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 0eaa65884..b8dea0cdd 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -170,9 +170,7 @@ class Strings(interfaces.plugins.PluginInterface): proc_layer_name = process.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py new file mode 100644 index 000000000..cec51ed37 --- /dev/null +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -0,0 +1,148 @@ +import logging + +from typing import Dict +import functools + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +import volatility3.plugins.windows.pslist as pslist +import volatility3.plugins.windows.threads as threads +import volatility3.plugins.windows.pe_symbols as pe_symbols + +from volatility3.framework.objects import utility + +vollog = logging.getLogger(__name__) + + +class SuspendedThreads(interfaces.plugins.PluginInterface): + """Enumerates suspended threads.""" + + _required_framework_version = (2, 13, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="threads", component=threads.Threads, version=(1, 0, 0) + ), + ] + + def _generator(self): + """ + The goal of this plugin is to report on threads that are suspended + + Legitimate programs can start threads suspended but then will later resume them + + Subsets of malware techniques, such as EDR evasion and process hollowing, + create suspended threads and do not resume them. These are the threads that this + plugin is designed to catch. + + See the whitepaper from our DEF CON 2024 presentation for more details: + + https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + """ + kernel = self.context.modules[self.config["kernel"]] + + vads_cache: Dict[int, pe_symbols.PESymbols.ranges_type] = {} + + proc_modules = None + + # walk the threads of each process checking for suspended threads + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ): + for thread in threads.Threads.list_threads(kernel, proc): + try: + # we only care if the thread is suspended + if thread.Tcb.SuspendCount == 0: + continue + + # 4 == terminated + if thread.Tcb.State == 4: + continue + + owner_proc = thread.owning_process() + owner_proc_pid = thread.Cid.UniqueProcess + owner_proc_name = utility.array_to_string(owner_proc.ImageFileName) + thread_tid = thread.Cid.UniqueThread + thread_start_addr = thread.StartAddress + thread_win32_addr = thread.Win32StartAddress + except exceptions.InvalidAddressException: + continue + + # Nothing useful to report if a process doesn't have VADs.. Also a sign of smear/terminated + vads = pe_symbols.PESymbols.get_vads_for_process_cache( + vads_cache, owner_proc + ) + if not vads: + continue + + # Only compute this if needed as its expensive and 99.9% of samples + # will not have suspended threads + if not proc_modules: + proc_modules = pe_symbols.PESymbols.get_process_modules( + self.context, kernel.layer_name, kernel.symbol_table_name, None + ) + + path_and_symbol = functools.partial( + pe_symbols.PESymbols.path_and_symbol_for_address, + self.context, + self.config_path, + proc_modules, + ) + + start_file, start_sym = path_and_symbol(vads, thread_start_addr) + win32_file, win32_sym = path_and_symbol(vads, thread_win32_addr) + + # the only false positive found in mass scanning of samples + if start_file and start_file.endswith("\\WorkFoldersShell.dll"): + continue + + if win32_file and win32_file.endswith("\\WorkFoldersShell.dll"): + continue + + yield ( + 0, + ( + owner_proc_name, + owner_proc_pid, + thread_tid, + start_file or renderers.NotAvailableValue(), + start_sym or renderers.NotAvailableValue(), + format_hints.Hex(thread_start_addr), + win32_file or renderers.NotAvailableValue(), + win32_sym or renderers.NotAvailableValue(), + format_hints.Hex(thread_win32_addr), + ), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("TID", int), + ("StartFile", str), + ("StartSymbol", str), + ("StartAddress", format_hints.Hex), + ("Win32StartFile", str), + ("Win32StartSymbol", str), + ("Win32StartAddress", format_hints.Hex), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index a59581063..8a64084c5 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -18,6 +18,7 @@ vollog = logging.getLogger(__name__) class SvcList(svcscan.SvcScan): """Lists services contained with the services.exe doubly linked list of services""" + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) def __init__(self, *args, **kwargs): @@ -41,7 +42,7 @@ class SvcList(svcscan.SvcScan): @classmethod def _get_exe_range(cls, proc) -> Optional[Tuple[int, int]]: """ - Returns a tuple of starting,ending address for + Returns a tuple of starting address and size of the VAD containing services.exe """ @@ -85,9 +86,7 @@ class SvcList(svcscan.SvcScan): layer_name = proc.add_process_layer() except exceptions.InvalidAddressException: vollog.warning( - "Unable to access memory of services.exe running with PID: {}".format( - proc.UniqueProcessId - ) + f"Unable to access memory of services.exe running with PID: {proc.UniqueProcessId}" ) continue @@ -105,11 +104,10 @@ class SvcList(svcscan.SvcScan): scanner=scanners.BytesScanner(needle=b"Sc27"), sections=exe_range, ): - for record in cls.enumerate_vista_or_later_header( + yield from cls.enumerate_vista_or_later_header( context, service_table_name, service_binary_dll_map, layer_name, offset, - ): - yield record + ) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index ca390561f..bd477ba27 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -26,13 +26,9 @@ from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) -ServiceBinaryInfo = NamedTuple( - "ServiceBinaryInfo", - [ - ("dll", Union[str, interfaces.renderers.BaseAbsentValue]), - ("binary", Union[str, interfaces.renderers.BaseAbsentValue]), - ], -) +class ServiceBinaryInfo(NamedTuple): + dll: Union[str, interfaces.renderers.BaseAbsentValue] + binary: Union[str, interfaces.renderers.BaseAbsentValue] class SvcScan(interfaces.plugins.PluginInterface): @@ -306,9 +302,7 @@ class SvcScan(interfaces.plugins.PluginInterface): proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index b812a15ff..c0963e754 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -82,7 +82,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ethread.get_exit_time() ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object except exceptions.InvalidAddressException: - vollog.debug("Thread invalid address {:#x}".format(ethread.vol.offset)) + vollog.debug(f"Thread invalid address {ethread.vol.offset:#x}") return None return ( diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index 98a3169a5..84daa8595 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -3,7 +3,7 @@ # import logging -from typing import Callable, Iterable, List, Generator +from typing import Iterable, List, Generator from volatility3.framework import interfaces, constants from volatility3.framework.configuration import requirements @@ -82,5 +82,4 @@ class Threads(thrdscan.ThrdScan): symbol_table=symbol_table_name, filter_func=filter_func, ): - for thread in cls.list_threads(module, proc): - yield thread + yield from cls.list_threads(module, proc) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index d49c28784..cd8101a95 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -14,7 +14,7 @@ from volatility3.framework import ( ) from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols.windows import versions +from volatility3.framework.symbols.windows import versions, extensions from volatility3.plugins.windows import ssdt, kpcrs vollog = logging.getLogger(__name__) @@ -24,7 +24,7 @@ class Timers(interfaces.plugins.PluginInterface): """Print kernel timers and associated module DPCs""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -49,7 +49,7 @@ class Timers(interfaces.plugins.PluginInterface): kernel_module_name: str, layer_name: str, symbol_table: str, - ) -> Iterable[Tuple[str, int, str]]: + ) -> Iterable[extensions.KTIMER]: """Lists all kernel timers. Args: @@ -141,7 +141,7 @@ class Timers(interfaces.plugins.PluginInterface): if dpc.DeferredRoutine == 0: continue deferred_routine = dpc.DeferredRoutine - except Exception as e: + except Exception: continue module_symbols = list( diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index 1a1e59940..5b21225c8 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -191,7 +191,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): # gather processes on small_idx since these are the malware infected ones for pid, pname in cb[small_idx]: - ps.append("{:d}:{}".format(pid, pname)) + ps.append(f"{pid:d}:{pname}") proc_names = ", ".join(ps) diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py index 01e575818..077fe33cb 100644 --- a/volatility3/framework/plugins/windows/unloadedmodules.py +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -116,8 +116,7 @@ class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInt ) unloadedmodules_array.UnloadedDrivers.count = unloaded_count - for mod in unloadedmodules_array.UnloadedDrivers: - yield mod + yield from unloadedmodules_array.UnloadedDrivers def _generator(self): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 2c6ed4daf..0c4a8aaca 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -169,9 +169,7 @@ class VadInfo(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 4a06ed0c9..4930789d2 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -212,9 +212,7 @@ class VerInfo(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/virtmap.py b/volatility3/framework/plugins/windows/virtmap.py index 3f3f270e2..e02cca89e 100644 --- a/volatility3/framework/plugins/windows/virtmap.py +++ b/volatility3/framework/plugins/windows/virtmap.py @@ -138,8 +138,7 @@ class VirtMap(interfaces.plugins.PluginInterface): mapping = cls.determine_map(module) for entry in mapping: if "Unused" not in entry: - for value in mapping[entry]: - yield value + yield from mapping[entry] def run(self): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 43bb59a21..093edf8cc 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -83,14 +83,11 @@ class TreeNode(interfaces.renderers.TreeNode): raise TypeError( "Values must be a list of objects made up of simple types and number the same as the columns" ) - for index in range(len(self._treegrid.columns)): - column = self._treegrid.columns[index] + for index, column in enumerate(self._treegrid.columns): val = values[index] if not isinstance(val, (column.type, interfaces.renderers.BaseAbsentValue)): raise TypeError( - "Values item with index {} is the wrong type for column {} (got {} but expected {})".format( - index, column.name, type(val), column.type - ) + f"Values item with index {index} is the wrong type for column {column.name} (got {type(val)} but expected {column.type})" ) # TODO: Consider how to deal with timezone naive/aware datetimes (and alert plugin uses to be precise) # if isinstance(val, datetime.datetime): @@ -189,9 +186,7 @@ class TreeGrid(interfaces.renderers.TreeGrid): is_simple_type = issubclass(column_type, self.base_types) if not is_simple_type: raise TypeError( - "Column {}'s type is not a simple type: {}".format( - name, column_type.__class__.__name__ - ) + f"Column {name}'s type is not a simple type: {column_type.__class__.__name__}" ) converted_columns.append(interfaces.renderers.Column(name, column_type)) self.RowStructure = RowStructureConstructor( @@ -218,7 +213,7 @@ class TreeGrid(interfaces.renderers.TreeGrid): def populate( self, - function: interfaces.renderers.VisitorSignature = None, + function: Optional[interfaces.renderers.VisitorSignature] = None, initial_accumulator: Any = None, fail_on_errors: bool = True, ) -> Optional[Exception]: @@ -417,8 +412,7 @@ class ColumnSortKey(interfaces.renderers.ColumnSortKey): _index = None self._type = None self.ascending = ascending - for i in range(len(treegrid.columns)): - column = treegrid.columns[i] + for i, column in enumerate(treegrid.columns): if column.name.lower() == column_name.lower(): _index = i self._type = column.type @@ -434,10 +428,10 @@ class ColumnSortKey(interfaces.renderers.ColumnSortKey): value = datetime.datetime.min elif self._type in [int, float]: value = -1 - elif self._type == bool: + elif self._type is bool: value = False elif self._type in [str, renderers.Disassembly]: value = "-" - elif self._type == bytes: + elif self._type is bytes: value = b"" return value diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index 194e38099..d57c7e9f1 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -70,15 +70,21 @@ class MultiTypeData(bytes): ) -BinOrAbsent = lambda x: ( - Bin(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -HexOrAbsent = lambda x: ( - Hex(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -HexBytesOrAbsent = lambda x: ( - HexBytes(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -MultiTypeDataOrAbsent = lambda x: ( - MultiTypeData(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) +def BinOrAbsent(x): + return Bin(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def HexOrAbsent(x): + return Hex(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def HexBytesOrAbsent(x): + return HexBytes(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def MultiTypeDataOrAbsent(x): + return ( + MultiTypeData(x) + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else x + ) diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index a8753bd4d..87f2288d7 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -53,10 +53,10 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} - def clear_symbol_cache(self, table_name: str = None) -> None: + def clear_symbol_cache(self, table_name: Optional[str] = None) -> None: """Clears the symbol cache for the specified table name. If no table name is specified, the caches of all symbol tables are cleared.""" - table_list: List[interfaces.symbols.BaseSymbolTableInterface] = list() + table_list: List[interfaces.symbols.BaseSymbolTableInterface] = [] if table_name is None: table_list = list(self._dict.values()) else: @@ -81,7 +81,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): yield table + constants.BANG + symbol_name def get_symbols_by_location( - self, offset: int, size: int = 0, table_name: str = None + self, offset: int, size: int = 0, table_name: Optional[str] = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( @@ -128,7 +128,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self, producer: str, validator: Callable[[Optional[Tuple], Optional[datetime.datetime]], bool], - tables: List[str] = None, + tables: Optional[List[str]] = None, ) -> bool: """Verifies the producer metadata and version of tables diff --git a/volatility3/framework/symbols/generic/__init__.py b/volatility3/framework/symbols/generic/__init__.py index 9d6da5aa4..7dd00fa75 100644 --- a/volatility3/framework/symbols/generic/__init__.py +++ b/volatility3/framework/symbols/generic/__init__.py @@ -4,7 +4,7 @@ import random import string -from typing import Union +from typing import Optional, Union from volatility3.framework import objects, interfaces @@ -14,8 +14,8 @@ class GenericIntelProcess(objects.StructType): self, context: interfaces.context.ContextInterface, dtb: Union[int, interfaces.objects.ObjectInterface], - config_prefix: str = None, - preferred_name: str = None, + config_prefix: Optional[str] = None, + preferred_name: Optional[str] = None, ) -> str: """Constructs a new layer based on the process's DirectoryTableBase.""" diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 5f558bf12..6802af7d6 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -86,7 +86,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): config_path: str, name: str, isf_url: str, - native_types: interfaces.symbols.NativeTableInterface = None, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, validate: bool = True, class_types: Optional[ @@ -101,7 +101,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): Args: context: The volatility context for the symbol table config_path: The configuration path for the symbol table - name: The name for the symbol table (this is used in symbols e.g. table!symbol ) + name: The name for the symbol table (this is used in symbols e.g. table!symbol) isf_url: The URL pointing to the ISF file location native_types: The NativeSymbolTable that contains the native types for this symbol table table_mapping: A dictionary linking names referenced in the file with symbol tables in the context @@ -111,7 +111,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): """ # Check there are no obvious errors # Open the file and test the version - self._versions = dict([(x.version, x) for x in class_subclasses(ISFormatTable)]) + self._versions = dict((x.version, x) for x in class_subclasses(ISFormatTable)) with resources.ResourceAccessor().open(isf_url) as fp: reader = codecs.getreader("utf-8") json_object = json.load(reader(fp)) # type: ignore @@ -166,12 +166,12 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): format. An interface version such as Major.Minor.Patch means that Major - of the provider must be equal to that of the consumer, and the + of the provider must be equal to that of the consumer, and the provider (the JSON in this instance) must have a greater minor - (indicating that only additive changes have been made) than + (indicating that only additive changes have been made) than the consumer (in this case, the file reader). """ - major, minor, patch = [int(x) for x in version.split(".")] + major, minor, patch = (int(x) for x in version.split(".")) supported_versions = [x for x in versions if x[0] == major and x[1] >= minor] if not supported_versions: raise ValueError( @@ -319,7 +319,7 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass=ABCMeta): config_path: str, name: str, json_object: Any, - native_types: interfaces.symbols.NativeTableInterface = None, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, ) -> None: self._json_object = json_object diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..5aa27b964 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -76,7 +76,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 1) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -483,6 +483,22 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return kernel + @classmethod + def convert_fourcc_code(cls, code: int) -> str: + """Convert a fourcc integer back to its fourcc string representation. + + Args: + code: the numerical representation of the fourcc + + Returns: + The fourcc code string. + """ + + code_bytes_length = (code.bit_length() + 7) // 8 + return "".join( + [chr((code >> (i * 8)) & 0xFF) for i in range(code_bytes_length)] + ) + class IDStorage(ABC): """Abstraction to support both XArray and RadixTree""" @@ -629,8 +645,7 @@ class IDStorage(ABC): if self.is_valid_node(nodep): yield nodep else: - for child_node in self._iter_node(nodep, height - 1): - yield child_node + yield from self._iter_node(nodep, height - 1) def get_entries(self, root: interfaces.objects.ObjectInterface) -> Iterator[int]: """Walks the tree data structure @@ -659,8 +674,7 @@ class IDStorage(ABC): if self.is_valid_node(nodep): yield nodep else: - for child_node in self._iter_node(nodep, height): - yield child_node + yield from self._iter_node(nodep, height) class XArray(IDStorage): @@ -798,7 +812,7 @@ class RadixTree(IDStorage): return True -class PageCache(object): +class PageCache: """Linux Page Cache abstraction""" def __init__( diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 829622154..34d0fcba9 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -200,8 +200,7 @@ class module(generic.GenericIntelProcess): count=num_sects, ) - for attr in arr: - yield attr + yield from arr def get_elf_table_name(self): elf_table_name = intermed.IntermediateSymbolTable.create( @@ -237,8 +236,7 @@ class module(generic.GenericIntelProcess): count=self.num_symtab + 1, ) if self.section_strtab: - for sym in syms: - yield sym + yield from syms def get_symbols_names_and_addresses(self) -> Iterable[Tuple[str, int]]: """Get names and addresses for each symbol of the module @@ -310,7 +308,7 @@ class module(generic.GenericIntelProcess): class task_struct(generic.GenericIntelProcess): def add_process_layer( - self, config_prefix: str = None, preferred_name: str = None + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: """Constructs a new layer based on the process's DTB. @@ -635,6 +633,20 @@ class task_struct(generic.GenericIntelProcess): # root time namespace, not within the task's own time namespace return boottime + task_start_time_timedelta + def get_parent_pid(self) -> int: + """Returns the parent process ID (PPID) + + This method replicates the Linux kernel's `getppid` syscall behavior. + Avoid using `task.parent`; instead, use this function for accurate results. + """ + + if self.real_parent and self.real_parent.is_readable(): + ppid = self.real_parent.tgid + else: + ppid = 0 + + return ppid + class fs_struct(objects.StructType): def get_root_dentry(self): @@ -1469,7 +1481,7 @@ class mount(objects.StructType): def next_peer(self): table_name = self.vol.type_name.split(constants.BANG)[0] - mount_struct = "{0}{1}mount".format(table_name, constants.BANG) + mount_struct = f"{table_name}{constants.BANG}mount" offset = self._context.symbol_space.get_type( mount_struct ).relative_child_offset("mnt_share") @@ -2067,13 +2079,40 @@ class cred(objects.StructType): return int(value) @property - def euid(self): + def uid(self) -> int: + """Returns the real user ID + + Returns: + The real user ID value + """ + return self._get_cred_int_value("uid") + + @property + def gid(self) -> int: + """Returns the real user ID + + Returns: + The real user ID value + """ + return self._get_cred_int_value("gid") + + @property + def euid(self) -> int: """Returns the effective user ID + Returns: + The effective user ID value + """ + return self._get_cred_int_value("euid") + + @property + def egid(self) -> int: + """Returns the effective group ID + Returns: int: the effective user ID value """ - return self._get_cred_int_value("euid") + return self._get_cred_int_value("egid") class kernel_cap_struct(objects.StructType): @@ -2487,7 +2526,7 @@ class address_space(objects.StructType): class page(objects.StructType): @property - @functools.lru_cache() + @functools.lru_cache def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values @@ -2665,8 +2704,7 @@ class IDR(objects.StructType): id_storage = linux.IDStorage.choose_id_storage( self._context, kernel_module_name="kernel" ) - for page_addr in id_storage.get_entries(root=self.idr_rt): - yield page_addr + yield from id_storage.get_entries(root=self.idr_rt) def get_entries(self) -> Iterable[int]: """Walks the IDR and yield a pointer associated with each element. @@ -2684,8 +2722,7 @@ class IDR(objects.StructType): # Kernels < 4.11 get_entries_func = self._old_kernel_get_entries - for page_addr in get_entries_func(): - yield page_addr + yield from get_entries_func() class rb_root(objects.StructType): diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index c695ca77a..dc54a8371 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Iterator, Any, Iterable, List, Tuple, Set +from typing import Iterator, Any, Iterable, List, Optional, Tuple, Set from volatility3.framework import interfaces, objects, exceptions, constants from volatility3.framework.symbols import intermed @@ -97,7 +97,7 @@ class MacUtilities(interfaces.configuration.VersionableInterface): context: interfaces.context.ContextInterface, handlers: Iterator[Any], target_address, - kernel_module_name: str = None, + kernel_module_name: Optional[str] = None, ): mod_name = "UNKNOWN" symbol_name = "N/A" @@ -232,10 +232,9 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "tqh_first", "tqe_next", next_member, max_elements - ): - yield element + ) @classmethod def walk_list_head( @@ -244,10 +243,9 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "lh_first", "le_next", next_member, max_elements - ): - yield element + ) @classmethod def walk_slist( @@ -256,7 +254,6 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "slh_first", "sle_next", next_member, max_elements - ): - yield element + ) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 15fe7aeda..cc700f209 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -18,7 +18,7 @@ class proc(generic.GenericIntelProcess): return self.task.dereference().cast("task") def add_process_layer( - self, config_prefix: str = None, preferred_name: str = None + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: """Constructs a new layer based on the process's DTB. @@ -237,7 +237,7 @@ class vm_map_entry(objects.StructType): def get_path(self, context, config_prefix): node = self.get_vnode(context, config_prefix) - if type(node) == str and node == "sub_map": + if type(node) is str and node == "sub_map": ret = node elif node: path = [] diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 7e069e518..ea635f1f1 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -25,7 +25,7 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): return self._json_data.get("version", "") @property - def version(self) -> Optional[Tuple[int]]: + def version(self) -> Optional[Tuple[int, ...]]: """Returns the version of the ISF file producer""" version = self.version_string if not version: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 793e506c3..f12fd3f5b 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -405,10 +405,24 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the attached device's objects""" - device = self.AttachedDevice.dereference() + seen = set() + + try: + device = self.AttachedDevice.dereference() + except exceptions.InvalidAddressException: + return + while device: + if device.vol.offset in seen: + break + seen.add(device.vol.offset) + yield device - device = device.AttachedDevice.dereference() + + try: + device = device.AttachedDevice.dereference() + except exceptions.InvalidAddressException: + return class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): @@ -421,10 +435,24 @@ class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" - device = self.DeviceObject.dereference() + seen = set() + + try: + device = self.DeviceObject.dereference() + except exceptions.InvalidAddressException: + return + while device: + if device.vol.offset in seen: + return + seen.add(device.vol.offset) + yield device - device = device.NextDevice.dereference() + + try: + device = device.NextDevice.dereference() + except exceptions.InvalidAddressException: + return def is_valid(self) -> bool: """Determine if the object is valid.""" @@ -519,7 +547,8 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False - if not (1998 < ctime.year < 2030): + current_year = datetime.datetime.now().year + if not (1998 < ctime.year < current_year + 10): return False except exceptions.InvalidAddressException: @@ -692,7 +721,9 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return True - def add_process_layer(self, config_prefix: str = None, preferred_name: str = None): + def add_process_layer( + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None + ): """Constructs a new layer based on the process's DirectoryTableBase.""" parent_layer = self._context.layers[self.vol.layer_name] @@ -749,11 +780,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() - for entry in peb.Ldr.InLoadOrderModuleList.to_list( + yield from peb.Ldr.InLoadOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InLoadOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None @@ -762,11 +792,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() - for entry in peb.Ldr.InInitializationOrderModuleList.to_list( + yield from peb.Ldr.InInitializationOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InInitializationOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None @@ -775,11 +804,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() - for entry in peb.Ldr.InMemoryOrderModuleList.to_list( + yield from peb.Ldr.InMemoryOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InMemoryOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None @@ -797,7 +825,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return renderers.UnreadableValue() - def get_session_id(self): + def get_session_id(self) -> Union[int, interfaces.renderers.BaseAbsentValue]: try: if self.has_member("Session"): if self.Session == 0: @@ -813,23 +841,36 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): offset=kvo, native_layer_name=self.vol.native_layer_name, ) - session = ntkrnlmp.object( - object_type="_MM_SESSION_SPACE", offset=self.Session, absolute=True - ) - - if session.has_member("SessionId"): - return session.SessionId + try: + session = ntkrnlmp.object( + object_type="_MM_SESSION_SPACE", + offset=self.Session, + absolute=True, + ) + if session.has_member("SessionId"): + return session.SessionId + except exceptions.SymbolError: + # In Windows 11 24H2, the _MM_SESSION_SPACE type was + # replaced with _PSP_SESSION_SPACE, and the kernel PDB + # doesn't contain information about its members (otherwise, + # we would just fall back to the new type). However, it + # appears to be, for our purposes, functionally identical + # to the _MM_SESSION_SPACE. Because _MM_SESSION_SPACE + # stores its session ID at offset 8 as an unsigned long, we + # create an unsigned long at that offset and use that + # instead. + session_id = ntkrnlmp.object( + object_type="unsigned long", + offset=self.Session + 8, + absolute=True, + ) + return session_id except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.Session.SessionId at {self.vol.offset:#x}", ) - except exceptions.SymbolError: - vollog.log( - constants.LOGLEVEL_VVV, - "Could not lookup _MM_SESSION_SPACE in symbol table", - ) return renderers.UnreadableValue() @@ -1081,7 +1122,7 @@ class KTIMER(objects.StructType): return self.Header.Type in self.VALID_TYPES def get_due_time(self): - return "{0:#010x}:{1:#010x}".format(self.DueTime.HighPart, self.DueTime.LowPart) + return f"{self.DueTime.HighPart:#010x}:{self.DueTime.LowPart:#010x}" def get_dpc(self): """Return Dpc, and if Windows 7 or later, decode it""" @@ -1388,7 +1429,7 @@ class SHARED_CACHE_MAP(objects.StructType): ) # Iterate through the entries - for counter in range(0, self.VACB_ARRAY): + for counter in range(self.VACB_ARRAY): # Check if the VACB entry is in use if not vacb_array[counter]: continue @@ -1472,7 +1513,7 @@ class SHARED_CACHE_MAP(objects.StructType): if not section_size > self.VACB_SIZE_OF_FIRST_LEVEL: array_head = vacb_obj - for counter in range(0, full_blocks): + for counter in range(full_blocks): vacb_entry = self._context.object( symbol_table_name + constants.BANG + "pointer", layer_name=self.vol.layer_name, @@ -1531,7 +1572,7 @@ class SHARED_CACHE_MAP(objects.StructType): # Walk the array and if any entry points to the shared cache map object then we extract it. # Otherwise, if it is non-zero, then traverse to the next level. - for counter in range(0, self.VACB_ARRAY): + for counter in range(self.VACB_ARRAY): if not vacb_array[counter]: continue diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 2312149c7..9666fd79c 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -73,7 +73,7 @@ class ROW(objects.StructType): ) for i in range(0, len(char_row), 3) ) - except Exception as e: + except Exception: line = "" if truncate: @@ -107,11 +107,10 @@ class EXE_ALIAS_LIST(objects.StructType): def get_aliases(self) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Generator for the individual aliases for a particular executable.""" - for alias in self.AliasList.to_list( + yield from self.AliasList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_ALIAS", "ListEntry", - ): - yield alias + ) class SCREEN_INFORMATION(objects.StructType): @@ -245,11 +244,10 @@ class CONSOLE_INFORMATION(objects.StructType): def get_histories( self, ) -> Generator[interfaces.objects.ObjectInterface, None, None]: - for cmd_hist in self.HistoryList.to_list( + yield from self.HistoryList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_COMMAND_HISTORY", "ListEntry", - ): - yield cmd_hist + ) def get_exe_aliases( self, @@ -258,20 +256,18 @@ class CONSOLE_INFORMATION(objects.StructType): # Windows 10 22000 and Server 20348 made this a Pointer if isinstance(exe_alias_list, objects.Pointer): exe_alias_list = exe_alias_list.dereference() - for exe_alias_list_item in exe_alias_list.to_list( + yield from exe_alias_list.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_EXE_ALIAS_LIST", "ListEntry", - ): - yield exe_alias_list_item + ) def get_processes( self, ) -> Generator[interfaces.objects.ObjectInterface, None, None]: - for proc in self.ConsoleProcessList.to_list( + yield from self.ConsoleProcessList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_CONSOLE_PROCESS_LIST", "ListEntry", - ): - yield proc + ) def get_title(self) -> Union[str, None]: try: @@ -393,8 +389,7 @@ class COMMAND_HISTORY(objects.StructType): rest are coalesced. """ - for i, cmd in self.scan_command_bucket(self.CommandBucket.End): - yield i, cmd + yield from self.scan_command_bucket(self.CommandBucket.End) win10_x64_class_types = { diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index afdc73a17..078c4beb0 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -8,12 +8,7 @@ from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: """Get Disk Signature (GUID).""" - return "{0:02x}-{1:02x}-{2:02x}-{3:02x}".format( - self.DiskSignature[0], - self.DiskSignature[1], - self.DiskSignature[2], - self.DiskSignature[3], - ) + return f"{self.DiskSignature[0]:02x}-{self.DiskSignature[1]:02x}-{self.DiskSignature[2]:02x}-{self.DiskSignature[3]:02x}" class PARTITION_ENTRY(objects.StructType): diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 9b7573c2e..e41ac6a05 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -4,7 +4,7 @@ import logging import socket -from typing import Dict, Tuple, List, Union +from typing import Dict, Tuple, List, Union, Optional from volatility3.framework import exceptions from volatility3.framework import objects, interfaces @@ -22,7 +22,7 @@ def inet_ntop(address_family: int, packed_ip: Union[List[int], Array]) -> str: raise RuntimeError( "This version of python does not have socket.inet_ntop, please upgrade" ) - raise socket.error("[Errno 97] Address family not supported by protocol") + raise OSError("[Errno 97] Address family not supported by protocol") # Python's socket.AF_INET6 is 0x1e but Microsoft defines it @@ -86,19 +86,29 @@ class _TCP_LISTENER(objects.StructType): except exceptions.InvalidAddressException: return None - def get_owner_pid(self): - if self.get_owner().is_valid(): - if self.get_owner().has_valid_member("UniqueProcessId"): - return self.get_owner().UniqueProcessId + def get_owner_pid(self) -> Optional[int]: + owner = self.get_owner() + + if owner is None: + return None + + if owner.is_valid(): + if owner.has_valid_member("UniqueProcessId"): + return owner.UniqueProcessId return None - def get_owner_procname(self): - if self.get_owner().is_valid(): - if self.get_owner().has_valid_member("ImageFileName"): - return self.get_owner().ImageFileName.cast( + def get_owner_procname(self) -> Optional[str]: + owner = self.get_owner() + + if owner is None: + return None + + if owner.is_valid(): + if owner.has_valid_member("ImageFileName"): + return owner.ImageFileName.cast( "string", - max_length=self.get_owner().ImageFileName.vol.count, + max_length=owner.ImageFileName.vol.count, errors="replace", ) @@ -167,11 +177,9 @@ class _TCP_LISTENER(objects.StructType): def is_valid(self): try: - if not self.get_address_family() in (AF_INET, AF_INET6): + if self.get_address_family() not in (AF_INET, AF_INET6): vollog.debug( - "netw obj 0x{:x} invalid due to invalid address_family {}".format( - self.vol.offset, self.get_address_family() - ) + f"netw obj 0x{self.vol.offset:x} invalid due to invalid address_family {self.get_address_family()}" ) return False @@ -211,7 +219,13 @@ class _TCP_ENDPOINT(_TCP_LISTENER): return None def is_valid(self): - if self.State not in self.State.choices.values(): + # netstat calls this before validating the object itself + try: + state = self.State + except exceptions.InvalidAddressException: + return False + + if state not in state.choices.values(): vollog.debug( f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid tcp state {self.State}" ) diff --git a/volatility3/framework/symbols/windows/extensions/pe.py b/volatility3/framework/symbols/windows/extensions/pe.py index 3f34fc3dd..2c7400f25 100644 --- a/volatility3/framework/symbols/windows/extensions/pe.py +++ b/volatility3/framework/symbols/windows/extensions/pe.py @@ -101,9 +101,9 @@ class IMAGE_DOS_HEADER(objects.StructType): ) except OverflowError: vollog.warning( - "Volatility was unable to fix the image base for the PE file at base address {:#x}. " + f"Volatility was unable to fix the image base for the PE file at base address {self.vol.offset:#x}. " "This will cause issues with many static analysis tools if you do not inform the " - "tool of the in-memory load address.".format(self.vol.offset) + "tool of the in-memory load address." ) new_pe = raw_data diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index b761ddad8..ff65acdeb 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -217,7 +217,7 @@ class POOL_HEADER(objects.StructType): yield mem_object @classmethod - @functools.lru_cache() + @functools.lru_cache def _calculate_optional_header_lengths( cls, context: interfaces.context.ContextInterface, symbol_table_name: str ) -> Tuple[List[str], List[int]]: @@ -362,7 +362,7 @@ class OBJECT_HEADER(objects.StructType): return True def get_object_type( - self, type_map: Dict[int, str], cookie: int = None + self, type_map: Dict[int, str], cookie: Optional[int] = None ) -> Optional[str]: """Across all Windows versions, the _OBJECT_HEADER embeds details on the type of object (i.e. process, file) but the way its embedded @@ -376,7 +376,16 @@ class OBJECT_HEADER(objects.StructType): try: # vista and earlier have a Type member - self._vol["object_header_object_type"] = self.Type.Name.String + length = self.Type.member("Name").Length + if length == 0 or length > 128: + string = None + else: + string = self.Type.Name.String + if len(string) == 0 or len(string) > 128: + string = None + + self._vol["object_header_object_type"] = string + except AttributeError: # windows 7 and later have a TypeIndex, but windows 10 # further encodes the index value with nt1!ObHeaderCookie @@ -430,9 +439,7 @@ class OBJECT_HEADER(objects.StructType): if header_offset == 0: raise ValueError( - "Could not find _OBJECT_HEADER_NAME_INFO for object at {} of layer {}".format( - self.vol.offset, self.vol.layer_name - ) + f"Could not find _OBJECT_HEADER_NAME_INFO for object at {self.vol.offset} of layer {self.vol.layer_name}" ) header = ntkrnlmp.object( diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index bebfaea89..9e2f8df3b 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -196,9 +196,7 @@ class CM_KEY_NODE(objects.StructType): yield cast("CM_KEY_NODE", node) else: vollog.debug( - "Unexpected node type encountered when traversing subkeys: {}, signature: {}".format( - node.vol.type_name, signature - ) + f"Unexpected node type encountered when traversing subkeys: {node.vol.type_name}, signature: {signature}" ) if listjump: diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index 82ec31ccb..248ef7d0c 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -128,7 +128,10 @@ class PdbReader: self._layer_name, self._context = self.load_pdb_layer(context, location) self._dbiheader: Optional[interfaces.objects.ObjectInterface] = None if not progress_callback: - progress_callback = lambda x, y: None + + def progress_callback(x, y): + return None + self._progress_callback = progress_callback self.types: List[ Tuple[ @@ -263,9 +266,7 @@ class PdbReader: ) if header.index_max < header.index_min: raise ValueError( - "Maximum {} index is smaller than minimum TPI index, found: {} < {} ".format( - stream_name, header.index_max, header.index_min - ) + f"Maximum {stream_name} index is smaller than minimum TPI index, found: {header.index_max} < {header.index_min} " ) # Reset the state info_references: Dict[str, int] = {} @@ -976,14 +977,16 @@ class PdbRetreiver: if __name__ == "__main__": import argparse - class PrintedProgress(object): + class PrintedProgress: """A progress handler that prints the progress value and the description onto the command line.""" def __init__(self): self._max_message_len = 0 - def __call__(self, progress: Union[int, float], description: str = None): + def __call__( + self, progress: Union[int, float], description: Optional[str] = None + ): """A simple function for providing text-based feedback. .. warning:: Only for development use. diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 3816312cd..b5e8ca70a 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -36,7 +36,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): layer_name: str, offset: int, symbol_table_class: str = "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path: str = None, + config_path: Optional[str] = None, progress_callback: constants.ProgressCallback = None, ) -> Optional[str]: """Produces the name of a symbol table loaded from the offset for an MZ header @@ -94,7 +94,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): if not requirements.VersionRequirement.matches_required( (1, 0, 0), symbol_cache.SqliteCache.version ): - vollog.debug(f"Required version of SQLiteCache not found") + vollog.debug("Required version of SQLiteCache not found") return None identifiers_path = os.path.join( @@ -291,9 +291,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): break except PermissionError: vollog.warning( - "Cannot write necessary symbol file, please check permissions on {}".format( - potential_output_filename - ) + f"Cannot write necessary symbol file, please check permissions on {potential_output_filename}" ) continue finally: @@ -390,8 +388,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, ) -> str: """Creates symbol table for a module in the specified layer_name. @@ -420,8 +418,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, create_module: bool = False, ) -> Tuple[Optional[str], Optional[str]]: if module_offset is None: @@ -480,8 +478,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, ) -> str: """Creates a module in the specified layer_name based on a pdb name. diff --git a/volatility3/plugins/windows/registry/__init__.py b/volatility3/plugins/windows/registry/__init__.py index 8915cdfad..aeeaa87f2 100644 --- a/volatility3/plugins/windows/registry/__init__.py +++ b/volatility3/plugins/windows/registry/__init__.py @@ -15,5 +15,5 @@ import os import sys # This is necessary to ensure the core plugins are available, whilst still be overridable -parent_module, module_name = ".".join(__name__.split(".")[:-1]), __name__.split(".")[-1] +parent_module, module_name = __name__.rsplit(".", maxsplit=1) __path__ = [os.path.join(x, module_name) for x in sys.modules[parent_module].__path__] diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 5ef840f32..8587b3719 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -60,7 +60,7 @@ class Certificates(interfaces.plugins.PluginInterface): open_method: Type[interfaces.plugins.FileHandlerInterface], ) -> Optional[interfaces.plugins.FileHandlerInterface]: try: - dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) + dump_name = f"{hive_offset}-{reg_section}-{key_hash}.crt" file_handle = open_method(dump_name) file_handle.write(certificate_data) return file_handle diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 7f56b75f8..e7557dc0c 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -64,9 +64,7 @@ class Statistics(plugins.PluginInterface): other_invalid += 1 page_size = expected_page_size vollog.debug( - "A non-page lookup invalid address exception occurred at: {} in layer {}".format( - hex(excp.invalid_address), excp.layer_name - ) + f"A non-page lookup invalid address exception occurred at: {hex(excp.invalid_address)} in layer {excp.layer_name}" ) page_addr += page_size diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 3ca00e5dc..90cfaba48 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -20,7 +20,7 @@ def load_cached_validations() -> Set[str]: to revalidate them.""" validhashes: Set = set() if os.path.exists(cached_validation_filepath): - with open(cached_validation_filepath, "r") as f: + with open(cached_validation_filepath) as f: validhashes.update(json.load(f)) return validhashes @@ -46,7 +46,7 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: if not os.path.exists(schema_path): vollog.debug(f"Schema for format not found: {schema_path}") return False - with open(schema_path, "r") as s: + with open(schema_path) as s: schema = json.load(s) return valid(input, schema, use_cache) @@ -66,7 +66,7 @@ def create_json_hash( if not os.path.exists(schema_path): vollog.debug(f"Schema for format not found: {schema_path}") return None - with open(schema_path, "r") as s: + with open(schema_path) as s: schema = json.load(s) return hashlib.sha1( bytes(json.dumps((input, schema), sort_keys=True), "utf-8")