From 772ae98eb1966b4b0fa7451673352c6f8f97095c Mon Sep 17 00:00:00 2001 From: Malware Utkonos Date: Mon, 4 Jul 2022 13:27:17 -0400 Subject: [PATCH 01/32] Style changes including yapf according to .style.yapf in package root --- .../framework/plugins/windows/dumpfiles.py | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 58166ee7f..26b637fc3 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -5,6 +5,7 @@ import logging import ntpath from typing import List, Tuple, Type, Optional, Generator + from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -32,8 +33,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.IntRequirement(name = 'pid', description = "Process ID to include (all other processes are excluded)", optional = True), @@ -98,12 +100,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): :param open_method: class for constructing output files :param file_obj: the FILE_OBJECT """ - # Filtering by these types of devices prevents us from processing other types of devices that # use the "File" object type, such as \Device\Tcp and \Device\NamedPipe. if file_obj.DeviceObject.DeviceType not in [FILE_DEVICE_DISK, FILE_DEVICE_NETWORK_FILE_SYSTEM]: - vollog.log(constants.LOGLEVEL_VVV, - f"The file object at {file_obj.vol.offset:#x} is not a file on disk") + vollog.log(constants.LOGLEVEL_VVV, f"The file object at {file_obj.vol.offset:#x} is not a file on disk") return # Depending on the type of object (DataSection, ImageSection, SharedCacheMap) we may need to @@ -120,7 +120,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): # layer to read from, # file extension to apply, # ) - dump_parameters = [] + dump_parameters = list() # The DataSectionObject and ImageSectionObject caches are handled in basically the same way. # We carve these "pages" from the memory_layer. @@ -131,8 +131,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if control_area.is_valid(): dump_parameters.append((control_area, memory_layer, extension)) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") # The SharedCacheMap is handled differently than the caches above. # We carve these "pages" from the primary_layer. @@ -142,8 +141,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if shared_cache_map.is_valid(): dump_parameters.append((shared_cache_map, primary_layer, "vacb")) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] @@ -151,7 +149,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): memory_object.vol.offset, cache_name, ntpath.basename(obj_name), extension) - file_handle = DumpFiles.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) + file_handle = cls.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) file_output = "Error dumping file" if file_handle: @@ -185,8 +183,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): try: object_table = proc.ObjectTable except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") continue for entry in handles_plugin.handles(object_table): @@ -218,12 +215,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue - for result in self.process_file_object(self.context, kernel.layer_name, self.open, - file_obj): + for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot extract file from VAD at {vad.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"Cannot extract file from VAD at {vad.vol.offset:#x}") elif offsets: # Now process any offsets explicitly requested by the user. @@ -234,10 +229,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not is_virtual: layer_name = self.context.layers[layer_name].config["memory_layer"] - file_obj = self.context.object( - kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", + file_obj = self.context.object(kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", layer_name = layer_name, - native_layer_name = kernel.layer_name, + native_layer_name = kernel.layer_name, offset = offset) for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) @@ -246,9 +240,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): def run(self): # a list of tuples (, ) where is the address and is True for virtual. - offsets = [] + offsets = list() # a list of processes matching the pid filter. all files for these process(es) will be dumped. - procs = [] + procs = list() kernel = self.context.modules[self.config['kernel']] if self.config.get("virtaddr", None) is not None: From 84d26ba4bdf46b0280b343b3bd3c3b7ee54238c8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 5 Jul 2022 11:08:41 +0100 Subject: [PATCH 02/32] Core: Add in API_CHANGES updates --- API_CHANGES.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/API_CHANGES.md b/API_CHANGES.md index 274d1d8bb..4d8733286 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,14 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.3.0 +===== +Add in `child_template` to template class + +2.2.0 +===== +Changes to linux core calls + 2.1.0 ===== Add in the linux `task.get_threads` method to the API. From df277b9e802899368186aa04c4d106ba06de1e9b Mon Sep 17 00:00:00 2001 From: Frank Gomulka Date: Wed, 13 Jul 2022 13:49:30 -0500 Subject: [PATCH 03/32] Add testing framework --- .github/workflows/test.yaml | 54 +++++ test/README.md | 34 +++ test/conftest.py | 40 ++++ test/known_files.json | 19 ++ test/requirements-testing.txt | 8 + test/test_volatility.py | 381 ++++++++++++++++++++++++++++++++++ 6 files changed, 536 insertions(+) create mode 100644 .github/workflows/test.yaml create mode 100644 test/README.md create mode 100644 test/conftest.py create mode 100644 test/known_files.json create mode 100644 test/requirements-testing.txt create mode 100644 test/test_volatility.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..5a3f90565 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,54 @@ +name: Test Volatility3 +on: [push] +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install Cmake + pip install setuptools wheel + pip install -U pytest + pip install -r ./test/requirements-testing.txt + + - name: Build PyPi packages + run: | + python setup.py sdist --formats=gztar,zip + python setup.py bdist_wheel + + - name: Download images + run: | + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz" + gunzip linux-sample-1.bin.gz + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" + gunzip win-xp-laptop-2005-06-25.img.gz + + - name: Download and Extract symbols + run: | + cd ./volatility3/symbols + curl -sLO https://downloads.volatilityfoundation.org/volatility3/symbols/linux.zip + unzip linux.zip + cd - + + - name: Testing... + run: | + py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v + py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + + - name: Clean up post-test + run: | + rm -rf *.lime + rm -rf *.img + cd volatility3/symbols + rm -rf linux + rm -rf linux.zip + cd - diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..dcbe289b0 --- /dev/null +++ b/test/README.md @@ -0,0 +1,34 @@ +# Volatility 3 Testing Framework + +## Requirements + +The Volatility 3 Testing Framework requires the same version of Python as Volatility3 itself. To install the current set of dependencies that the framework requires, use a command like this: + +```shell +pip3 install -r requirements-testing.txt +``` + +NOTE: `requirements-testing.txt` can be found in this current `test/` directory. + +## Quick Start: Manual Testing + +1. To test Volatility 3 on an image, first download one with a command such as: + +```shell +curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" +gunzip win-xp-laptop-2005-06-25.img.gz +``` + +2. In many cases, more symbols are required to be downloaded to the `./volatility3/symbols` directory. + +3. To manually run the tests, run a command, such as: + +```shell +py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows +``` + +The above command runs all available tests for windows on the `win-xp-laptop-2005-06-25.img` image. To choose a more specific set of tests, change the phrase after `-k` in this command. + +## Github Actions + +This framework currently tests two images (one linux image and one windows image) after every push on any branch. For more information/context, find the actions setup in `./github/workflows/test.yaml` \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 000000000..9d3d27fc5 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,40 @@ +# This file is used to augment the test configuration + +import os +import pytest + +def pytest_addoption(parser): + parser.addoption("--volatility", action="store", default=None, + required=True, + help="path to the volatility script") + + parser.addoption("--python", action="store", default="python3", + help="The name of the interpreter to use when running the volatility script") + + parser.addoption("--image", action="append", default=[], + help="path to an image to test") + + parser.addoption("--image-dir", action="append", default=[], + help="path to a directory containing images to test") + +def pytest_generate_tests(metafunc): + """Parameterize tests based on image names""" + + images = metafunc.config.getoption('image') + for d in metafunc.config.getoption('image_dir'): + images = images + [os.path.join(d, x) for x in os.listdir(d)] + + # tests with "image" parameter are run against images + if 'image' in metafunc.fixturenames: + metafunc.parametrize("image", + images, + ids=[os.path.basename(image) for image in images]) + +# Fixtures +@pytest.fixture +def volatility(request): + return request.config.getoption("--volatility") + +@pytest.fixture +def python(request): + return request.config.getoption("--python") diff --git a/test/known_files.json b/test/known_files.json new file mode 100644 index 000000000..fbc40e48b --- /dev/null +++ b/test/known_files.json @@ -0,0 +1,19 @@ +{ + "windows_dumpfiles": { + "win-xp-laptop-2005-06-25.img": { + "0x82220e78": [ + "9bdd5532286f1660f3778e68bc36efe6", + "e3bc1e9e7370e3b5a661ebe591ecf4ec" + ], + "0x82350bf8": [ + "e5c5e8d97b6280745b41f6572c85d1f0", + "8589f1463422884dbf1411aaad278465" + ], + "0x81eaf418": [ + "f7a1ae2060a58f8470b97affdb46dccf", + "54fd611021fa784912530b8007545986" + ], + "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" + } + } + } \ No newline at end of file diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt new file mode 100644 index 000000000..d37dc93c3 --- /dev/null +++ b/test/requirements-testing.txt @@ -0,0 +1,8 @@ +# These packages are required for core functionality. +pefile>=2017.8.1 #foo + +# The following packages are optional. +# If certain packages are not necessary, place a comment (#) at the start of the line. + +# This is required for the yara plugins +yara-python>=3.8.0 diff --git a/test/test_volatility.py b/test/test_volatility.py new file mode 100644 index 000000000..527d86dd3 --- /dev/null +++ b/test/test_volatility.py @@ -0,0 +1,381 @@ +# volatility3 tests +# + +# +# IMPORTS +# + +import os +import subprocess +import sys +import shutil +import tempfile +import hashlib +import ntpath +import json + +import pytest + +# +# HELPER FUNCTIONS +# + +def runvol(args, volatility, python): + volpy = volatility + python_cmd = python + + cmd = [python_cmd, volpy] + args + print(" ".join(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + print("stdout:") + sys.stdout.write(str(stdout)) + print("") + print("stderr:") + sys.stdout.write(str(stderr)) + print("") + + return p.returncode, stdout, stderr + +def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): + args = globalargs + [ + "--single-location", + img, + "-q", + plugin, + ] + pluginargs + + return runvol(args, volatility, python) + +# +# TESTS +# + +# WINDOWS + +def test_windows_pslist(image, volatility, python): + rc, out, err = runvol_plugin("windows.pslist.PsList", image, volatility, python) + out = out.lower() + assert out.find(b"system") != -1 + assert out.find(b"csrss.exe") != -1 + assert out.find(b"svchost.exe") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + + rc, out, err = runvol_plugin( + "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"]) + out = out.lower() + assert out.find(b"system") != -1 + assert out.count(b"\n") < 10 + assert rc == 0 + assert rc == 0 + +def test_windows_psscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) + out = out.lower() + assert out.find(b"system") != -1 + assert out.find(b"csrss.exe") != -1 + assert out.find(b"svchost.exe") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_dlllist(image, volatility, python): + rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) + out = out.lower() + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_modules(image, volatility, python): + rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) + out = out.lower() + assert out.count(b"\n") > 10 + assert rc == 0 + assert rc == 0 + +def test_windows_hivelist(image, volatility, python): + rc, out, err = runvol_plugin("windows.registry.hivelist.HiveList", image, volatility, python) + out = out.lower() + + not_xp = out.find(b"\\systemroot\\system32\\config\\software") + if not_xp == -1: + assert out.find(b"\\device\\harddiskvolume1\\windows\\system32\\config\\software") != -1 + + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_windows_dumpfiles(image, volatility, python): + + json_file = open('./test/known_files.json') + + known_files = json.load(json_file) + + failed_chksms = 0 + + if sys.platform == 'win32': + file_name = ntpath.basename(image) + else: + file_name = os.path.basename(image) + + try: + for addr in known_files["windows_dumpfiles"][file_name]: + + path = tempfile.mkdtemp() + + rc, out, err = runvol_plugin("windows.dumpfiles.DumpFiles", image, volatility, python, globalargs=["-o", path], pluginargs=["--virtaddr", addr]) + + for file in os.listdir(path): + fp = open(os.path.join(path, file), "rb") + if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: + failed_chksms += 1 + fp.close() + + shutil.rmtree(path) + + json_file.close() + + assert failed_chksms == 0 + assert rc == 0 + except Exception as e: + json_file.close() + print("Key Error raised on " + str(e)) + assert False + +def test_windows_handles(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"System Pid 4") != -1 + assert out.find(b"MACHINE\\SYSTEM\\CONTROLSET001\\CONTROL\\SESSION MANAGER\\MEMORY MANAGEMENT\\PREFETCHPARAMETERS") != -1 + assert out.find(b"MACHINE\\SYSTEM\\SETUP") != -1 + assert out.count(b"\n") > 500 + assert rc == 0 + +def test_windows_svcscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) + + assert out.find(b"Microsoft ACPI Driver") != -1 + assert out.count(b"\n") > 250 + assert rc == 0 + +def test_windows_privileges(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"SeCreateTokenPrivilege") != -1 + assert out.find(b"SeCreateGlobalPrivilege") != -1 + assert out.find(b"SeAssignPrimaryTokenPrivilege") != -1 + assert out.count(b"\n") > 20 + assert rc == 0 + +def test_windows_getsids(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"Local System") != -1 + assert out.find(b"Administrators") != -1 + assert out.find(b"Everyone") != -1 + assert out.find(b"Authenticated Users") != -1 + assert rc == 0 + +def test_windows_envars(image, volatility, python): + rc, out, err = runvol_plugin("windows.envars.Envars", image, volatility, python) + + assert out.find(b"PATH") != -1 + assert out.find(b"PROCESSOR_ARCHITECTURE") != -1 + assert out.find(b"USERNAME") != -1 + assert out.find(b"SystemRoot") != -1 + assert out.find(b"CommonProgramFiles") != -1 + assert out.count(b"\n") > 500 + assert rc == 0 + +def test_windows_callbacks(image, volatility, python): + rc, out, err = runvol_plugin("windows.callbacks.Callbacks", image, volatility, python) + + assert out.find(b"PspCreateProcessNotifyRoutine") != -1 + assert out.find(b"KeBugCheckCallbackListHead") != -1 + assert out.find(b"KeBugCheckReasonCallbackListHead") != -1 + assert out.count(b"KeBugCheckReasonCallbackListHead ") > 5 + assert rc == 0 + +# LINUX + +def test_linux_pslist(image, volatility, python): + rc, out, err = runvol_plugin("linux.pslist.PsList", image, volatility, python) + out = out.lower() + + assert ((out.find(b"init") != -1) or (out.find(b"systemd") != -1)) + assert out.find(b"watchdog") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_check_idt(image, volatility, python): + rc, out, err = runvol_plugin("linux.check_idt.Check_idt", image, volatility, python) + out = out.lower() + + assert out.count(b"__kernel__") >= 10 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_check_syscall(image, volatility, python): + rc, out, err = runvol_plugin("linux.check_syscall.Check_syscall", image, volatility, python) + out = out.lower() + + assert out.find(b"sys_close") != -1 + assert out.find(b"sys_open") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_linux_lsmod(image, volatility, python): + rc, out, err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_lsof(image, volatility, python): + rc, out, err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) + out = out.lower() + + assert out.count(b"socket:") >= 10 + assert out.count(b"\n") > 35 + assert rc == 0 + +def test_linux_proc_maps(image, volatility, python): + rc, out, err = runvol_plugin("linux.proc.Maps", image, volatility, python) + out = out.lower() + + assert out.count(b"anonymous mapping") >= 10 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_linux_tty_check(image, volatility, python): + rc, out, err = runvol_plugin("linux.tty_check.tty_check", image, volatility, python) + out = out.lower() + + assert out.find(b"__kernel__") != -1 + assert out.count(b"\n") >= 5 + assert rc == 0 + +# MAC + +def test_mac_pslist(image, volatility, python): + rc, out, err = runvol_plugin("mac.pslist.PsList", image, volatility, python) + out = out.lower() + + assert ((out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1)) + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_check_syscall(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_syscall.Check_syscall", image, volatility, python) + out = out.lower() + + assert out.find(b"chmod") != -1 + assert out.find(b"chown") != -1 + assert out.find(b"nosys") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_mac_check_sysctl(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_sysctl.Check_sysctl", image, volatility, python) + out = out.lower() + + assert out.find(b"__kernel__") != -1 + assert out.count(b"\n") > 250 + assert rc == 0 + +def test_mac_check_trap_table(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_trap_table.Check_trap_table", image, volatility, python) + out = out.lower() + + assert out.count(b"kern_invalid") >= 10 + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_ifconfig(image, volatility, python): + rc, out, err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) + out = out.lower() + + assert out.find(b"127.0.0.1") != -1 + assert out.find(b"false") != -1 + assert out.count(b"\n") > 9 + assert rc == 0 + +def test_mac_lsmod(image, volatility, python): + rc, out, err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) + out = out.lower() + + assert out.find(b"com.apple") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_lsof(image, volatility, python): + rc, out, err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_malfind(image, volatility, python): + rc, out, err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 20 + assert rc == 0 + +def test_mac_mount(image, volatility, python): + rc, out, err = runvol_plugin("mac.mount.Mount", image, volatility, python) + out = out.lower() + + assert out.find(b"/dev") != -1 + assert out.count(b"\n") > 7 + assert rc == 0 + +def test_mac_netstat(image, volatility, python): + rc, out, err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) + + assert out.find(b"TCP") != -1 + assert out.find(b"UDP") != -1 + assert out.find(b"UNIX") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_proc_maps(image, volatility, python): + rc, out, err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) + out = out.lower() + + assert out.find(b"[heap]") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_mac_psaux(image, volatility, python): + rc, out, err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) + out = out.lower() + + assert out.find(b"executable_path") != -1 + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_socket_filters(image, volatility, python): + rc, out, err = runvol_plugin("mac.socket_filters.Socket_filters", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 9 + assert rc == 0 + +def test_mac_timers(image, volatility, python): + rc, out, err = runvol_plugin("mac.timers.Timers", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 6 + assert rc == 0 + +def test_mac_trustedbsd(image, volatility, python): + rc, out, err = runvol_plugin("mac.trustedbsd.Trustedbsd", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 From f96f004b13c15b3c0334f7ca96b2c0459a8f9cf1 Mon Sep 17 00:00:00 2001 From: Frank Gomulka Date: Fri, 15 Jul 2022 18:01:48 -0500 Subject: [PATCH 04/32] @digitalisx suggested changes --- .github/workflows/test.yaml | 6 +++--- test/test_volatility.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5a3f90565..a3ecd7c7e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,5 +1,5 @@ name: Test Volatility3 -on: [push] +on: [push, pull_request] jobs: build: @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.x + - name: Set up Python 3.6 uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.6' - name: Install dependencies run: | diff --git a/test/test_volatility.py b/test/test_volatility.py index 527d86dd3..a55dffb27 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -127,10 +127,9 @@ def test_windows_dumpfiles(image, volatility, python): rc, out, err = runvol_plugin("windows.dumpfiles.DumpFiles", image, volatility, python, globalargs=["-o", path], pluginargs=["--virtaddr", addr]) for file in os.listdir(path): - fp = open(os.path.join(path, file), "rb") - if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: - failed_chksms += 1 - fp.close() + with open(os.path.join(path, file), "rb") as fp: + if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: + failed_chksms += 1 shutil.rmtree(path) From f295e5d91a6b7ecc7a1f03099d2984a98cf43eda Mon Sep 17 00:00:00 2001 From: fgomulka <60993471+fgomulka@users.noreply.github.com> Date: Sat, 16 Jul 2022 16:34:47 -0500 Subject: [PATCH 05/32] Add newline Co-authored-by: Donghyun Kim --- test/known_files.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/known_files.json b/test/known_files.json index fbc40e48b..089896714 100644 --- a/test/known_files.json +++ b/test/known_files.json @@ -16,4 +16,5 @@ "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" } } - } \ No newline at end of file + } + \ No newline at end of file From 3e748e7d488eeb96904d94039ca02d57c6368fff Mon Sep 17 00:00:00 2001 From: fgomulka <60993471+fgomulka@users.noreply.github.com> Date: Sat, 16 Jul 2022 16:35:26 -0500 Subject: [PATCH 06/32] Use more descriptive variable names Co-authored-by: Donghyun Kim --- test/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 9d3d27fc5..9057e1676 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -21,8 +21,8 @@ def pytest_generate_tests(metafunc): """Parameterize tests based on image names""" images = metafunc.config.getoption('image') - for d in metafunc.config.getoption('image_dir'): - images = images + [os.path.join(d, x) for x in os.listdir(d)] + for image_dir in metafunc.config.getoption('image_dir'): + images = images + [os.path.join(image_dir, dir) for dir in os.listdir(image_dir)] # tests with "image" parameter are run against images if 'image' in metafunc.fixturenames: From 5bc517aa42f09bb467136866d92811760a92169b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 00:15:57 +0000 Subject: [PATCH 07/32] Automagic: Use sqlite to cache identifiers --- volatility3/framework/automagic/linux.py | 35 +- volatility3/framework/automagic/mac.py | 28 +- .../framework/automagic/symbol_cache.py | 480 ++++++++++++------ .../framework/automagic/symbol_finder.py | 25 +- .../framework/configuration/requirements.py | 12 +- volatility3/framework/constants/__init__.py | 7 +- volatility3/framework/interfaces/automagic.py | 9 +- volatility3/framework/plugins/isfinfo.py | 39 +- volatility3/framework/symbols/intermed.py | 3 +- .../framework/symbols/windows/pdbutil.py | 58 +-- 10 files changed, 417 insertions(+), 279 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f1d6c91e4..2c152996d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -5,8 +5,9 @@ import logging from typing import Optional, Tuple, Type -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder +from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import linux @@ -23,6 +24,13 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify linux within this layer.""" + # Version check the SQlite cache + required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + vollog.info( + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + return None + # Bail out by default unless we can stack properly layer = context.layers[layer_name] join = interfaces.configuration.path_join @@ -32,7 +40,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - linux_banners = LinuxBannerCache.load_banners() + linux_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + operating_system = 'linux') # If we have no banners, don't bother scanning if not linux_banners: vollog.info("No Linux banners found - if this is a linux plugin, please check your symbol files location") @@ -43,15 +52,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = linux_banners.get(banner, None) - if symbol_files: - if len(symbol_files) > 1: - using = "*" - vollog.warning(f"Multiple symbol files identified (using {using}):") - for symbol_file in symbol_files: - vollog.warning(f" {using} {symbol_file}") - using = " " - isf_path = symbol_files[0] + isf_path = linux_banners.get(banner, None) + if isf_path: table_name = context.symbol_space.free_table_name('LintelStacker') table = linux.LinuxKernelIntermedSymbols(context, 'temporary.' + table_name, @@ -147,20 +149,11 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): return addr - 0xc0000000 -class LinuxBannerCache(symbol_cache.SymbolBannerCache): - """Caches the banners found in the Linux symbol files.""" - - os = "linux" - symbol_name = "linux_banner" - banner_path = constants.LINUX_BANNERS_PATH - exclusion_list = ['mac', 'windows'] - - class LinuxSymbolFinder(symbol_finder.SymbolFinder): """Linux symbol loader based on uname signature strings.""" banner_config_key = "kernel_banner" - banner_cache = LinuxBannerCache + operating_system = 'linux' symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] exclusion_list = ['mac', 'windows'] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index c37aef463..246462878 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -6,8 +6,9 @@ import logging import struct from typing import Optional -from volatility3.framework import interfaces, constants, layers, exceptions +from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.automagic import symbol_cache, symbol_finder +from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import mac @@ -24,6 +25,13 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify mac within this layer.""" + # Version check the SQlite cache + required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + vollog.info( + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + return None + # Bail out by default unless we can stack properly layer = context.layers[layer_name] new_layer = None @@ -34,7 +42,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - mac_banners = MacBannerCache.load_banners() + mac_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + operating_system = 'mac') # If we have no banners, don't bother scanning if not mac_banners: vollog.info("No Mac banners found - if this is a mac plugin, please check your symbol files location") @@ -46,9 +55,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = mac_banners.get(banner, None) - if symbol_files: - isf_path = symbol_files[0] + isf_path = mac_banners.get(banner, None) + if isf_path: table_name = context.symbol_space.free_table_name('MacintelStacker') table = mac.MacKernelIntermedSymbols(context = context, config_path = join('temporary', table_name), @@ -197,19 +205,11 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): yield offset, banner -class MacBannerCache(symbol_cache.SymbolBannerCache): - """Caches the banners found in the Mac symbol files.""" - os = "mac" - symbol_name = "version" - banner_path = constants.MAC_BANNERS_PATH - exclusion_list = ['windows', 'linux'] - - class MacSymbolFinder(symbol_finder.SymbolFinder): """Mac symbol loader based on uname signature strings.""" banner_config_key = 'kernel_banner' - banner_cache = MacBannerCache + operating_system = 'mac' find_aslr = MacIntelStacker.find_aslr symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols" exclusion_list = ['windows', 'linux'] diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 7b6adf9b4..fe717b8be 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -2,18 +2,20 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import base64 -import gc import json import logging import os -import pickle +import sqlite3 import urllib import urllib.parse import urllib.request -import zipfile -from typing import Dict, List, Optional +from abc import abstractmethod +from typing import Dict, Generator, List, Optional -from volatility3.framework import constants, exceptions, interfaces +import volatility3.framework +import volatility3.schemas +from volatility3.framework import constants, interfaces +from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources from volatility3.framework.symbols import intermed @@ -22,164 +24,324 @@ vollog = logging.getLogger(__name__) BannersType = Dict[bytes, List[str]] -class SymbolBannerCache(interfaces.automagic.AutomagicInterface): - """Runs through all symbols tables and caches their banners.""" +### Identifiers - # Since this is necessary for ConstructionMagic, we set a lower priority - # The user would run it eventually either way, but running it first means it can be used that run +class IdentifierProcessor: + operating_system = None + + def __init__(self): + pass + + @classmethod + @abstractmethod + def get_identifier(cls, json) -> Optional[bytes]: + """Method to extract the identifier from a particular operating system's JSON + + Returns: + identifier is valid or None if not found + """ + raise NotImplemented("This base class has no get_identifier method defined") + + +class WindowsIdentifier(IdentifierProcessor): + operating_system = 'windows' + separator = '|' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + """Returns the identifier for the file if one can be found""" + windows_metadata = json.get('metadata', {}).get('windows', {}).get('pdb', {}) + if windows_metadata: + guid = windows_metadata.get('GUID', None) + age = windows_metadata.get('age', None) + database = windows_metadata.get('database', None) + if guid and age and database: + return cls.generate(database, guid, age) + return None + + @classmethod + def generate(cls, pdb_name: str, guid: str, age: int) -> bytes: + return bytes(cls.separator.join([pdb_name, guid.upper(), str(age)]), 'latin-1') + + +class MacIdentifier(IdentifierProcessor): + operating_system = 'mac' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + mac_banner = json.get('symbols', {}).get('version', {}).get('constant_data', None) + if mac_banner: + return base64.b64decode(mac_banner) + return None + + +class LinuxIdentifier(IdentifierProcessor): + operating_system = 'linux' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + linux_banner = json.get('symbols', {}).get('linux_banner', {}).get('constant_data', None) + if linux_banner: + return base64.b64decode(linux_banner) + return None + + +### CacheManagers + +class CacheManagerInterface(interfaces.configuration.VersionableInterface): + def __init__(self, filename: str): + super().__init__() + self._filename = filename + self._classifiers = {} + for subclazz in volatility3.framework.class_subclasses(IdentifierProcessor): + self._classifiers[subclazz.operating_system] = subclazz + + def add_identifier(self, location: str, operating_system: str, identifier: str): + """Adds an identifier to the store""" + pass + + def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + """Returns the location of the symbol file given the identifier + + Args: + identifier: string that uniquely identifies a particular symbolt table + operating_system: optional string to restrict identifiers to just those for a particular operating system + + Returns: + The location of the symbols file that matches the identifier + """ + pass + + def get_local_locations(self) -> List[str]: + """Returns a list of all the local locations""" + pass + + def update(self): + """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. + This also updates remote locations based on a cache timeout. + + """ + pass + + def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ + Dict[bytes, str]: + """Returns a dictionary of identifiers and locations + + Args: + operating_system: If set, limits responses to a specific operating system + local_only: Returns only local locations + + Returns: + A dictionary of identifiers mapped to a location + """ + pass + + def get_identifier(self, location: str) -> Optional[bytes]: + """Returns an identifier based on a specific location or None""" + pass + + def get_identifiers(self, operating_system: Optional[str]): + """Returns all identifiers for a particular operating system""" + pass + + +class SqliteCache(CacheManagerInterface): + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + def __init__(self, filename: str): + super().__init__(filename) + try: + self._database = self._connect_storage(filename) + except sqlite3.DatabaseError: + os.unlink(filename) + self._database = self._connect_storage(filename) + + def _connect_storage(self, path: str): + database = sqlite3.connect(path, isolation_level = None) + database.row_factory = sqlite3.Row + database.cursor().execute( + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + return database + + def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + """Returns the location of the symbol file given the identifier. + If multiple locations exist for an identifier, the last found is returned + + Args: + identifier: string that uniquely identifies a particular symbolt table + operating_system: optional string to restrict identifiers to just those for a particular operating system + + Returns: + The location of the symbols file that matches the identifier or None + """ + statement = 'SELECT location FROM cache WHERE identifier = ?' + parameters = (identifier,) + if operating_system is not None: + statement = 'SELECT location FROM cache WHERE identifier = ? AND operating_system = ?' + parameters = (identifier, operating_system) + results = self._database.cursor().execute(statement, parameters).fetchall() + result = None + for row in results: + result = row['location'] + return result + + def get_local_locations(self) -> Generator[str, None, None]: + result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = True').fetchall() + for row in result: + yield row['location'] + + def is_url_local(self, url: str) -> bool: + """Determines whether an url is local or not""" + parsed = urllib.parse.urlparse(url) + if parsed.scheme in ['file', 'jar']: + return True + + def get_identifier(self, location: str) -> Optional[bytes]: + results = self._database.cursor().execute('SELECT identifier FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['identifier'] + return None + + def update(self, progress_callback = None): + """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. + This also updates remote locations based on a cache timeout. + + """ + on_disk_locations = set([filename for filename in intermed.IntermediateSymbolTable.file_symbol_url('')]) + cached_locations = set(self.get_local_locations()) + + new_locations = on_disk_locations.difference(cached_locations) + missing_locations = cached_locations.difference(on_disk_locations) + + cache_update = set() + files_to_timestamp = on_disk_locations.intersection(cached_locations) + if files_to_timestamp: + result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " + "AND cached < date('now', '-3 days');") + for row in result: + if row['location'] in files_to_timestamp: + cache_update.add(row['location']) + + idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + + counter = 0 + files_to_process = new_locations.union(cache_update) + number_files_to_process = len(files_to_process) + for location in files_to_process: + # Open location + counter += 1 + progress_callback(counter * 100 / number_files_to_process, + "Updating caches for {number_files_to_process} files...") + try: + with resources.ResourceAccessor().open(location) as fp: + json_obj = json.load(fp) + identifier = None + for idextractor in idextractors: + identifier = idextractor.get_identifier(json_obj) + operating_system = idextractor.operating_system + if identifier is not None: + break + if identifier is not None: + # We don't try to validate schemas here, we do that on first use + # Store in database + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") + else: + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + None, + None, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") + except Exception as excp: + vollog.log(constants.LOGLEVEL_VVVV, excp) + + if not constants.OFFLINE and constants.REMOTE_ISF_URL: + remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) + for operating_system in ['mac', 'linux', 'windows']: + identifiers = remote_identifiers.process({}, operating_system = operating_system) + for identifier in identifiers: + for location in identifiers[identifier]: + self._database.cursor().execute( + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now')", + (location, identifier, operating_system, False) + ) + + if missing_locations: + self._database.cursor().execute( + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + + def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ + Dict[bytes, str]: + output = {} + additions = [] + statement = 'SELECT location, identifier FROM cache' + if local_only: + additions.append('local = True') + if operating_system: + additions.append(f"operating_system = '{operating_system}'") + if additions: + statement += f" WHERE {' AND '.join(additions)}" + results = self._database.cursor().execute(statement) + for row in results: + if row['identifier'] in output and row['identifier'] and row['location']: + vollog.debug( + f"Duplicate entry for identifier {row['identifier']}: {row['location']} and {output[row['identifier']]}") + output[row['identifier']] = row['location'] + return output + + def get_identifiers(self, operating_system: Optional[str]): + if operating_system: + results = self._database.cursor().execute('SELECT identifier FROM cache WHERE operating_system = ?', + (operating_system,)).fetchall() + else: + results = self._database.cursor().execute('SELECT identifier FROM cache').fetchall() + output = [] + for row in results: + output.append(row['identifier']) + return output + + +### Automagic + +class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): + """Runs through all symbol tables and caches their identifiers""" priority = 0 - os: Optional[str] = None - symbol_name: str = "banner_name" - banner_path: Optional[str] = None - - @classmethod - def load_banners(cls) -> BannersType: - if not cls.banner_path: - raise ValueError("Banner_path not appropriately set") - banners: BannersType = {} - if os.path.exists(cls.banner_path): - with open(cls.banner_path, "rb") as f: - # We use pickle over JSON because we're dealing with bytes objects - banners.update(pickle.load(f)) - - # Remove possibilities that can't exist locally. - remove_banners = [] - for banner in banners: - for path in banners[banner]: - url = urllib.parse.urlparse(path) - if url.scheme == 'file' and not os.path.exists(urllib.request.url2pathname(url.path)): - vollog.log( - constants.LOGLEVEL_VV, "Removing cached path {} for banner {}: file does not exist".format( - path, str(banner or b'', 'latin-1'))) - banners[banner].remove(path) - # This is probably excessive, but it's here if we need it - if url.scheme == 'jar': - zip_file, zip_path = url.path.split("!") - zip_file = urllib.parse.urlparse(zip_file).path - if ((not os.path.exists(zip_file)) or (zip_path not in zipfile.ZipFile(zip_file).namelist())): - vollog.log(constants.LOGLEVEL_VV, - "Removing cached path {} for banner {}: file does not exist".format(path, banner)) - banners[banner].remove(path) - - if not banners[banner]: - remove_banners.append(banner) - for remove_banner in remove_banners: - del banners[remove_banner] - return banners - - @classmethod - def save_banners(cls, banners): - - with open(cls.banner_path, "wb") as f: - pickle.dump(banners, f) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cache = SqliteCache(constants.IDENTIFIERS_PATH) def __call__(self, context, config_path, configurable, progress_callback = None): """Runs the automagic over the configurable.""" - - # Bomb out if we're just the generic interface - if self.os is None: - return - - # We only need to be called once, so no recursion necessary - banners = self.load_banners() - - cacheables = self.find_new_banner_files(banners, self.os) - - new_banners = self.read_new_banners(context, config_path, cacheables, self.symbol_name, self.os, - progress_callback) - - # Add in any new banners to the existing list - for new_banner in new_banners: - banner_list = banners.get(new_banner, []) - banners[new_banner] = list(set(banner_list + new_banners[new_banner])) - - # Do remote banners *after* the JSON loading, so that it doesn't pull down all the remote JSON - self.remote_banners(banners, self.os) - - # Rewrite the cached banners each run, since writing is faster than the banner_cache validation portion - self.save_banners(banners) - - if progress_callback is not None: - progress_callback(100, f"Built {self.os} caches") + self._cache.update(progress_callback) @classmethod - def read_new_banners(cls, context: interfaces.context.ContextInterface, config_path: str, new_urls: List[str], - symbol_name: str, operating_system: str = None, - progress_callback = None) -> Optional[Dict[bytes, List[str]]]: - """Reads the any new banners for the OS in question""" - if operating_system is None: - return None - - banners = {} - - total = len(new_urls) - if total > 0: - vollog.info(f"Building {operating_system} caches...") - for current in range(total): - if progress_callback is not None: - progress_callback(current * 100 / total, f"Building {operating_system} caches") - isf_url = new_urls[current] - - isf = None - try: - # Loading the symbol table will be very slow until it's been validated - isf = intermed.IntermediateSymbolTable(context, config_path, "temp", isf_url, validate = False) - - # We should store the banner against the filename - # We don't bother with the hash (it'll likely take too long to validate) - # but we should check at least that the banner matches on load. - banner = isf.get_symbol(symbol_name).constant_data - vollog.log(constants.LOGLEVEL_VV, f"Caching banner {banner} for file {isf_url}") - - bannerlist = banners.get(banner, []) - bannerlist.append(isf_url) - banners[banner] = bannerlist - except exceptions.SymbolError: - pass - except json.JSONDecodeError: - vollog.log(constants.LOGLEVEL_VV, f"Caching file {isf_url} failed due to JSON error") - finally: - # Get rid of the loaded file, in case it sits in memory - if isf: - del isf - gc.collect() - return banners - - @classmethod - def find_new_banner_files(cls, banners: Dict[bytes, List[str]], operating_system: str) -> List[str]: - """Gathers all files and remove existing banners""" - cacheables = list(intermed.IntermediateSymbolTable.file_symbol_url(operating_system)) - for banner in banners: - for json_file in banners[banner]: - if json_file in cacheables: - cacheables.remove(json_file) - return cacheables - - @classmethod - def remote_banners(cls, banners: Dict[bytes, List[str]], operating_system = None, banner_location = None): - """Adds remote URLs to the banner list""" - if operating_system is None: - return None - - if banner_location is None: - banner_location = constants.REMOTE_ISF_URL - - if not constants.OFFLINE and banner_location is not None: - try: - rbf = RemoteBannerFormat(banner_location) - rbf.process(banners, operating_system) - except urllib.error.URLError: - vollog.debug(f"Unable to download remote banner list from {banner_location}") + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + """Returns a list of RequirementInterface objects required by this + object.""" + return [requirements.VersionRequirement(name = 'SQLiteCache', component = SqliteCache, version = (1, 0, 0))] -class RemoteBannerFormat: +class RemoteIdentifierFormat: def __init__(self, location: str): self._location = location with resources.ResourceAccessor().open(url = location) as fp: self._data = json.load(fp) if not self._verify(): - raise ValueError("Unsupported version for remote banner list format") + raise ValueError("Unsupported version for remote identifier list format") def _verify(self) -> bool: version = self._data.get('version', 0) @@ -188,23 +350,23 @@ class RemoteBannerFormat: return True return False - def process(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): - raise ValueError("Banner List version not verified") + def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]): + raise ValueError("Identifier List version not verified") - def process_v1(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): + def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]): if operating_system in self._data: - for banner in self._data[operating_system]: - binary_banner = base64.b64decode(banner) - file_list = banners.get(binary_banner, []) - for value in self._data[operating_system][banner]: + for identifier in self._data[operating_system]: + binary_identifier = base64.b64decode(identifier) + file_list = identifiers.get(binary_identifier, []) + for value in self._data[operating_system][identifier]: if value not in file_list: file_list = file_list + [value] - banners[binary_banner] = file_list + identifiers[binary_identifier] = file_list if 'additional' in self._data: for location in self._data['additional']: try: - subrbf = RemoteBannerFormat(location) - subrbf.process(banners, operating_system) + subrbf = RemoteIdentifierFormat(location) + subrbf.process(identifiers, operating_system) except IOError: vollog.debug(f"Remote file not found: {location}") - return banners + return identifiers diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 143abd02e..610ed0e18 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -3,9 +3,9 @@ # import logging -from typing import Any, Iterable, List, Tuple, Type, Optional, Callable +from typing import Any, Callable, Iterable, List, Optional, Tuple -from volatility3.framework import interfaces, constants, layers +from volatility3.framework import constants, interfaces, layers from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners @@ -18,7 +18,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): priority = 40 banner_config_key: str = "banner" - banner_cache: Optional[Type[symbol_cache.SymbolBannerCache]] = None + operating_system: Optional[str] = None symbol_class: Optional[str] = None find_aslr: Optional[Callable] = None @@ -27,14 +27,21 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): self._requirements: List[Tuple[str, interfaces.configuration.RequirementInterface]] = [] self._banners: symbol_cache.BannersType = {} + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.VersionRequirement(name = 'SQLiteCache', + component = symbol_cache.SqliteCache, + version = (1, 0, 0)) + ] + @property def banners(self) -> symbol_cache.BannersType: """Creates a cached copy of the results, but only it's been requested.""" if not self._banners: - if not self.banner_cache: - raise RuntimeError(f"Cache has not been properly defined for {self.__class__.__name__}") - self._banners = self.banner_cache.load_banners() + cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system) return self._banners def __call__(self, @@ -103,8 +110,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): vollog.debug(f"Identified banner: {repr(banner)}") symbol_files = self.banners.get(banner, None) if symbol_files: - isf_path = symbol_files[0] - vollog.debug(f"Using symbol library: {symbol_files[0]}") + isf_path = symbol_files + vollog.debug(f"Using symbol library: {symbol_files}") clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join @@ -117,7 +124,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): break else: if symbol_files: - vollog.debug(f"Symbol library path not found: {symbol_files[0]}") + vollog.debug(f"Symbol library path not found: {symbol_files}") # print("Kernel", banner, hex(banner_offset)) else: vollog.debug("No existing banners found") diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 4edc6d17c..b31c4767f 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -408,13 +408,19 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type config_path = interfaces.configuration.path_join(config_path, self.name) - if len(self._version) > 0 and self._component.version[0] != self._version[0]: - return {config_path: self} - if len(self._version) > 1 and self._component.version[1] < self._version[1]: + if not self.matches_required(self._version, self._component.version): return {config_path: self} context.config[interfaces.configuration.path_join(config_path, self.name)] = True return {} + @classmethod + def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]): + if len(required) > 0 and version[0] != required[0]: + return False + if len(required) > 1 and version[1] < required[1]: + return False + return True + class PluginRequirement(VersionRequirement): diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index f08819f29..322e574e1 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -68,10 +68,13 @@ if sys.platform == 'win32': os.makedirs(CACHE_PATH, exist_ok = True) LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") -""""Default location to record information about available linux banners""" +"""Default location to record information about available linux banners""" MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") -""""Default location to record information about available mac banners""" +"""Default location to record information about available mac banners""" + +IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") +"""Default location to record information about available identifiers""" BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index c96c9bdbe..713f91da0 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -9,9 +9,9 @@ that a user has not filled. """ import logging from abc import ABCMeta -from typing import Any, List, Optional, Tuple, Union, Type +from typing import Any, List, Optional, Tuple, Type, Union -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -47,9 +47,10 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla super().__init__(context, config_path) for requirement in self.get_requirements(): if not isinstance(requirement, (interfaces.configuration.SimpleTypeRequirement, - requirements.ChoiceRequirement, requirements.ListRequirement)): + requirements.ChoiceRequirement, requirements.ListRequirement, + requirements.VersionRequirement)): raise TypeError( - "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement or ListRequirement") + "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement, ListRequirement or VersionRequirement") def __call__(self, context: interfaces.context.ContextInterface, diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 575f25426..b2960733d 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -1,17 +1,16 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import base64 import json import logging import os import pathlib import zipfile -from typing import List, Type, Any, Generator +from typing import Generator, List from volatility3 import schemas, symbols -from volatility3.framework import interfaces, renderers, constants -from volatility3.framework.automagic import mac, linux, symbol_cache +from volatility3.framework import constants, interfaces, renderers +from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import resources @@ -23,7 +22,7 @@ class IsfInfo(plugins.PluginInterface): """Determines information about the currently available ISF files, or a specific one""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -39,7 +38,10 @@ class IsfInfo(plugins.PluginInterface): requirements.BooleanRequirement(name = 'validate', description = 'Validate against schema if possible', default = False, - optional = True) + optional = True), + requirements.VersionRequirement(name = 'SQLiteCache', + component = symbol_cache.SqliteCache, + version = (1, 0, 0)) ] @classmethod @@ -62,14 +64,6 @@ class IsfInfo(plugins.PluginInterface): if filename.endswith(extension): yield pathlib.Path(base_name).as_uri() - def _get_banner(self, clazz: Type[symbol_cache.SymbolBannerCache], data: Any) -> str: - """Gets a banner from an ISF file""" - banner_symbol = data.get('symbols', {}).get(clazz.symbol_name, {}).get('constant_data', - renderers.NotAvailableValue()) - if not isinstance(banner_symbol, interfaces.renderers.BaseAbsentValue): - banner_symbol = str(base64.b64decode(banner_symbol), encoding = 'latin-1') - return banner_symbol - def _generator(self): if self.config.get('isf', None) is not None: file_list = [self.config['isf']] @@ -101,7 +95,6 @@ class IsfInfo(plugins.PluginInterface): # Process the filtered list for entry in filtered_list: num_types = num_enums = num_bases = num_symbols = 0 - windows_info = linux_banner = mac_banner = renderers.NotAvailableValue() valid = "Unknown" with resources.ResourceAccessor().open(url = entry) as fp: try: @@ -111,20 +104,20 @@ class IsfInfo(plugins.PluginInterface): num_enums = len(data.get('enums', [])) num_bases = len(data.get('base_types', [])) - linux_banner = self._get_banner(linux.LinuxBannerCache, data) - mac_banner = self._get_banner(mac.MacBannerCache, data) - if not linux_banner and not mac_banner: - windows_info = os.path.splitext(os.path.basename(entry))[0] + identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifier = identifier_cache.get_identifier(location = entry) + if identifier: + identifier = identifier.decode('utf-8', errors = 'replace') + else: + identifier = renderers.NotAvailableValue() valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {entry}") - yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, windows_info, linux_banner, - mac_banner)) + yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) # Try to open the file, load it as JSON, read the data from it def run(self): return renderers.TreeGrid([("URI", str), ("Valid", str), ("Number of base_types", int), ("Number of types", int), ("Number of symbols", int), - ("Number of enums", int), ("Windows info", str), ("Linux banner", str), - ("Mac banner", str)], self._generator()) + ("Number of enums", int), ("Identifying infomration", str)], self._generator()) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index a6a7a0fae..1fceb1bcc 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -202,8 +202,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): pass # Finally try looking in zip files - zip_path = os.path.join(path, sub_path + ".zip") - if os.path.exists(zip_path): + for zip_path in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + '.zip'): # We have a zipfile, so run through it and look for sub files that match the filename with zipfile.ZipFile(zip_path) as zfile: for name in zfile.namelist(): diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 41037d464..af3741bbe 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -14,6 +14,8 @@ from urllib import parse, request from volatility3 import symbols from volatility3.framework import constants, contexts, exceptions, interfaces +from volatility3.framework.automagic import symbol_cache +from volatility3.framework.configuration import requirements from volatility3.framework.configuration.requirements import SymbolTableRequirement from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbconv @@ -74,9 +76,15 @@ class PDBUtility(interfaces.configuration.VersionableInterface): isf_path = None # Take the first result of search for the intermediate file - for value in intermed.IntermediateSymbolTable.file_symbol_url("windows", filter_string): + if not requirements.VersionRequirement.matches_required((1, 0, 0), symbol_cache.SqliteCache.version): + vollog.debug(f"Required version of SQLiteCache not found") + return None + + value = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).find_location( + symbol_cache.WindowsIdentifier.generate(pdb_name.strip('\x00'), guid.upper(), age), 'windows') + + if value: isf_path = value - break else: # If none are found, attempt to download the pdb, convert it and try again cls.download_pdb_isf(context, guid.upper(), age, pdb_name, progress_callback) @@ -336,46 +344,12 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - module_name = guid["pdb_name"].strip('.pdb') - - symbol_table_name = cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) - - new_module_name = None - if create_module: - new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], - symbol_table_name = symbol_table_name) - new_module_name = new_module.name - - return new_module_name, symbol_table_name - - @classmethod - def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, - pdb_name: str, module_offset: int = None, module_size: int = None) -> str: - """Creates a module in the specified layer_name based on a pdb name. - - Searches the memory section of the loaded module for its PDB GUID - and loads the associated symbol table into the symbol space. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - config_path: The config path where to find symbol files - layer_name: The name of the layer on which to operate - module_offset: This memory dump's module image offset - module_size: The size of the module for this dump - - Returns: - The name of the constructed and loaded symbol table - """ - - module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, - module_size, create_module = True) - - return module_name + return cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) class PdbSignatureScanner(interfaces.layers.ScannerInterface): From 2729d25d89576b3d31785c1326671eb86455495e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 00:40:01 +0000 Subject: [PATCH 08/32] Automagic: speed up caching by db commit when necessary --- .../framework/automagic/symbol_cache.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index fe717b8be..4b27b8e0f 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -10,7 +10,7 @@ import urllib import urllib.parse import urllib.request from abc import abstractmethod -from typing import Dict, Generator, List, Optional +from typing import Dict, Generator, List, Optional, Tuple import volatility3.framework import volatility3.schemas @@ -158,10 +158,11 @@ class SqliteCache(CacheManagerInterface): self._database = self._connect_storage(filename) def _connect_storage(self, path: str): - database = sqlite3.connect(path, isolation_level = None) + database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + database.commit() return database def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: @@ -229,6 +230,7 @@ class SqliteCache(CacheManagerInterface): counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) + cursor = self._database.cursor() for location in files_to_process: # Open location counter += 1 @@ -246,7 +248,7 @@ class SqliteCache(CacheManagerInterface): if identifier is not None: # We don't try to validate schemas here, we do that on first use # Store in database - self._database.cursor().execute( + cursor.execute( "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", ( location, @@ -256,7 +258,7 @@ class SqliteCache(CacheManagerInterface): )) vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") else: - self._database.cursor().execute( + cursor.execute( "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", ( location, @@ -267,21 +269,27 @@ class SqliteCache(CacheManagerInterface): vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") except Exception as excp: vollog.log(constants.LOGLEVEL_VVVV, excp) + self._database.commit() if not constants.OFFLINE and constants.REMOTE_ISF_URL: + progress_callback(0, 'Reading remote ISF list') remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) + progress_callback(50, 'Reading remote ISF list') + cursor = self._database.cursor() for operating_system in ['mac', 'linux', 'windows']: identifiers = remote_identifiers.process({}, operating_system = operating_system) - for identifier in identifiers: - for location in identifiers[identifier]: - self._database.cursor().execute( - "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now')", - (location, identifier, operating_system, False) - ) + for identifier, location in identifiers: + cursor.execute( + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + (location, identifier, operating_system, False) + ) + progress_callback(100, 'Reading remote ISF list') + self._database.commit() if missing_locations: self._database.cursor().execute( f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ Dict[bytes, str]: @@ -350,23 +358,23 @@ class RemoteIdentifierFormat: return True return False - def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]): + def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]) -> Generator[ + Tuple[bytes, str], None, None]: raise ValueError("Identifier List version not verified") - def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]): + def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]) -> Generator[ + Tuple[bytes, str], None, None]: if operating_system in self._data: for identifier in self._data[operating_system]: binary_identifier = base64.b64decode(identifier) file_list = identifiers.get(binary_identifier, []) for value in self._data[operating_system][identifier]: - if value not in file_list: - file_list = file_list + [value] - identifiers[binary_identifier] = file_list + yield binary_identifier, value if 'additional' in self._data: for location in self._data['additional']: try: subrbf = RemoteIdentifierFormat(location) - subrbf.process(identifiers, operating_system) + yield from subrbf.process(identifiers, operating_system) except IOError: vollog.debug(f"Remote file not found: {location}") return identifiers From 57a202ae1d69de5968a6a49e9bc199724d364152 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 01:04:32 +0000 Subject: [PATCH 09/32] Automagic: Use cache delay for remote locations --- volatility3/framework/automagic/symbol_cache.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 4b27b8e0f..3c7049986 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -149,6 +149,8 @@ class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) + cache_period = '-3 days' + def __init__(self, filename: str): super().__init__(filename) try: @@ -220,13 +222,15 @@ class SqliteCache(CacheManagerInterface): files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " - "AND cached < date('now', '-3 days');") + f"AND cached < date('now', {self.cache_period});") for row in result: if row['location'] in files_to_timestamp: cache_update.add(row['location']) idextractors = list(volatility3.framework.class_subclasses(IdentifierProcessor)) + # New or not recently updated + counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) @@ -271,11 +275,15 @@ class SqliteCache(CacheManagerInterface): vollog.log(constants.LOGLEVEL_VVVV, excp) self._database.commit() + # Remote Entries + if not constants.OFFLINE and constants.REMOTE_ISF_URL: progress_callback(0, 'Reading remote ISF list') + cursor = self._database.cursor() + cursor.execute( + f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') - cursor = self._database.cursor() for operating_system in ['mac', 'linux', 'windows']: identifiers = remote_identifiers.process({}, operating_system = operating_system) for identifier, location in identifiers: @@ -286,6 +294,8 @@ class SqliteCache(CacheManagerInterface): progress_callback(100, 'Reading remote ISF list') self._database.commit() + # Missing entries + if missing_locations: self._database.cursor().execute( f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) From fe466386406556a17ba2f474558257e4bb4e8457 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Jan 2022 01:36:17 +0000 Subject: [PATCH 10/32] Automagic: Update to use more recent OS categories --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 3c7049986..8bbedf3e8 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -284,7 +284,7 @@ class SqliteCache(CacheManagerInterface): f"SELECT cached FROM cache WHERE remote = True and cached < datetime('now', {self.cache_period})") remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, 'Reading remote ISF list') - for operating_system in ['mac', 'linux', 'windows']: + for operating_system in constants.OS_CATEGORIES: identifiers = remote_identifiers.process({}, operating_system = operating_system) for identifier, location in identifiers: cursor.execute( From 371267f38a61f03007bde4f880b9c45a4b4c2e41 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 21:50:08 +0000 Subject: [PATCH 11/32] Automagic: Ensure partial caching survives --- .../framework/automagic/symbol_cache.py | 80 ++++++++++--------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 8bbedf3e8..54ee13ca2 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -222,7 +222,7 @@ class SqliteCache(CacheManagerInterface): files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: result = self._database.cursor().execute("SELECT location FROM cache WHERE local = True " - f"AND cached < date('now', {self.cache_period});") + f"AND cached < date('now', '{self.cache_period}');") for row in result: if row['location'] in files_to_timestamp: cache_update.add(row['location']) @@ -235,45 +235,47 @@ class SqliteCache(CacheManagerInterface): files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) cursor = self._database.cursor() - for location in files_to_process: - # Open location - counter += 1 - progress_callback(counter * 100 / number_files_to_process, - "Updating caches for {number_files_to_process} files...") - try: - with resources.ResourceAccessor().open(location) as fp: - json_obj = json.load(fp) - identifier = None - for idextractor in idextractors: - identifier = idextractor.get_identifier(json_obj) - operating_system = idextractor.operating_system + try: + for location in files_to_process: + # Open location + counter += 1 + progress_callback(counter * 100 / number_files_to_process, + f"Updating caches for {number_files_to_process} files...") + try: + with resources.ResourceAccessor().open(location) as fp: + json_obj = json.load(fp) + identifier = None + for idextractor in idextractors: + identifier = idextractor.get_identifier(json_obj) + operating_system = idextractor.operating_system + if identifier is not None: + break if identifier is not None: - break - if identifier is not None: - # We don't try to validate schemas here, we do that on first use - # Store in database - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - identifier, - operating_system, - self.is_url_local(location) - )) - vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") - else: - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - None, - None, - self.is_url_local(location) - )) - vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") - except Exception as excp: - vollog.log(constants.LOGLEVEL_VVVV, excp) - self._database.commit() + # We don't try to validate schemas here, we do that on first use + # Store in database + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") + else: + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + ( + location, + None, + None, + self.is_url_local(location) + )) + vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") + except Exception as excp: + vollog.log(constants.LOGLEVEL_VVVV, excp) + finally: + self._database.commit() # Remote Entries From d16861b5925a473c0bf36a0949bc052321197399 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 22:15:35 +0000 Subject: [PATCH 12/32] Documentation: Update documentation for isf caching feature --- doc/source/symbol-tables.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 4dea6077d..d41e8797a 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -12,20 +12,20 @@ Volatility will automatically decompress them on use. It will also cache their under the user's home directory, in :file:`.cache/volatility3`, along with other useful data. The cache directory currently cannot be altered. -Symbol table JSON files live, by default, under the :file:`volatility3/symbols`, underneath an operating system directory -(currently one of :file:`windows`, :file:`mac` or :file:`linux`). The symbols directory is configurable within the framework and can -usually be set within the user interface. +Symbol table JSON files live, by default, under the :file:`volatility3/symbols` directory. The symbols directory is +configurable within the framework and can usually be set within the user interface. These files can also be compressed into ZIP files, which Volatility will process in order to locate symbol files. -The ZIP file must be named after the appropriate operating system (such as `linux.zip`, `mac.zip` or `windows.zip`). -Inside the ZIP file, the directory structure should match the uncompressed operating system directory. + +Volatility maintains a cache mapping the appropriate identifier for each symbol file against its filename. This cache +is update by automagic called as part of the standard automagic that's run each time a plugin is run. Windows symbol tables --------------------- For Windows systems, Volatility accepts a string made up of the GUID and Age of the required PDB file. It then -searches all files under the configured symbol directories under the windows subdirectory. Any that match the filename -pattern of :file:`/-.json` (or any compressed variant) will be used. If such a symbol table cannot be found, then +searches all files under the configured symbol directories under the windows subdirectory. Any that contain metadata +which matches the pdb name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then the associated PDB file will be downloaded from Microsoft's Symbol Server and converted into the appropriate JSON format, and will be saved in the correct location. @@ -41,11 +41,10 @@ or a virtual environment. Mac/Linux symbol tables ----------------------- -For Mac/Linux systems, both use the same mechanism for identification. JSON files live under the symbol directories, -under either the :file:`linux` or :file:`mac` directories. The generated files contain an identifying string (the operating system +For Mac/Linux systems, both use the same mechanism for identification. The generated files contain an identifying string (the operating system banner), which Volatility's automagic can detect. Volatility caches the mapping between the strings and the symbol tables they come from, meaning the precise file names don't matter and can be organized under any necessary hierarchy -under the operating system directory. +under the symbols directory. Linux and Mac symbol tables can be generated from a DWARF file using a tool called `dwarf2json `_. Currently a kernel with debugging symbols is the only suitable means for recovering all the information required by From 2d64deb18ec0b341a40f416429da3e8b0d1ddb44 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 22:52:04 +0000 Subject: [PATCH 13/32] Plugins: Update isfinfo to use the cache unless --live --- .../framework/automagic/symbol_cache.py | 96 +++++++++++++++---- volatility3/framework/constants/__init__.py | 3 + volatility3/framework/plugins/isfinfo.py | 56 ++++++----- 3 files changed, 112 insertions(+), 43 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 54ee13ca2..c09904713 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -104,7 +104,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns the location of the symbol file given the identifier Args: - identifier: string that uniquely identifies a particular symbolt table + identifier: string that uniquely identifies a particular symbol table operating_system: optional string to restrict identifiers to just those for a particular operating system Returns: @@ -144,6 +144,18 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns all identifiers for a particular operating system""" pass + def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + """Returns ISF statistics based on the location + + Returns: + A tuple of base_types, types, enums, symbols, or None is location not found""" + + def get_verified(self, location: str) -> bool: + """Returns whether a location ISF has been verified against its schema""" + + def set_verified(self, location: str, state: bool = True) -> None: + """Sets the verified state of a location based on whether it has been successfully verified against its schema""" + class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) @@ -163,7 +175,23 @@ class SqliteCache(CacheManagerInterface): database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( - 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, local BOOL, cached DATETIME)') + f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCEMA_VERSION})') + schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() + if not schema_version: + database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCEMA_VERSION})') + elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCEMA_VERSION: + # All good, so pass and move on + pass + else: + vollog.info(f"Previous cache schema version found: {schema_version['schema_version']}") + # TODO: Implement code if the schema changes + # Current this should never happen so we start over again + database.close() + os.unlink(path) + return self._connect_storage(path) + database.cursor().execute( + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, verified BOOL DEFAULT False,' + 'stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)') database.commit() return database @@ -207,6 +235,25 @@ class SqliteCache(CacheManagerInterface): return row['identifier'] return None + def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + results = self._database.cursor().execute( + 'SELECT stats_base_types, stats_types, stats_enums, stats_symbols FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['stats_base_types'], row['stats_types'], row['stats_enums'], row['stats_symbols'] + return None + + def get_verified(self, location: str) -> bool: + results = self._database.cursor().execute('SELECT verified FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['verified'] + return False + + def set_verified(self, location: str, state: bool = True) -> None: + self._database.cursor().execute('UPDATE cache (verified) VALUES (?) WHERE location = ?', + (state, location,)) + def update(self, progress_callback = None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. @@ -245,32 +292,39 @@ class SqliteCache(CacheManagerInterface): with resources.ResourceAccessor().open(location) as fp: json_obj = json.load(fp) identifier = None + + # Get stats + stats_base_types = len(json_obj.get('base_types', {})) + stats_types = len(json_obj.get('types', {})) + stats_enums = len(json_obj.get('enums', {})) + stats_symbols = len(json_obj.get('symbols', {})) + + operating_system = None for idextractor in idextractors: identifier = idextractor.get_identifier(json_obj) - operating_system = idextractor.operating_system if identifier is not None: + operating_system = idextractor.operating_system break + + # We don't try to validate schemas here, we do that on first use + # Store in database + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, " + "stats_base_types, stats_types, stats_enums, stats_symbols, " + "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + stats_base_types, + stats_types, + stats_enums, + stats_symbols, + self.is_url_local(location) + )) if identifier is not None: - # We don't try to validate schemas here, we do that on first use - # Store in database - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - identifier, - operating_system, - self.is_url_local(location) - )) vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") else: - cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - ( - location, - None, - None, - self.is_url_local(location) - )) vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") except Exception as excp: vollog.log(constants.LOGLEVEL_VVVV, excp) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 322e574e1..3b499adea 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -76,6 +76,9 @@ MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") """Default location to record information about available identifiers""" +CACHE_SQLITE_SCEMA_VERSION = 1 +"""Version for the sqlite3 cache schema""" + BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" ProgressCallback = Optional[Callable[[float, str], None]] diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index b2960733d..b94cfd69a 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -41,7 +41,11 @@ class IsfInfo(plugins.PluginInterface): optional = True), requirements.VersionRequirement(name = 'SQLiteCache', component = symbol_cache.SqliteCache, - version = (1, 0, 0)) + version = (1, 0, 0)), + requirements.BooleanRequirement(name = 'live', + description = 'Traverse all files, rather than use the cache', + default = False, + optional = True) ] @classmethod @@ -92,28 +96,36 @@ class IsfInfo(plugins.PluginInterface): def check_valid(data): return "Unknown" - # Process the filtered list - for entry in filtered_list: - num_types = num_enums = num_bases = num_symbols = 0 - valid = "Unknown" - with resources.ResourceAccessor().open(url = entry) as fp: - try: - data = json.load(fp) - num_symbols = len(data.get('symbols', [])) - num_types = len(data.get('user_types', [])) - num_enums = len(data.get('enums', [])) - num_bases = len(data.get('base_types', [])) + if self.config['live']: + # Process the filtered list + for entry in filtered_list: + num_types = num_enums = num_bases = num_symbols = 0 + valid = "Unknown" + with resources.ResourceAccessor().open(url = entry) as fp: + try: + data = json.load(fp) + num_symbols = len(data.get('symbols', [])) + num_types = len(data.get('user_types', [])) + num_enums = len(data.get('enums', [])) + num_bases = len(data.get('base_types', [])) - identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) - identifier = identifier_cache.get_identifier(location = entry) - if identifier: - identifier = identifier.decode('utf-8', errors = 'replace') - else: - identifier = renderers.NotAvailableValue() - valid = check_valid(data) - except (UnicodeDecodeError, json.decoder.JSONDecodeError): - vollog.warning(f"Invalid ISF: {entry}") - yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) + identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifier = identifier_cache.get_identifier(location = entry) + if identifier: + identifier = identifier.decode('utf-8', errors = 'replace') + else: + identifier = renderers.NotAvailableValue() + valid = check_valid(data) + except (UnicodeDecodeError, json.decoder.JSONDecodeError): + vollog.warning(f"Invalid ISF: {entry}") + yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) + else: + cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + valid = 'Unknown' + for identifier, location in cache.get_identifier_dictionary().items(): + num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) + if identifier: + yield (0, (location, valid, num_bases, num_types, num_symbols, num_enums, str(identifier))) # Try to open the file, load it as JSON, read the data from it From 1f02fea5d10be5c193f2b100bb18c973335504be Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 26 Feb 2022 23:40:47 +0000 Subject: [PATCH 14/32] Automagic: Change database to store ISF hash instead of verified state --- .../framework/automagic/symbol_cache.py | 27 ++++++++----------- volatility3/framework/plugins/isfinfo.py | 12 +++++++++ volatility3/schemas/__init__.py | 16 +++++++++-- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index c09904713..77bc46265 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -14,6 +14,7 @@ from typing import Dict, Generator, List, Optional, Tuple import volatility3.framework import volatility3.schemas +from volatility3 import schemas from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -150,11 +151,8 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A tuple of base_types, types, enums, symbols, or None is location not found""" - def get_verified(self, location: str) -> bool: - """Returns whether a location ISF has been verified against its schema""" - - def set_verified(self, location: str, state: bool = True) -> None: - """Sets the verified state of a location based on whether it has been successfully verified against its schema""" + def get_hash(self, location: str) -> bool: + """Returns the hash of the JSON from within a location ISF""" class SqliteCache(CacheManagerInterface): @@ -190,7 +188,7 @@ class SqliteCache(CacheManagerInterface): os.unlink(path) return self._connect_storage(path) database.cursor().execute( - 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, verified BOOL DEFAULT False,' + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, hash TEXT,' 'stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)') database.commit() return database @@ -243,16 +241,11 @@ class SqliteCache(CacheManagerInterface): return row['stats_base_types'], row['stats_types'], row['stats_enums'], row['stats_symbols'] return None - def get_verified(self, location: str) -> bool: - results = self._database.cursor().execute('SELECT verified FROM cache WHERE location = ?', + def get_hash(self, location: str) -> Optional[str]: + results = self._database.cursor().execute('SELECT hash FROM cache WHERE location = ?', (location,)).fetchall() for row in results: - return row['verified'] - return False - - def set_verified(self, location: str, state: bool = True) -> None: - self._database.cursor().execute('UPDATE cache (verified) VALUES (?) WHERE location = ?', - (state, location,)) + return row['hash'] def update(self, progress_callback = None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. @@ -291,6 +284,7 @@ class SqliteCache(CacheManagerInterface): try: with resources.ResourceAccessor().open(location) as fp: json_obj = json.load(fp) + hash = schemas.create_json_hash(json_obj) identifier = None # Get stats @@ -309,13 +303,14 @@ class SqliteCache(CacheManagerInterface): # We don't try to validate schemas here, we do that on first use # Store in database cursor.execute( - "INSERT OR REPLACE INTO cache (location, identifier, operating_system, " + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, hash," "stats_base_types, stats_types, stats_enums, stats_symbols, " - "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", + "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", ( location, identifier, operating_system, + hash, stats_base_types, stats_types, stats_enums, diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index b94cfd69a..af095b69d 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -125,6 +125,18 @@ class IsfInfo(plugins.PluginInterface): for identifier, location in cache.get_identifier_dictionary().items(): num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) if identifier: + json_hash = cache.get_hash(location) + if json_hash and json_hash in schemas.cached_validations: + valid = 'True (cached)' + if self.config['validate']: + # Even if we're not live, if we've been explicitly asked to validate, then do-so + with resources.ResourceAccessor().open(url = location) as fp: + try: + data = json.load(fp) + valid = check_valid(data) + except (UnicodeDecodeError, json.decoder.JSONDecodeError): + vollog.warning(f"Invalid ISF: {location}") + yield (0, (location, valid, num_bases, num_types, num_symbols, num_enums, str(identifier))) # Try to open the file, load it as JSON, read the data from it diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 65329a4f5..8666680b3 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -6,7 +6,7 @@ import hashlib import json import logging import os -from typing import Set, Any, Dict +from typing import Any, Dict, Optional, Set from volatility3.framework import constants @@ -51,9 +51,21 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: return valid(input, schema, use_cache) -def create_json_hash(input: Dict[str, Any], schema: Dict[str, Any]) -> str: +def create_json_hash(input: Dict[str, Any], schema: Optional[Dict[str, Any]] = None) -> Optional[str]: """Constructs the hash of the input and schema to create a unique identifier for a particular JSON file.""" + if schema is None: + format = input.get('metadata', {}).get('format', None) + if not format: + vollog.debug("No schema format defined") + return None + basepath = os.path.abspath(os.path.dirname(__file__)) + schema_path = os.path.join(basepath, 'schema-' + format + '.json') + if not os.path.exists(schema_path): + vollog.debug(f"Schema for format not found: {schema_path}") + return None + with open(schema_path, 'r') as s: + schema = json.load(s) return hashlib.sha1(bytes(json.dumps((input, schema), sort_keys = True), 'utf-8')).hexdigest() From bda200168a378f79c76acacf93edb6500f766e55 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sat, 28 May 2022 23:49:15 +0100 Subject: [PATCH 15/32] Update volatility3/framework/plugins/isfinfo.py Yep, good spot as ever, thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/plugins/isfinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index af095b69d..6b13f10b6 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -144,4 +144,4 @@ class IsfInfo(plugins.PluginInterface): def run(self): return renderers.TreeGrid([("URI", str), ("Valid", str), ("Number of base_types", int), ("Number of types", int), ("Number of symbols", int), - ("Number of enums", int), ("Identifying infomration", str)], self._generator()) + ("Number of enums", int), ("Identifying information", str)], self._generator()) From ebab09e53a0c56632edc45260e013b42f8097af2 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sat, 28 May 2022 23:50:23 +0100 Subject: [PATCH 16/32] Update volatility3/framework/automagic/symbol_cache.py Cool, I always forget about that, I think it's just what I'm used to, thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 77bc46265..2908774c0 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -276,7 +276,7 @@ class SqliteCache(CacheManagerInterface): number_files_to_process = len(files_to_process) cursor = self._database.cursor() try: - for location in files_to_process: + for counter, location in enumerate(files_to_process): # Open location counter += 1 progress_callback(counter * 100 / number_files_to_process, From a4aa93f05945ab3c3778a25a0c0d3e4709e62c01 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 28 May 2022 23:51:54 +0100 Subject: [PATCH 17/32] Core: Clean up unneeded counter variable, now we're using enumerate --- volatility3/framework/automagic/symbol_cache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 2908774c0..156a1e8c2 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -271,14 +271,12 @@ class SqliteCache(CacheManagerInterface): # New or not recently updated - counter = 0 files_to_process = new_locations.union(cache_update) number_files_to_process = len(files_to_process) cursor = self._database.cursor() try: for counter, location in enumerate(files_to_process): # Open location - counter += 1 progress_callback(counter * 100 / number_files_to_process, f"Updating caches for {number_files_to_process} files...") try: From 1e80bb54deb5c8a7cf82057f996bf197058828e7 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:34:44 +0100 Subject: [PATCH 18/32] Update volatility3/framework/configuration/requirements.py Yep, not sure why I forgot, thanks 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index b31c4767f..cc4f05ae6 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -414,7 +414,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): return {} @classmethod - def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]): + def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]) -> bool: if len(required) > 0 and version[0] != required[0]: return False if len(required) > 1 and version[1] < required[1]: From 6f34e1350e67ca893d0f1c5984c45813d9892b5a Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:35:42 +0100 Subject: [PATCH 19/32] Update volatility3/framework/automagic/symbol_cache.py Hehehe, I guess I'm just a little shy about handing out complex objects, but you're right and it is a private method. 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 156a1e8c2..efe1ce601 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -169,7 +169,7 @@ class SqliteCache(CacheManagerInterface): os.unlink(filename) self._database = self._connect_storage(filename) - def _connect_storage(self, path: str): + def _connect_storage(self, path: str) -> sqlite3.Connection: database = sqlite3.connect(path) database.row_factory = sqlite3.Row database.cursor().execute( From 98f7fe17433b11a2ef3950e87fabaab8e757e67d Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:49:20 +0100 Subject: [PATCH 20/32] Update volatility3/framework/automagic/symbol_cache.py Quite right, thanks for the catch! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index efe1ce601..47ff66121 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -151,7 +151,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A tuple of base_types, types, enums, symbols, or None is location not found""" - def get_hash(self, location: str) -> bool: + def get_hash(self, location: str) -> Optional[str]: """Returns the hash of the JSON from within a location ISF""" From 504229e46886d9f6d8d3c6a6b8782d67e3656600 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 May 2022 10:52:29 +0100 Subject: [PATCH 21/32] Automagic: include fixes from @digitalisx on review --- volatility3/framework/automagic/symbol_cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 47ff66121..378424ef5 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -10,7 +10,7 @@ import urllib import urllib.parse import urllib.request from abc import abstractmethod -from typing import Dict, Generator, List, Optional, Tuple +from typing import Dict, Generator, Iterable, List, Optional, Tuple import volatility3.framework import volatility3.schemas @@ -113,7 +113,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """ pass - def get_local_locations(self) -> List[str]: + def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" pass @@ -141,7 +141,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): """Returns an identifier based on a specific location or None""" pass - def get_identifiers(self, operating_system: Optional[str]): + def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" pass @@ -369,7 +369,7 @@ class SqliteCache(CacheManagerInterface): output[row['identifier']] = row['location'] return output - def get_identifiers(self, operating_system: Optional[str]): + def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: if operating_system: results = self._database.cursor().execute('SELECT identifier FROM cache WHERE operating_system = ?', (operating_system,)).fetchall() From 47accf520bb322040e0cbf1facfa74e32dc944bb Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 29 May 2022 10:55:17 +0100 Subject: [PATCH 22/32] Update volatility3/framework/automagic/symbol_cache.py Yep, you're quite right, not sure how that got left behind. Thanks! 5:) Co-authored-by: Donghyun Kim --- volatility3/framework/automagic/symbol_cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 378424ef5..ed64746fd 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -426,7 +426,6 @@ class RemoteIdentifierFormat: if operating_system in self._data: for identifier in self._data[operating_system]: binary_identifier = base64.b64decode(identifier) - file_list = identifiers.get(binary_identifier, []) for value in self._data[operating_system][identifier]: yield binary_identifier, value if 'additional' in self._data: From c475b792a53305fe7769134f46d1cf502e29e9b6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jun 2022 17:34:22 +0100 Subject: [PATCH 23/32] Automgic: Fix removing stale entries --- volatility3/framework/automagic/symbol_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index ed64746fd..46431b773 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -347,7 +347,7 @@ class SqliteCache(CacheManagerInterface): if missing_locations: self._database.cursor().execute( - f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", *missing_locations) + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", [x for x in missing_locations]) self._database.commit() def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ From 225c36631403fe3fa58208befc9d36ad93424b83 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jun 2022 18:54:00 +0100 Subject: [PATCH 24/32] Windows: Update PDB to store correct age value --- volatility3/framework/symbols/windows/pdbconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index da8254ffd..15b5c733a 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -521,7 +521,7 @@ class PdbReader: self.metadata['windows']['pdb'] = { "GUID": self.convert_bytes_to_guid(pdb_info.GUID), - "age": pdb_info.age, + "age": self._dbiheader.age, "database": self._database_name or 'unknown.pdb', "machine_type": self._dbiheader.machine } From ae48a8ab479cc1f60550eef6fe9502b0d49f2e74 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 20 Jul 2022 20:40:23 +0100 Subject: [PATCH 25/32] Documentation: Update text about long cache updates --- README.md | 3 +++ doc/source/symbol-tables.rst | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9f9c1bbb7..348121e44 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,9 @@ Symbol tables zip files must be placed, as named, into the `volatility3/symbols` Windows symbols that cannot be found will be queried, downloaded, generated and cached. Mac and Linux symbol tables must be manually produced by a tool such as [dwarf2json](https://github.com/volatilityfoundation/dwarf2json). +Important: The first run of volatility with new symbol files will require the cache to be updated. The symbol packs contain a large number of symbol files and so may take some time to update! +However, this process only needs to be run once on each new symbol file, so assuming the pack stays in the same location will not need to be done again. Please also note it can be interrupted and next run will restart itself. + Please note: These are representative and are complete up to the point of creation for Windows and Mac. Due to the ease of compiling Linux kernels and the inability to uniquely distinguish them, an exhaustive set of Linux symbol tables cannot easily be supplied. ## Documentation diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index d41e8797a..fd8b8933e 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -18,7 +18,9 @@ configurable within the framework and can usually be set within the user interfa These files can also be compressed into ZIP files, which Volatility will process in order to locate symbol files. Volatility maintains a cache mapping the appropriate identifier for each symbol file against its filename. This cache -is update by automagic called as part of the standard automagic that's run each time a plugin is run. +is updated by automagic called as part of the standard automagic that's run each time a plugin is run. If a large number of new +symbols file are detected, this may take some time, but can be safely interrupted and restarted and will not need to run again +as long as the symbol files stay in the same location. Windows symbol tables --------------------- @@ -92,4 +94,4 @@ file, the banners must match exactly (down to the compilation date). * Copy the `.json` file to the symbols directory into `[symbols directory]/linux` - * For Mac change `linux` to `mac` \ No newline at end of file + * For Mac change `linux` to `mac` From d5c7ef1a9e61fca00b40db1dab7ed52343637278 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:16 +0900 Subject: [PATCH 26/32] Remove: pytest install command --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a3ecd7c7e..cf70b66cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -17,7 +17,6 @@ jobs: python -m pip install --upgrade pip pip install Cmake pip install setuptools wheel - pip install -U pytest pip install -r ./test/requirements-testing.txt - name: Build PyPi packages From 5db182d4303db48dbc4ba401f22b9ad5ff354f7c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:39 +0900 Subject: [PATCH 27/32] Add: .gitignore for test --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index d26e17d91..b3c86d49b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ ENV/ # Memory dump files *.dmp *.vmem +*.img + +# PyTest cache files +.pytest_cache/ \ No newline at end of file From 723fd9b4293b5e5231a2dacd05aa973567c0a9ca Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:49:51 +0900 Subject: [PATCH 28/32] Fix: json prettier --- test/known_files.json | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/test/known_files.json b/test/known_files.json index 089896714..a579c8053 100644 --- a/test/known_files.json +++ b/test/known_files.json @@ -1,20 +1,19 @@ { - "windows_dumpfiles": { - "win-xp-laptop-2005-06-25.img": { - "0x82220e78": [ - "9bdd5532286f1660f3778e68bc36efe6", - "e3bc1e9e7370e3b5a661ebe591ecf4ec" - ], - "0x82350bf8": [ - "e5c5e8d97b6280745b41f6572c85d1f0", - "8589f1463422884dbf1411aaad278465" - ], - "0x81eaf418": [ - "f7a1ae2060a58f8470b97affdb46dccf", - "54fd611021fa784912530b8007545986" - ], - "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" - } + "windows_dumpfiles": { + "win-xp-laptop-2005-06-25.img": { + "0x82220e78": [ + "9bdd5532286f1660f3778e68bc36efe6", + "e3bc1e9e7370e3b5a661ebe591ecf4ec" + ], + "0x82350bf8": [ + "e5c5e8d97b6280745b41f6572c85d1f0", + "8589f1463422884dbf1411aaad278465" + ], + "0x81eaf418": [ + "f7a1ae2060a58f8470b97affdb46dccf", + "54fd611021fa784912530b8007545986" + ], + "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" } } - \ No newline at end of file +} From 8e9bf4f27cf26cb4578a053808d5c305c15d171c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:50:46 +0900 Subject: [PATCH 29/32] Add: pytest in requirements-test.txt --- test/requirements-testing.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index d37dc93c3..e47f72fa2 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -6,3 +6,5 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 + +pytest>=7.1.2 \ No newline at end of file From 929d19aa50b8b7eef492bcc4215f39b29055fc16 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:51:54 +0900 Subject: [PATCH 30/32] Add: EOF in requirements-test.txt --- test/requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index e47f72fa2..e5966906c 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -7,4 +7,4 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 -pytest>=7.1.2 \ No newline at end of file +pytest>=7.1.2 From 3587828820a21d02fdb901a6d2c61dd1733ae935 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:54:36 +0900 Subject: [PATCH 31/32] Add: EOF in requirements-test.txt --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b3c86d49b..328ba5f83 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,4 @@ ENV/ *.img # PyTest cache files -.pytest_cache/ \ No newline at end of file +.pytest_cache/ From 986088b1a1b0084c8739200b7ff0792f025cc79b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 21 Jul 2022 05:56:17 +0900 Subject: [PATCH 32/32] Fix: pytest version --- test/requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index e5966906c..7afe19b94 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -7,4 +7,4 @@ pefile>=2017.8.1 #foo # This is required for the yara plugins yara-python>=3.8.0 -pytest>=7.1.2 +pytest>=7.0.0