diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 77fe26931..061c8359a 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -15,14 +15,16 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.6"] steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.x - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..8f044a365 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,55 @@ +name: Test Volatility3 +on: [push, pull_request] +jobs: + + build: + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.6"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install Cmake + pip install setuptools wheel + pip install -r ./test/requirements-testing.txt + + - name: Build PyPi packages + run: | + python setup.py sdist --formats=gztar,zip + python setup.py bdist_wheel + + - name: Download images + run: | + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz" + gunzip linux-sample-1.bin.gz + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" + gunzip win-xp-laptop-2005-06-25.img.gz + + - name: Download and Extract symbols + run: | + cd ./volatility3/symbols + curl -sLO https://downloads.volatilityfoundation.org/volatility3/symbols/linux.zip + unzip linux.zip + cd - + + - name: Testing... + run: | + py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v + py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + + - name: Clean up post-test + run: | + rm -rf *.lime + rm -rf *.img + cd volatility3/symbols + rm -rf linux + rm -rf linux.zip + cd - diff --git a/.gitignore b/.gitignore index c6da33754..328ba5f83 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,18 @@ config*.json # Pyinstaller files build dist + +# Environments +.env +.venv +env/ +venv/ +ENV/ + +# Memory dump files +*.dmp +*.vmem +*.img + +# PyTest cache files +.pytest_cache/ diff --git a/.style.yapf b/.style.yapf index 8159be910..3f154e07b 100644 --- a/.style.yapf +++ b/.style.yapf @@ -107,7 +107,7 @@ each_dict_entry_on_separate_line=True i18n_comment= # The i18n function call names. The presence of this function stops -# reformattting on that line, because the string it has cannot be moved +# reformatting on that line, because the string it has cannot be moved # away from the i18n comment. i18n_function_call= diff --git a/API_CHANGES.md b/API_CHANGES.md index 4a65de04b..98a08f09d 100644 --- a/API_CHANGES.md +++ b/API_CHANGES.md @@ -4,6 +4,41 @@ API Changes When an addition to the existing API is made, the minor version is bumped. When an API feature or function is removed or changed, the major version is bumped. +2.4.0 +===== +Add a `get_size()` method to Windows VAD structures and fix several off-by-one issues when calculating VAD sizes. + +2.3.1 +===== +Update in the windows `_EPROCESS.owning_process` method to support Windows Vista and later versions. + +2.3.0 +===== +Add in `child_template` to template class + +2.2.0 +===== +Changes to linux core calls + +2.1.0 +===== +Add in the linux `task.get_threads` method to the API. + +2.0.3 +===== +Add in the windows `DEVICE_OBJECT.get_attached_devices` and `DRIVER_OBJECT.get_devices` methods to the API. + +2.0.2 +===== +Fix the behaviour of the offsets returned by the PDB scanner. + +2.0.0 +===== +Remove the `symbol_shift` mechanism, where symbol tables could alter their own symbols. +Symbols from a symbol table are now always the offset values. They can be added to a Module +and when symbols are requested from a Module they are shifted by the module's offset to get +an absolute offset. This can be done with `Module.get_absolute_symbol_address` or as part of +`Module.object_from_symbol(absolute = False, ...)`. 1.2.0 ===== diff --git a/LICENSE.txt b/LICENSE.txt index 96f222187..2a37fd0ed 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -31,7 +31,7 @@ If you make any Additions available to others, such as by providing copies of th - You are responsible to ensure you have rights in Additions necessary to comply with this section. Contributing -If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution." +If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution." Trademarks This license grants you no rights to any trademarks or service marks. diff --git a/MANIFEST.in b/MANIFEST.in index 504c7d89a..1cec729f6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ prune development include * .* -include doc/make.bat doc/Makefile +include doc/make.bat doc/Makefile doc/requirements.txt recursive-include doc/source * recursive-include volatility3 *.json recursive-exclude doc/source volatility3.*.rst diff --git a/README.md b/README.md index 9f9c1bbb7..348121e44 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,9 @@ Symbol tables zip files must be placed, as named, into the `volatility3/symbols` Windows symbols that cannot be found will be queried, downloaded, generated and cached. Mac and Linux symbol tables must be manually produced by a tool such as [dwarf2json](https://github.com/volatilityfoundation/dwarf2json). +Important: The first run of volatility with new symbol files will require the cache to be updated. The symbol packs contain a large number of symbol files and so may take some time to update! +However, this process only needs to be run once on each new symbol file, so assuming the pack stays in the same location will not need to be done again. Please also note it can be interrupted and next run will restart itself. + Please note: These are representative and are complete up to the point of creation for Windows and Mac. Due to the ease of compiling Linux kernels and the inability to uniquely distinguish them, an exhaustive set of Linux symbol tables cannot easily be supplied. ## Documentation diff --git a/development/stock-linux-json.py b/development/stock-linux-json.py index c863d41e4..877f78e1c 100644 --- a/development/stock-linux-json.py +++ b/development/stock-linux-json.py @@ -87,7 +87,7 @@ class Downloader: output_filename = 'unknown-kernel.json' for named_file in named_files: prefix = '--system-map' - if not 'System' in named_files[named_file]: + if 'System' not in named_files[named_file]: prefix = '--elf' output_filename = './' + '-'.join((named_file.split('/')[-1]).split('-')[2:])[:-4] + '.json.xz' args += [prefix, named_files[named_file]] diff --git a/doc/requirements.txt b/doc/requirements.txt index d646d22ce..93d6ea70a 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,4 @@ # These packages are required for building the documentation. -sphinx>=1.8.2 +sphinx>=4.0.0 sphinx_autodoc_typehints>=1.4.0 -sphinx-rtd-theme>=0.4.3 \ No newline at end of file +sphinx-rtd-theme>=0.4.3 diff --git a/doc/source/complex-plugin.rst b/doc/source/complex-plugin.rst index f06b398e8..8ab8a5186 100644 --- a/doc/source/complex-plugin.rst +++ b/doc/source/complex-plugin.rst @@ -300,7 +300,7 @@ This will mean that when a specific structure is loaded from the symbol_space, i `StructType`, but instead is instantiated using the NewStructureClass, meaning new methods can be called directly on it. If the situation really calls for an entirely new object, that isn't covered by one of the existing -:py:class:`~volatility3.framework.objects.PrimativeObject` objects (such as +:py:class:`~volatility3.framework.objects.PrimitiveObject` objects (such as :py:class:`~volatility3.framework.objects.Integer`, :py:class:`~volatility3.framework.objects.Boolean`, :py:class:`~volatility3.framework.objects.Float`, diff --git a/doc/source/conf.py b/doc/source/conf.py index 3be41db5d..cadf6d3f2 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -84,6 +84,15 @@ def setup(app): for line in submodule_lines: contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins')) + # Clear up the framework.plugins page + with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "rb") as contents: + real_lines = contents.readlines() + + with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "wb") as contents: + for line in real_lines: + if b'volatility3.framework.plugins.' not in line: + contents.write(line) + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -102,9 +111,11 @@ needs_sphinx = '2.0' # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'sphinx.ext.viewcode' + 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.autosectionlabel' ] +autosectionlabel_prefix_document = True + try: import sphinx_autodoc_typehints diff --git a/doc/source/development.rst b/doc/source/development.rst new file mode 100644 index 000000000..ade068322 --- /dev/null +++ b/doc/source/development.rst @@ -0,0 +1,8 @@ +Writing Plugins +=============== + +.. toctree:: + + simple-plugin + complex-plugin + using-as-a-library diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst new file mode 100644 index 000000000..26ad2c2e4 --- /dev/null +++ b/doc/source/getting-started-linux-tutorial.rst @@ -0,0 +1,192 @@ +Linux Tutorial +============== + +This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite. + +Acquiring memory +---------------- + +Volatility3 does not provide the ability to acquire memory. Below are some examples of tools that can be used to acquire memory, but more are available: + +* `AVML - Acquire Volatile Memory for Linux `_ +* `LiME - Linux Memory Extract `_ + + +Procedure to create symbol tables for linux +-------------------------------------------- + +To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. + +.. tip:: It may be possible to locate pre-made ISF files from the `Linux ISF Server `_ , + which is built and maintained by `kevthehermit `_. + After creating the file or downloading it from the ISF server, place the file under the directory ``volatility3/symbols/linux``. + If necessary create a linux directory under the symbols directory (this will become unnecessary in future versions). + + +Listing plugins +--------------- + +The following is a sample of the linux plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. + +.. code-block:: shell-session + + $ python3 vol.py --help | grep -i linux. | head -n 5 + banners.Banners Attempts to identify potential linux banners in an + linux.bash.Bash Recovers bash command history from memory. + linux.check_afinfo.Check_afinfo + linux.check_creds.Check_creds + linux.check_idt.Check_idt + +.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of linux plugins. + + +Using plugins +------------- + +The following is the syntax to run the volatility CLI. + +.. code-block:: shell-session + + $ python3 vol.py -f + + +Example +------- + +banners +~~~~~~~ + +In this example we will be using a memory dump from the Insomni'hack teaser 2020 CTF Challenge called Getdents. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge. +Thanks go to `stuxnet `_ for providing this memory dump and `writeup `_. + + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem banners + + Volatility 3 Framework 2.0.1 + + Progress: 100.00 PDB scanning finished + Offset Banner + + 0x141c1390 Linux version 4.15.0-42-generic (buildd@lgw01-amd64-023) (gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)) #45-Ubuntu SMP Thu Nov 15 19:32:57 UTC 2018 (Ubuntu 4.15.0-42.45-generic 4.15.18) + 0x63a00160 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x6455c4d4 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x6e1e055f Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x7fde0010 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + + +The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. + +.. tip:: Use the banner text which is most repeated to search from ISF Server. + +linux.pslist +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.pslist + + Volatility 3 Framework 2.0.1 Stacking attempts finished + + PID PPID COMM + + 1 0 systemd + 2 0 kthreadd + 3 2 kworker/0:0 + 4 2 kworker/0:0H + 5 2 kworker/u256:0 + 6 2 mm_percpu_wq + 7 2 ksoftirqd/0 + 8 2 rcu_sched + 9 2 rcu_bh + 10 2 migration/0 + 11 2 watchdog/0 + 12 2 cpuhp/0 + 13 2 kdevtmpfs + 14 2 netns + 15 2 rcu_tasks_kthre + 16 2 kauditd + ..... + +``linux.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. + +linux.pstree +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.pstree + Volatility 3 Framework 2.0.1 + Progress: 100.00 Stacking attempts finished + PID PPID COMM + + 1 0 systemd + * 636 1 polkitd + * 514 1 acpid + * 1411 1 pulseaudio + * 517 1 rsyslogd + * 637 1 cups-browsed + * 903 1 whoopsie + * 522 1 ModemManager + * 525 1 cron + * 526 1 avahi-daemon + ** 542 526 avahi-daemon + * 657 1 unattended-upgr + * 914 1 kerneloops + * 532 1 dbus-daemon + * 1429 1 ibus-x11 + * 929 1 kerneloops + * 1572 1 gsd-printer + * 933 1 upowerd + * 1071 1 rtkit-daemon + * 692 1 gdm3 + ** 1234 692 gdm-session-wor + *** 1255 1234 gdm-x-session + **** 1257 1255 Xorg + **** 1266 1255 gnome-session-b + ***** 1537 1266 gsd-clipboard + ***** 1539 1266 gsd-color + ***** 1542 1266 gsd-datetime + ***** 2950 1266 deja-dup-monito + ***** 1546 1266 gsd-housekeepin + ***** 1548 1266 gsd-keyboard + ***** 1550 1266 gsd-media-keys + +``linux.pstree`` helps us to display the parent child relationships between processes. + +linux.bash +~~~~~~~~~~ + +Now to find the commands that were run in the bash shell by using ``linux.bash``. + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.bash + + Volatility 3 Framework 2.0.1 + Progress: 100.00 Stacking attempts finished + PID Process CommandTime Command + + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 AWAVH�� + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 rub + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 uname -a + 1733 bash 2020-01-16 14:00:36.000000 uname -a + 1733 bash 2020-01-16 14:00:36.000000 sudo apt autoclean + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:41.000000 chmod +x meterpreter + 1733 bash 2020-01-16 14:00:42.000000 sudo ./meterpreter diff --git a/doc/source/getting-started-windows-tutorial.rst b/doc/source/getting-started-windows-tutorial.rst new file mode 100644 index 000000000..c89b065f5 --- /dev/null +++ b/doc/source/getting-started-windows-tutorial.rst @@ -0,0 +1,124 @@ +Windows Tutorial +================ + +This guide provides a brief introduction to how volatility3 works as a demonstration of several of the plugins available in the suite. + +Acquiring memory +---------------- + +Volatility does not provide the ability to acquire memory. +Memory can be acquired using a number of tools, below are some examples but others exist: + +* `WinPmem `_ +* `FTK Imager `_ + +Listing Plugins +--------------- + +The following is a sample of the windows plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. + +.. code-block:: shell-session + + $ python3 vol.py --help | grep windows | head -n 5 + windows.bigpools.BigPools + windows.cmdline.CmdLine + windows.crashinfo.Crashinfo + windows.dlllist.DllList + +.. note:: Here the the command is piped to grep and head in-order to provide the start of a list of the available windows plugins. + +Using plugins +------------- + +The following is the syntax to run the volatility CLI. + +.. code-block:: shell-session + + $ python3 vol.py -f plugin_name plugin_option + + +Example +------- + +windows.pslist +~~~~~~~~~~~~~~ + +In this example we will be using a memory dump from the PragyanCTF'22. +We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenges. + +When using windows plugins in volatility 3, the required ISF file can often be generated from PDB files automatically +downloaded from Microsoft servers, and therefore does not require locating or adding specific ISF files to the volatility 3 symbols directory. + +.. code-block:: shell-session + + $ python3 vol.py -f MemDump.DMP windows.pslist | head -n 10 + + Volatility 3 Framework 2.0.1 PDB scanning finished + + PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output + + 4 0 System 0xfa8000cbc040 85 492 N/A False 2022-02-07 16:30:12.000000 N/A Disabled + 276 4 smss.exe 0xfa8001e04040 2 29 N/A False 2022-02-07 16:30:12.000000 N/A Disabled + 352 336 csrss.exe 0xfa8002110b30 9 375 0 False 2022-02-07 16:30:13.000000 N/A Disabled + 404 336 wininit.exe 0xfa800219f060 3 74 0 False 2022-02-07 16:30:13.000000 N/A Disabled + 412 396 csrss.exe 0xfa80021c5b30 9 224 1 False 2022-02-07 16:30:13.000000 N/A Disabled + 468 396 winlogon.exe 0xfa8002284060 5 113 1 False 2022-02-07 16:30:14.000000 N/A Disabled + +``windows.pslist`` helps list the processes running while the memory dump was taken. + +windows.pstree +~~~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f MemDump.DMP windows.pstree | head -n 20 + Volatility 3 Framework 2.0.1 PDB scanning finished + + PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime + + 4 0 System 0xfa8000cbc040 85 492 N/A False 2022-02-07 16:30:12.000000 N/A + * 276 4 smss.exe 0xfa8001e04040 2 29 N/A False 2022-02-07 16:30:12.000000 N/A + 352 336 csrss.exe 0xfa8002110b30 9 375 0 False 2022-02-07 16:30:13.000000 N/A + 404 336 wininit.exe 0xfa800219f060 3 74 0 False 2022-02-07 16:30:13.000000 N/A + * 504 404 services.exe 0xfa80022ccb30 7 190 0 False 2022-02-07 16:30:14.000000 N/A + ** 960 504 svchost.exe 0xfa8001c17b30 39 1003 0 False 2022-02-07 16:30:14.000000 N/A + ** 1216 504 svchost.exe 0xfa80026e0b30 18 311 0 False 2022-02-07 16:30:15.000000 N/A + ** 1312 504 svchost.exe 0xfa8002740380 19 287 0 False 2022-02-07 16:30:15.000000 N/A + ** 1984 504 taskhost.exe 0xfa8002eb1b30 8 129 1 False 2022-02-07 16:30:27.000000 N/A + ** 804 504 svchost.exe 0xfa80024ca5f0 20 450 0 False 2022-02-07 16:30:14.000000 N/A + *** 100 804 audiodg.exe 0xfa80025b4b30 6 131 0 False 2022-02-07 16:30:14.000000 N/A + ** 1568 504 SearchIndexer. 0xfa800254b480 12 616 0 False 2022-02-07 16:30:32.000000 N/A + ** 744 504 svchost.exe 0xfa8002477b30 8 265 0 False 2022-02-07 16:30:14.000000 N/A + ** 1096 504 svchost.exe 0xfa800260db30 14 357 0 False 2022-02-07 16:30:14.000000 N/A + ** 616 504 svchost.exe 0xfa8002b86ab0 13 314 0 False 2022-02-07 16:32:16.000000 N/A + ** 624 504 svchost.exe 0xfa8002410630 10 350 0 False 2022-02-07 16:30:14.000000 N/A + +``windows.pstree`` helps to display the parent child relationships between processes. + +.. note:: Here the the command is piped to head in-order to provide smaller output, here listing only the first 20. + +windows.hashdump +~~~~~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f MemDump.DMP windows.hashdump + Volatility 3 Framework 2.0.3 + Progress: 100.00 PDB scanning finished + User rid lmhash nthash + + Administrator 500 aad3b435b51404eeaad3b435b51404ee 31d6cfe0d16ae931b73c59d7e0c089c0 + Guest 501 aad3b435b51404eeaad3b435b51404ee 31d6cfe0d16ae931b73c59d7e0c089c0 + Frank Reynolds 1000 aad3b435b51404eeaad3b435b51404ee a88d1e18706d3aa676e01e5943d15911 + HomeGroupUser$ 1002 aad3b435b51404eeaad3b435b51404ee af10ecac6ea817d2bb56e3e5c33ce1cd + Dennis 1003 aad3b435b51404eeaad3b435b51404ee cf96684bbc7877920adaa9663698bf54 + +``windows.hashdump`` helps to list the hashes of the users in the system. + + + + + + diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 68bc41e4c..66dabfafe 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -145,7 +145,7 @@ Struct, Structure Symbol This is used in many different contexts, as a short term for many things. Within Volatility, a symbol is a - construct that usually encompasses a specific type :ref:`type` at a specfific :ref:`offset`, + construct that usually encompasses a specific type :ref:`type` at a specific :ref:`offset`, representing a particular instance of that type within the memory of a compiled and running program. An example would be the location in memory of a list of active tcp endpoints maintained by the networking stack within an operating system. diff --git a/doc/source/index.rst b/doc/source/index.rst index 0dc3b5025..9b1d05858 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -7,25 +7,33 @@ Volatility 3 is Open Source. :doc:`List of plugins ` -Here are some guidelines for using Volatility 3 effectively: +Below is the main documentation regarding volatility 3: .. toctree:: + :caption: Documentation basics - simple-plugin - vol2to3 - complex-plugin - using-as-a-library + development symbol-tables + vol2to3 volshell glossary -Python Packages -=============== +There is also some information to get you started quickly: .. toctree:: + :caption: Getting Started + + getting-started-linux-tutorial + getting-started-windows-tutorial + + +.. toctree:: + :caption: Python Packages + volatility3 + Indices and tables ================== diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 4e499b186..c4908caf3 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -6,6 +6,12 @@ This guide will step through how to construct a simple plugin using Volatility 3 The example plugin we'll use is :py:class:`~volatility3.plugins.windows.dlllist.DllList`, which features the main traits of a normal plugin, and reuses other plugins appropriately. +.. note:: + + This document will not include the complete code necessary for a + working plugin (such as imports, etc) since it's designed to focus on the necessary components for writing a plugin. + For complete and functioning plugins, the ``framework/plugins`` directory should be consulted. + Inherit from PluginInterface ---------------------------- @@ -30,20 +36,20 @@ to be able to run properly. Any that are defined as optional need not necessari :: + _version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + @classmethod def get_requirements(cls): - return [requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (1, 0, 0)), + return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.ListRequirement(name = 'pid', element_type = int, description = "Process IDs to include (all other processes are excluded)", - optional = True)] + optional = True), + requirements.PluginRequirement(name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0))] This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how @@ -51,69 +57,112 @@ to instantiate the plugin). At the moment these requirements are fairly straigh :: - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), -This requirement indicates that the plugin will operate on a single -:py:class:`TranslationLayer `. The name of the -loaded layer will appear in the plugin's configuration under the name ``primary``. Requirement values can be -accessed within the plugin through the plugin's `config` attribute (for example ``self.config['pid']``). +This requirement specifies the need for a particular submodule. Each module requires a +:py:class:`TranslationLayer ` and a +:py:class:`SymbolTable `, which are fulfilled by two +subrequirements: a +:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` and a +:py:class:`~volatility3.framework.configuration.requirements.SymbolTableRequirement`. At the moment, the automagic +only fills `ModuleRequirements` with kernels, and so has relatively few parameters. It requires the architecture for +the underlying TranslationLayer, and the offset of the module within that layer. -.. note:: The name itself is dynamic depending on the other layers already present in the Context. Always use the value - from the configuration rather than attempting to guess what the layer will be called. +The name of the module will be stored in the ``kernel`` configuration option, and the module object itself +can be accessed from the ``context.modules`` collection. This requirement is a Complex Requirement and therefore will +not be requested directly from the user. -Finally, this defines that the translation layer must be on the Intel Architecture. At the moment, this acts as a filter, -failing to be satisfied by memory images that do not match the architecture required. -Most plugins will only operate on a single layer, but it is entirely possible for a plugin to request two different -layers, for example a plugin that carries out some form of difference or statistics against multiple memory images. +.. note:: -This requirement (and the next two) are known as Complex Requirements, and user interfaces will likely not directly -request a value for this from a user. The value stored in the configuration tree for a -:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` is -the string name of a layer present in the context's memory that satisfies the requirement. + In previous versions of volatility 3, there was no `ModuleRequirement`, and instead two requirements were defined + a :py:class:`TranslationLayer ` and a `SymbolTableRequirement`. These still exist, and can be used, most plugins just + define a single `ModuleRequirement` for the kernel, which the automagic will populate. The `ModuleRequirement` has + two automatic sub-requirements, a `TranslationLayerRequirement` and a `SymbolTableRequirement`, but the module also + includes the offset of the module, and will allow future expansion to specify specific modules when application + level plugins become more common. Below are how the requirements would be specified: -:: + :: - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), -This requirement specifies the need for a particular -:py:class:`SymbolTable ` -to be loaded. This gets populated by various -:py:class:`Automagic ` as the nearest sibling to a particular -:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`. -This means that if the :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` -is satisfied and the :py:class:`Automagic ` can determine -the appropriate :py:class:`SymbolTable `, the -name of the :py:class:`SymbolTable ` will be stored in the configuration. + This requirement indicates that the plugin will operate on a single + :py:class:`TranslationLayer `. The name of the + loaded layer will appear in the plugin's configuration under the name ``primary``. Requirement values can be + accessed within the plugin through the plugin's `config` attribute (for example ``self.config['pid']``). -This requirement is also a Complex Requirement and therefore will not be requested directly from the user. + .. note:: The name itself is dynamic depending on the other layers already present in the Context. Always use the value + from the configuration rather than attempting to guess what the layer will be called. -:: + Finally, this defines that the translation layer must be on the Intel Architecture. At the moment, this acts as a filter, + failing to be satisfied by memory images that do not match the architecture required. - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (1, 0, 0)), + Most plugins will only operate on a single layer, but it is entirely possible for a plugin to request two different + layers, for example a plugin that carries out some form of difference or statistics against multiple memory images. -This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements -on that plugin. The version is specified in terms of Semantic Versioning, meaning that to be compatible, the major -versions must be identical and the minor version must be equal to or higher than the one provided. This requirement -does not make use of any data from the configuration, even if it were provided, it is merely a functional check before -running the plugin. + This requirement (and the next two) are known as Complex Requirements, and user interfaces will likely not directly + request a value for this from a user. The value stored in the configuration tree for a + :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` is + the string name of a layer present in the context's memory that satisfies the requirement. + + :: + + requirements.SymbolTableRequirement(name = "nt_symbols", + description = "Windows kernel symbols"), + + This requirement specifies the need for a particular + :py:class:`SymbolTable ` + to be loaded. This gets populated by various + :py:class:`Automagic ` as the nearest sibling to a particular + :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`. + This means that if the :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` + is satisfied and the :py:class:`Automagic ` can determine + the appropriate :py:class:`SymbolTable `, the + name of the :py:class:`SymbolTable ` will be stored in the configuration. + + This requirement is also a Complex Requirement and therefore will not be requested directly from the user. :: requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, - optional = True) + optional = True), -The final requirement is a List Requirement, populated by integers. The description will be presented to the user to +The next requirement is a List Requirement, populated by integers. The description will be presented to the user to describe what the value represents. The optional flag indicates that the plugin can function without the ``pid`` value being defined within the configuration tree at all. +:: + + requirements.PluginRequirement(name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0))] + +This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements +on that plugin. The version is specified in terms of Semantic Versioning meaning that, to be compatible, the major +versions must be identical and the minor version must be equal to or higher than the one provided. This requirement +does not make use of any data from the configuration, even if it were provided, it is merely a functional check before +running the plugin. To define the version of a plugin, populate the `_version` class variable as a tuple of version +numbers `(major, minor, patch)`. So for example: + +:: + + _version = (1, 0, 0) + +The plugin may also require a specific version of the framework, and this also uses Semantic Versioning, and can be +set by defining the `_required_framework_version`. The major version should match the version of volatility the plugin +is to be used with, which at the time of writing would be 2.2.0, and so would be specified as below. If only features, for example, +from 2.0.0 are used, then the lowest applicable version number should be used to support the greatest number of +installations: + +:: + + _required_framework_version = (2, 0, 0) + Define the `run` method ----------------------- @@ -129,6 +178,7 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] return renderers.TreeGrid([("PID", int), ("Process", str), @@ -137,8 +187,8 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. ("Name", str), ("Path", str)], self._generator(pslist.PsList.list_processes(self.context, - self.config['primary'], - self.config['nt_symbols'], + kernel.layer_name, + kernel.symbol_table_name, filter_func = filter_func))) In this instance, the plugin constructs a filter (using the PsList plugin's *classmethod* for creating filters). @@ -157,7 +207,8 @@ the :py:class:`~volatility3.plugins.windows.pslist.PsList` plugin. That plugin so that other plugins can call it. As such, it takes all the necessary parameters rather than accessing them from a configuration. Since it must be portable code, it takes a context, as well as the layer name, symbol table and optionally a filter. In this instance we unconditionally -pass it the values from the configuration for the ``primary`` and ``nt_symbols`` requirements. This will generate a list +pass it the values from the configuration for the layer and symbol table from the kernel module object, constructed from +the ``kernel`` configuration requirement. This will generate a list of :py:class:`~volatility3.framework.symbols.windows.extensions.EPROCESS` objects, as provided by the :py:class:`~volatility.plugins.windows.pslist.PsList` plugin, and is not covered here but is used as an example for how to share code across plugins (both as the provider and the consumer of the shared code). @@ -196,7 +247,7 @@ The plugin then defaults the ``BaseDllName`` and ``FullDllName`` variables to an which is a way of indicating to the user interface that the value couldn't be read for some reason (but that it isn't fatal). There are currently four different reasons a value may be unreadable: -* **Unreadble**: values which are empty because the data cannot be read +* **Unreadable**: values which are empty because the data cannot be read * **Unparsable**: values which are empty because the data cannot be interpreted correctly * **NotApplicable**: values which are empty because they don't make sense for this particular entry * **NotAvailable**: values which cannot be provided now (but might in a future run, via new symbols or an updated plugin) @@ -206,9 +257,9 @@ information may not be provided. The plugin then takes the process's ``BaseDllName`` value, and calls :py:meth:`~volatility3.framework.symbols.windows.extensions.UNICODE_STRING.get_string` on it. All structure attributes, as defined by the symbols, are directly accessible and use the case-style of the symbol library it came from (in Windows, -attributes are CamelCase), such as ``entry.BaseDllName`` in this instance. Any attribtues not defined by the symbol but added +attributes are CamelCase), such as ``entry.BaseDllName`` in this instance. Any attributes not defined by the symbol but added by Volatility extensions cannot be properties (in case they overlap with the attributes defined in the symbol libraries) -and are therefore always methods and prepended with ``get_``, in this example ``BaseDllName.get_string()``. +and are therefore always methods and pretended with ``get_``, in this example ``BaseDllName.get_string()``. Finally, ``FullDllName`` is populated. These operations read from memory, and as such, the memory image may be unable to read the data at a particular offset. This will cause an exception to be thrown. In Volatility 3, exceptions are thrown diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 31329b9eb..b7c26e046 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -12,20 +12,22 @@ Volatility will automatically decompress them on use. It will also cache their under the user's home directory, in :file:`.cache/volatility3`, along with other useful data. The cache directory currently cannot be altered. -Symbol table JSON files live, by default, under the :file:`volatility3/symbols`, underneath an operating system directory -(currently one of :file:`windows`, :file:`mac` or :file:`linux`). The symbols directory is configurable within the framework and can -usually be set within the user interface. +Symbol table JSON files live, by default, under the :file:`volatility3/symbols` directory. The symbols directory is +configurable within the framework and can usually be set within the user interface. These files can also be compressed into ZIP files, which Volatility will process in order to locate symbol files. -The ZIP file must be named after the appropriate operating system (such as `linux.zip`, `mac.zip` or `windows.zip`). -Inside the ZIP file, the directory structure should match the uncompressed operating system directory. + +Volatility maintains a cache mapping the appropriate identifier for each symbol file against its filename. This cache +is updated by automagic called as part of the standard automagic that's run each time a plugin is run. If a large number of new +symbols file are detected, this may take some time, but can be safely interrupted and restarted and will not need to run again +as long as the symbol files stay in the same location. Windows symbol tables --------------------- For Windows systems, Volatility accepts a string made up of the GUID and Age of the required PDB file. It then -searches all files under the configured symbol directories under the windows subdirectory. Any that match the filename -pattern of :file:`/-.json` (or any compressed variant) will be used. If such a symbol table cannot be found, then +searches all files under the configured symbol directories under the windows subdirectory. Any that contain metadata +which matches the pdb name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then the associated PDB file will be downloaded from Microsoft's Symbol Server and converted into the appropriate JSON format, and will be saved in the correct location. @@ -38,14 +40,13 @@ following command: The :envvar:`PYTHONPATH` environment variable is not required if the Volatility library is installed in the system's library path or a virtual environment. -Mac/Linux symbol tables ------------------------ +Mac or Linux symbol tables +-------------------------- -For Mac/Linux systems, both use the same mechanism for identification. JSON files live under the symbol directories, -under either the :file:`linux` or :file:`mac` directories. The generated files contain an identifying string (the operating system +For Mac/Linux systems, both use the same mechanism for identification. The generated files contain an identifying string (the operating system banner), which Volatility's automagic can detect. Volatility caches the mapping between the strings and the symbol tables they come from, meaning the precise file names don't matter and can be organized under any necessary hierarchy -under the operating system directory. +under the symbols directory. Linux and Mac symbol tables can be generated from a DWARF file using a tool called `dwarf2json `_. Currently a kernel with debugging symbols is the only suitable means for recovering all the information required by @@ -63,7 +64,7 @@ To determine the string for a particular memory image, use the `banners` plugin. try to locate that exact kernel debugging package for the operating system. Unfortunately each distribution provides its debugging packages under different package names and there are so many that the distribution may not keep all old versions of the debugging symbols, and therefore **it may not be possible to find the right symbols to analyze a linux -memory image with volatlity**. With Macs there are far fewer kernels and only one distribution, making it easier to +memory image with volatility**. With Macs there are far fewer kernels and only one distribution, making it easier to ensure that the right symbols can be found. Once a kernel with debugging symbols/appropriate DWARF file has been located, `dwarf2json `_ will convert it into an @@ -76,3 +77,21 @@ The banners available for volatility to use can be found using the `isfinfo` plu long time to run depending on the number of JSON files available. This will list all the JSON (ISF) files that volatility3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON file, the banners must match exactly (down to the compilation date). + +.. note:: + + Steps for constructing a new kernel ISF JSON file: + + * Run the `banners` plugin on the image to determine the necessary kernel + * Locate a copy of the debug kernel that matches the identified banner + + * Clone or update the dwarf2json repo: :code:`git clone https://github.com/volatilityfoundation/dwarf2json` + * Run :code:`go build` in the directory if the source has changed + + * Run :code:`dwarf2json linux --elf [path to debug kernel] > [kernel name].json` + + * For Mac change `linux` to `mac` + + * Copy the `.json` file to the symbols directory into `[symbols directory]/linux` + + * For Mac change `linux` to `mac` diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index 95d2b1080..c63adcfc3 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -131,7 +131,8 @@ A suitable list of automagics for a particular plugin (based on operating system automagics = automagic.choose_automagic(available_automagics, plugin) This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just -the automagics which apply to the operating system. +the automagics which apply to the operating system. Each automagic can exclude itself from being used for specific +operating systems, so that an automagic designed for linux is not used for windows or mac plugins. These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that the plugin will be constructed on ('plugins' by default) and a progress_callback. This is a callable which takes diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 9db29c818..cc6f7fe6a 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -9,7 +9,11 @@ Synopsis **volatility** [-h] [-c CONFIG] [--parallelism [{processes,threads,off}]] [-e EXTEND] [-p PLUGIN_DIRS] [-s SYMBOL_DIRS] [-v] [-l LOG] [-o OUTPUT_DIR] [-q] [-r RENDERER] [-f FILE] - [--write-config] [--single-location SINGLE_LOCATION] + [--write-config] [--save-config SAVE_CONFIG] + [--clear-cache] [--cache-path CACHE_PATH] + [--offline] + [--single-location SINGLE_LOCATION] + [--stackers [STACKERS ...]] [--single-swap-locations SINGLE_SWAP_LOCATIONS] ... @@ -98,6 +102,10 @@ Options attempt to build upon, and can be considered the input for the program. --write-config + *Deprecated* + Use of `--write-config` has been deprecated, replaced by `--save-config` + +--save-config This flag specifies that volatility should write or overwrite a file called config.json in the current directory. The file will contain the necessary JSON configuration to recreate the environment that the @@ -105,11 +113,25 @@ Options other plugins, but there's no guarantee that plugins use the same configuration options. +--clear-cache + Clears out all short-term cached items. + +--cache-path + Change the default path used to store the cache. + +--offline + Do not search online for additional JSON files. + Run offline mode (defaults to false) and for + remote windows symbol tables, linux/mac banner repositories. + --single-location SINGLE_LOCATION This specifies a URL which will be downloaded if necessary, and built upon by the automagic and, since most plugins require a single memory image, can be considered the input for the program. +--stackers STACKERS + Creates the list of stackers to use based on the config option. + --single-swap-locations SINGLE_SWAP_LOCATIONS A comma-separated list of swap files to be considered as part of the memory image specified by the single-location or file parameters. diff --git a/doc/source/vol2to3.rst b/doc/source/vol2to3.rst index eb33b6618..e768df0c2 100644 --- a/doc/source/vol2to3.rst +++ b/doc/source/vol2to3.rst @@ -62,6 +62,10 @@ automagic processes are clearly defined and can be enabled or disabled as necess included a stacker automagic to emulate the most common feature of Volatility 2, automatically stacking address spaces (now translation layers) on top of each other. +By default the automagic chosen to be run are determined based on the plugin requested, so that linux plugins get linux +specific automagic and windows plugins get windows specific automagic. This should reduce unnecessarily searching for +linux kernels in a windows image, for example. At the moment this is not user configurableS. + Searching and Scanning ---------------------- Scanning is very similar to scanning in Volatility 2, a scanner object (such as a diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 1b6846e6b..5a4b21ade 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -22,15 +22,17 @@ be run. When volshell starts, it will show the version of volshell, a brief message indicating how to get more help, the current operating system mode for volshell, and the current layer available for use. -.. code-block:: python +:: - Volshell (Volatility 3 Framework) 1.0.1 + Volshell (Volatility 3 Framework) 2.0.2 Readline imported successfully PDB scanning finished Call help() to see available functions - Volshell mode: Generic - Current Layer: primary + Volshell mode : Generic + Current Layer : primary + Current Symbol Table : None + Current Kernel Name : None (primary) >>> @@ -53,11 +55,11 @@ run our examples against. We'll start by creating a process variable, and putting the first result from `ps()` in it. Since the shell is a python environment, we can do the following: -.. code-block:: python +:: - (primary) >>> proc = ps()[0] - (primary) >>> proc - + (layer_name) >>> proc = ps()[0] + (layer_name) >>> proc + When printing a volatility structure, various information is output, in this case the `type_name`, the `layer` and `offset` that it's been constructed on, and the size of the structure. @@ -68,72 +70,72 @@ built-in mechanism for providing more information about a structure, called `dis either a type name (which if not prefixed with symbol table name, will use the kernel symbol table identified by the automagic). -.. code-block:: python +:: - (primary) >>> dt('_EPROCESS') - nt_symbols1!_EPROCESS (2624 bytes) - 0x0 : Pcb nt_symbols1!_KPROCESS - 0x438 : ProcessLock nt_symbols1!_EX_PUSH_LOCK - 0x440 : UniqueProcessId nt_symbols1!pointer - 0x448 : ActiveProcessLinks nt_symbols1!_LIST_ENTRY + (layer_name) >>> dt('_EPROCESS') + symbol_table_name1!_EPROCESS (1968 bytes) + 0x0 : Pcb symbol_table_name1!_KPROCESS + 0x2d8 : ProcessLock symbol_table_name1!_EX_PUSH_LOCK + 0x2e0 : RundownProtect symbol_table_name1!_EX_RUNDOWN_REF + 0x2e8 : UniqueProcessId symbol_table_name1!pointer ... It can also be provided with an object and will interpret the data for each in the process: -.. code-block:: python +:: - (primary) >>> dt(proc) - nt_symbols1!_EPROCESS (2624 bytes) - 0x0 : Pcb nt_symbols1!_KPROCESS 0x8c0bccf8d040 - 0x438 : ProcessLock nt_symbols1!_EX_PUSH_LOCK 0x8c0bccf8d478 - 0x440 : UniqueProcessId nt_symbols1!pointer 356 - 0x448 : ActiveProcessLinks nt_symbols1!_LIST_ENTRY 0x8c0bccf8d488 + (layer_name) >>> dt(proc) + symbol_table_name1!_EPROCESS (1968 bytes) + 0x0 : Pcb symbol_table_name1!_KPROCESS 0xe08ff2459040 + 0x2d8 : ProcessLock symbol_table_name1!_EX_PUSH_LOCK 0xe08ff2459318 + 0x2e0 : RundownProtect symbol_table_name1!_EX_RUNDOWN_REF 0xe08ff2459320 + 0x2e8 : UniqueProcessId symbol_table_name1!pointer 4 ... These values can be accessed directory as attributes -.. code-block:: python +:: - (primary) >>> proc.UniqueProcessId + (layer_name) >>> proc.UniqueProcessId 356 Pointer structures contain the value they point to, but attributes accessed are forwarded to the object they point to. This means that pointers do not need to be explicitly dereferenced to access underling objects. -.. code-block:: python +:: - (primary) >>> proc.Pcb.DirectoryTableBase + (layer_name) >>> proc.Pcb.DirectoryTableBase 4355817472 Running plugins --------------- -It's possible to run any plugin by importing it appropriately and passing it to the `display_plugin_ouptut` or `dpo` +It's possible to run any plugin by importing it appropriately and passing it to the `display_plugin_output` or `dpo` method. In the following example we'll provide no additional parameters. Volatility will show us which parameters were required: -.. code-block:: python +:: - (primary) >>> from volatility3.plugins.windows import pslist - (primary) >>> display_plugin_output(pslist.PsList) - Unable to validate the plugin requirements: ['plugins.Volshell.9QZLXJKFWESI0BAP3M1U7Y5VCT468GRN.PsList.primary', 'plugins.Volshell.9QZLXJKFWESI0BAP3M1U7Y5VCT468GRN.PsList.nt_symbols'] + (layer_name) >>> from volatility3.plugins.windows import pslist + (layer_name) >>> display_plugin_output(pslist.PsList) + Unable to validate the plugin requirements: ['plugins.Volshell.VH3FSA1JBG0QP9E62Z8OT5UCIMLNYKW4.PsList.kernel'] -We can see that it's made a temporary configuration path for the plugin, and that neither `primary` nor `nt_symbols` -was fulfilled. +We can see that it's made a temporary configuration path for the plugin, and that the `kernel` requirement +was not fulfilled. We can see all the options that the plugin can accept by access the `get_requirements()` method of the plugin. This is a classmethod, so can be called on an uninstantiated copy of the plugin. -.. code-block:: python +:: - (primary) >>> pslist.PsList.get_requirements() - [, , , , ] + (layer_name) >>> pslist.PsList.get_requirements() + [, , , ] We can provide arguments via the `dpo` method call: -.. code-block:: python +:: - (primary) >>> display_plugin_output(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) + (layer_name) >>> display_plugin_output(pslist.PsList, kernel = self.config['kernel']) PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output @@ -142,17 +144,18 @@ We can provide arguments via the `dpo` method call: 356 4 smss.exe 0x8c0bccf8d040 3 - N/A False 2021-03-13 17:25:33.000000 N/A Disabled ... -Here's we've provided the current layer as the TranslationLayerRequirement, and used the symbol tables requirement -requested by the volshell plugin itself. A different table could be loaded and provided instead. The context used +Here's we've provided the kernel name that was requested by the volshell plugin itself (the generic volshell does not +load a kernel module, and instead only has a TranslationLayerRequirement). +A different module could be created and provided instead. The context used by the `dpo` method is always `context`. Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by using the `generate_treegrid` or `gt` command. -.. code-block:: python +:: - (primary) >>> treegrid = gt(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) - (primary) >>> treegrid.populate() + (layer_name) >>> treegrid = gt(pslist.PsList, kernel = self.config['kernel']) + (layer_name) >>> treegrid.populate() Treegrids must be populated before the data in them can be accessed. This is where the plugin actually runs and produces data. diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..3ff7c50b8 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,26 @@ +# The following packages are required for core functionality. +pefile>=2017.8.1 + +# The following packages are optional. +# If certain packages are not necessary, place a comment (#) at the start of the line. + +# This is required for the yara plugins +yara-python>=3.8.0 + +# This is required for several plugins that perform malware analysis and disassemble code. +# It can also improve accuracy of Windows 8 and later memory samples. +capstone>=3.0.5 + +# This is required by plugins that decrypt passwords, password hashes, etc. +pycryptodome + +# This can improve error messages regarding improperly configured ISF files, +# but is only recommended for development +# jsonschema>=2.3.0 + +# This is required for memory acquisition via leechcore/pcileech. +leechcorepyc>=2.4.0 + +# This is required for analyzing Linux samples compressed using AVMLs native +# compression format. It is not required for AVML's standard LiME compression. +python-snappy==0.6.0 diff --git a/requirements.txt b/requirements.txt index 290d9ca97..1793012f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,9 +14,6 @@ capstone>=3.0.5 # This is required by plugins that decrypt passwords, password hashes, etc. pycryptodome -# This can improve error messages regarding improperly configured ISF files. -jsonschema>=2.3.0 - # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 diff --git a/setup.py b/setup.py index f6bb687f2..a4bd3fffe 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setuptools.setup(name = "volatility3", '': ['development', 'development.*'], 'development': ['*'] }, - packages = setuptools.find_packages(exclude = ["development", "development.*"]), + packages = setuptools.find_namespace_packages(exclude = ["development", "development.*"]), entry_points = { 'console_scripts': [ 'vol = volatility3.cli:main', diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..dcbe289b0 --- /dev/null +++ b/test/README.md @@ -0,0 +1,34 @@ +# Volatility 3 Testing Framework + +## Requirements + +The Volatility 3 Testing Framework requires the same version of Python as Volatility3 itself. To install the current set of dependencies that the framework requires, use a command like this: + +```shell +pip3 install -r requirements-testing.txt +``` + +NOTE: `requirements-testing.txt` can be found in this current `test/` directory. + +## Quick Start: Manual Testing + +1. To test Volatility 3 on an image, first download one with a command such as: + +```shell +curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" +gunzip win-xp-laptop-2005-06-25.img.gz +``` + +2. In many cases, more symbols are required to be downloaded to the `./volatility3/symbols` directory. + +3. To manually run the tests, run a command, such as: + +```shell +py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows +``` + +The above command runs all available tests for windows on the `win-xp-laptop-2005-06-25.img` image. To choose a more specific set of tests, change the phrase after `-k` in this command. + +## Github Actions + +This framework currently tests two images (one linux image and one windows image) after every push on any branch. For more information/context, find the actions setup in `./github/workflows/test.yaml` \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 000000000..9057e1676 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,40 @@ +# This file is used to augment the test configuration + +import os +import pytest + +def pytest_addoption(parser): + parser.addoption("--volatility", action="store", default=None, + required=True, + help="path to the volatility script") + + parser.addoption("--python", action="store", default="python3", + help="The name of the interpreter to use when running the volatility script") + + parser.addoption("--image", action="append", default=[], + help="path to an image to test") + + parser.addoption("--image-dir", action="append", default=[], + help="path to a directory containing images to test") + +def pytest_generate_tests(metafunc): + """Parameterize tests based on image names""" + + images = metafunc.config.getoption('image') + for image_dir in metafunc.config.getoption('image_dir'): + images = images + [os.path.join(image_dir, dir) for dir in os.listdir(image_dir)] + + # tests with "image" parameter are run against images + if 'image' in metafunc.fixturenames: + metafunc.parametrize("image", + images, + ids=[os.path.basename(image) for image in images]) + +# Fixtures +@pytest.fixture +def volatility(request): + return request.config.getoption("--volatility") + +@pytest.fixture +def python(request): + return request.config.getoption("--python") diff --git a/test/known_files.json b/test/known_files.json new file mode 100644 index 000000000..a579c8053 --- /dev/null +++ b/test/known_files.json @@ -0,0 +1,19 @@ +{ + "windows_dumpfiles": { + "win-xp-laptop-2005-06-25.img": { + "0x82220e78": [ + "9bdd5532286f1660f3778e68bc36efe6", + "e3bc1e9e7370e3b5a661ebe591ecf4ec" + ], + "0x82350bf8": [ + "e5c5e8d97b6280745b41f6572c85d1f0", + "8589f1463422884dbf1411aaad278465" + ], + "0x81eaf418": [ + "f7a1ae2060a58f8470b97affdb46dccf", + "54fd611021fa784912530b8007545986" + ], + "0x820588e8": "458efbc8fdb859488a6ab2b200cce809" + } + } +} diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt new file mode 100644 index 000000000..7afe19b94 --- /dev/null +++ b/test/requirements-testing.txt @@ -0,0 +1,10 @@ +# These packages are required for core functionality. +pefile>=2017.8.1 #foo + +# The following packages are optional. +# If certain packages are not necessary, place a comment (#) at the start of the line. + +# This is required for the yara plugins +yara-python>=3.8.0 + +pytest>=7.0.0 diff --git a/test/test_volatility.py b/test/test_volatility.py new file mode 100644 index 000000000..515bef1cc --- /dev/null +++ b/test/test_volatility.py @@ -0,0 +1,384 @@ +# volatility3 tests +# + +# +# IMPORTS +# + +import os +import subprocess +import sys +import shutil +import tempfile +import hashlib +import ntpath +import json + +# +# HELPER FUNCTIONS +# + +def runvol(args, volatility, python): + volpy = volatility + python_cmd = python + + cmd = [python_cmd, volpy] + args + print(" ".join(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + print("stdout:") + sys.stdout.write(str(stdout)) + print("") + print("stderr:") + sys.stdout.write(str(stderr)) + print("") + + return p.returncode, stdout, stderr + +def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): + args = globalargs + [ + "--single-location", + img, + "-q", + plugin, + ] + pluginargs + + return runvol(args, volatility, python) + +# +# TESTS +# + +# WINDOWS + +def test_windows_pslist(image, volatility, python): + rc, out, err = runvol_plugin("windows.pslist.PsList", image, volatility, python) + out = out.lower() + assert out.find(b"system") != -1 + assert out.find(b"csrss.exe") != -1 + assert out.find(b"svchost.exe") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + + rc, out, err = runvol_plugin( + "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"]) + out = out.lower() + assert out.find(b"system") != -1 + assert out.count(b"\n") < 10 + assert rc == 0 + +def test_windows_psscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) + out = out.lower() + assert out.find(b"system") != -1 + assert out.find(b"csrss.exe") != -1 + assert out.find(b"svchost.exe") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_windows_dlllist(image, volatility, python): + rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) + out = out.lower() + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_windows_modules(image, volatility, python): + rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) + out = out.lower() + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_windows_hivelist(image, volatility, python): + rc, out, err = runvol_plugin("windows.registry.hivelist.HiveList", image, volatility, python) + out = out.lower() + + not_xp = out.find(b"\\systemroot\\system32\\config\\software") + if not_xp == -1: + assert out.find(b"\\device\\harddiskvolume1\\windows\\system32\\config\\software") != -1 + + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_windows_dumpfiles(image, volatility, python): + + json_file = open('./test/known_files.json') + + known_files = json.load(json_file) + + failed_chksms = 0 + + if sys.platform == 'win32': + file_name = ntpath.basename(image) + else: + file_name = os.path.basename(image) + + try: + for addr in known_files["windows_dumpfiles"][file_name]: + + path = tempfile.mkdtemp() + + rc, out, err = runvol_plugin("windows.dumpfiles.DumpFiles", image, volatility, python, globalargs=["-o", path], pluginargs=["--virtaddr", addr]) + + for file in os.listdir(path): + with open(os.path.join(path, file), "rb") as fp: + if hashlib.md5(fp.read()).hexdigest() not in known_files["windows_dumpfiles"][file_name][addr]: + failed_chksms += 1 + + shutil.rmtree(path) + + json_file.close() + + assert failed_chksms == 0 + assert rc == 0 + except Exception as e: + json_file.close() + print("Key Error raised on " + str(e)) + assert False + +def test_windows_handles(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"System Pid 4") != -1 + assert out.find(b"MACHINE\\SYSTEM\\CONTROLSET001\\CONTROL\\SESSION MANAGER\\MEMORY MANAGEMENT\\PREFETCHPARAMETERS") != -1 + assert out.find(b"MACHINE\\SYSTEM\\SETUP") != -1 + assert out.count(b"\n") > 500 + assert rc == 0 + +def test_windows_svcscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) + + assert out.find(b"Microsoft ACPI Driver") != -1 + assert out.count(b"\n") > 250 + assert rc == 0 + +def test_windows_privileges(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"SeCreateTokenPrivilege") != -1 + assert out.find(b"SeCreateGlobalPrivilege") != -1 + assert out.find(b"SeAssignPrimaryTokenPrivilege") != -1 + assert out.count(b"\n") > 20 + assert rc == 0 + +def test_windows_getsids(image, volatility, python): + rc, out, err = runvol_plugin( + "windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"]) + + assert out.find(b"Local System") != -1 + assert out.find(b"Administrators") != -1 + assert out.find(b"Everyone") != -1 + assert out.find(b"Authenticated Users") != -1 + assert rc == 0 + +def test_windows_envars(image, volatility, python): + rc, out, err = runvol_plugin("windows.envars.Envars", image, volatility, python) + + assert out.find(b"PATH") != -1 + assert out.find(b"PROCESSOR_ARCHITECTURE") != -1 + assert out.find(b"USERNAME") != -1 + assert out.find(b"SystemRoot") != -1 + assert out.find(b"CommonProgramFiles") != -1 + assert out.count(b"\n") > 500 + assert rc == 0 + +def test_windows_callbacks(image, volatility, python): + rc, out, err = runvol_plugin("windows.callbacks.Callbacks", image, volatility, python) + + assert out.find(b"PspCreateProcessNotifyRoutine") != -1 + assert out.find(b"KeBugCheckCallbackListHead") != -1 + assert out.find(b"KeBugCheckReasonCallbackListHead") != -1 + assert out.count(b"KeBugCheckReasonCallbackListHead ") > 5 + assert rc == 0 + +def test_windows_devicetree(image, volatility, python): + rc, out, err = runvol_plugin("windows.devicetree.DeviceTree", image, volatility, python) + + assert out.find(b"DEV") != -1 + assert out.find(b"DRV") != -1 + assert out.find(b"ATT") != -1 + assert out.find(b"FILE_DEVICE_CONTROLLER") != -1 + assert out.find(b"FILE_DEVICE_DISK") != -1 + assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1 + assert rc == 0 + +# LINUX + +def test_linux_pslist(image, volatility, python): + rc, out, err = runvol_plugin("linux.pslist.PsList", image, volatility, python) + out = out.lower() + + assert ((out.find(b"init") != -1) or (out.find(b"systemd") != -1)) + assert out.find(b"watchdog") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_check_idt(image, volatility, python): + rc, out, err = runvol_plugin("linux.check_idt.Check_idt", image, volatility, python) + out = out.lower() + + assert out.count(b"__kernel__") >= 10 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_check_syscall(image, volatility, python): + rc, out, err = runvol_plugin("linux.check_syscall.Check_syscall", image, volatility, python) + out = out.lower() + + assert out.find(b"sys_close") != -1 + assert out.find(b"sys_open") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_linux_lsmod(image, volatility, python): + rc, out, err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_linux_lsof(image, volatility, python): + rc, out, err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) + out = out.lower() + + assert out.count(b"socket:") >= 10 + assert out.count(b"\n") > 35 + assert rc == 0 + +def test_linux_proc_maps(image, volatility, python): + rc, out, err = runvol_plugin("linux.proc.Maps", image, volatility, python) + out = out.lower() + + assert out.count(b"anonymous mapping") >= 10 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_linux_tty_check(image, volatility, python): + rc, out, err = runvol_plugin("linux.tty_check.tty_check", image, volatility, python) + out = out.lower() + + assert out.find(b"__kernel__") != -1 + assert out.count(b"\n") >= 5 + assert rc == 0 + +# MAC + +def test_mac_pslist(image, volatility, python): + rc, out, err = runvol_plugin("mac.pslist.PsList", image, volatility, python) + out = out.lower() + + assert ((out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1)) + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_check_syscall(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_syscall.Check_syscall", image, volatility, python) + out = out.lower() + + assert out.find(b"chmod") != -1 + assert out.find(b"chown") != -1 + assert out.find(b"nosys") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_mac_check_sysctl(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_sysctl.Check_sysctl", image, volatility, python) + out = out.lower() + + assert out.find(b"__kernel__") != -1 + assert out.count(b"\n") > 250 + assert rc == 0 + +def test_mac_check_trap_table(image, volatility, python): + rc, out, err = runvol_plugin("mac.check_trap_table.Check_trap_table", image, volatility, python) + out = out.lower() + + assert out.count(b"kern_invalid") >= 10 + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_ifconfig(image, volatility, python): + rc, out, err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) + out = out.lower() + + assert out.find(b"127.0.0.1") != -1 + assert out.find(b"false") != -1 + assert out.count(b"\n") > 9 + assert rc == 0 + +def test_mac_lsmod(image, volatility, python): + rc, out, err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) + out = out.lower() + + assert out.find(b"com.apple") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_lsof(image, volatility, python): + rc, out, err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_malfind(image, volatility, python): + rc, out, err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 20 + assert rc == 0 + +def test_mac_mount(image, volatility, python): + rc, out, err = runvol_plugin("mac.mount.Mount", image, volatility, python) + out = out.lower() + + assert out.find(b"/dev") != -1 + assert out.count(b"\n") > 7 + assert rc == 0 + +def test_mac_netstat(image, volatility, python): + rc, out, err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) + + assert out.find(b"TCP") != -1 + assert out.find(b"UDP") != -1 + assert out.find(b"UNIX") != -1 + assert out.count(b"\n") > 10 + assert rc == 0 + +def test_mac_proc_maps(image, volatility, python): + rc, out, err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) + out = out.lower() + + assert out.find(b"[heap]") != -1 + assert out.count(b"\n") > 100 + assert rc == 0 + +def test_mac_psaux(image, volatility, python): + rc, out, err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) + out = out.lower() + + assert out.find(b"executable_path") != -1 + assert out.count(b"\n") > 50 + assert rc == 0 + +def test_mac_socket_filters(image, volatility, python): + rc, out, err = runvol_plugin("mac.socket_filters.Socket_filters", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 9 + assert rc == 0 + +def test_mac_timers(image, volatility, python): + rc, out, err = runvol_plugin("mac.timers.Timers", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 6 + assert rc == 0 + +def test_mac_trustedbsd(image, volatility, python): + rc, out, err = runvol_plugin("mac.trustedbsd.Trustedbsd", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 diff --git a/vol.spec b/vol.spec index 42b69af3f..666526dde 100644 --- a/vol.spec +++ b/vol.spec @@ -26,7 +26,7 @@ except ImportError: # Volatility must be findable in sys.path in order for collect_submodules to work # This adds the current working directory, which should usually do the trick -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.abspath(SPEC))) vol_analysis = Analysis(['vol.py'], pathex = [], diff --git a/volatility3/__init__.py b/volatility3/__init__.py index db52aa9b0..b6da6e01e 100644 --- a/volatility3/__init__.py +++ b/volatility3/__init__.py @@ -37,9 +37,9 @@ class WarningFindSpec(abc.MetaPathFinder): first.""" if fullname.startswith("volatility3.framework.plugins."): warning = "Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins" - # Pyinstaller uses walk_packages to import, but needs to read the modules to figure out dependencies - # As such, we only print the warning when directly imported rather than from within walk_packages - if inspect.stack()[-2].function != 'walk_packages': + # Pyinstaller uses walk_packages/_collect_submodules to import, but needs to read the modules to figure out dependencies + # As such, we only print the warning when directly imported rather than from within walk_packages/_collect_submodules + if inspect.stack()[-2].function in ['walk_packages', '_collect_submodules']: raise Warning(warning) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 608fdf79c..8851e2b18 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,14 +19,14 @@ import os import sys import tempfile import traceback -from typing import Dict, Type, Union, Any +from typing import Any, Dict, Type, Union from urllib import parse, request import volatility3.plugins import volatility3.symbols from volatility3 import framework from volatility3.cli import text_renderer, volargparse -from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins, configuration +from volatility3.framework import automagic, configuration, constants, contexts, exceptions, interfaces, plugins from volatility3.framework.automagic import stacker from volatility3.framework.configuration import requirements @@ -157,6 +157,10 @@ class CommandLine: help = "Write configuration JSON file out to config.json", default = False, action = 'store_true') + parser.add_argument("--save-config", + help = "Save configuration JSON file to a file", + default = None, + type = str) parser.add_argument("--clear-cache", help = "Clears out all short-term cached items", default = False, @@ -320,9 +324,15 @@ class CommandLine: self.file_handler_class_factory()) if args.write_config: - vollog.debug("Writing out configuration data to config.json") - with open("config.json", "w") as f: + vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') + args.save_config = 'config.json' + if args.save_config: + vollog.debug("Writing out configuration data to {args.save_config}") + if os.path.exists(os.path.abspath(args.save_config)): + parser.error(f"Cannot write configuration: file {args.save_config} already exists") + with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + f.write("\n") except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") @@ -414,7 +424,7 @@ class CommandLine: detail = f"{excp}" caused_by = ["A required python module is not installed (install the module and re-run)"] else: - general = "Volatilty encountered an unexpected situation." + general = "Volatility encountered an unexpected situation." detail = "" caused_by = [ "Please re-run using with -vvv and file a bug with the output", f"at {constants.BUG_URL}" @@ -443,16 +453,17 @@ class CommandLine: print(f"Unsatisfied requirement {config_path}: {excp.unsatisfied[config_path].description}") - if symbols_failed: - print("\nA symbol table requirement was not fulfilled. Please verify that:\n" - "\tYou have the correct symbol file for the requirement\n" - "\tThe symbol file is under the correct directory or zip file\n" - "\tThe symbol file is named appropriately or contains the correct banner\n") if translation_failed: print("\nA translation layer requirement was not fulfilled. Please verify that:\n" "\tA file was provided to create this layer (by -f, --single-location or by config)\n" "\tThe file exists and is readable\n" - "\tThe necessary symbols are present and identified by volatility3") + "\tThe file is a valid memory image and was acquired cleanly") + if symbols_failed: + print("\nA symbol table requirement was not fulfilled. Please verify that:\n" + "\tThe associated translation layer requirement was fulfilled\n" + "\tYou have the correct symbol file for the requirement\n" + "\tThe symbol file is under the correct directory or zip file\n" + "\tThe symbol file is named appropriately or contains the correct banner\n") def populate_config(self, context: interfaces.context.ContextInterface, configurables_list: Dict[str, Type[interfaces.configuration.ConfigurableInterface]], @@ -543,7 +554,7 @@ class CommandLine: self._file = io.open(fd, mode = 'w+b') CLIFileHandler.__init__(self, filename) for item in dir(self._file): - if not item.startswith('_') and not item in ['closed', 'close', 'mode', 'name']: + if not item.startswith('_') and item not in ('closed', 'close', 'mode', 'name'): setattr(self, item, getattr(self._file, item)) def __getattr__(self, item): diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 35b2468e8..623153fae 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -1,6 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import csv import datetime import json import logging @@ -8,7 +9,7 @@ import random import string import sys from functools import wraps -from typing import Callable, Any, List, Tuple, Dict +from typing import Any, Callable, Dict, List, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.renderers import format_hints @@ -66,7 +67,6 @@ def multitypedata_as_text(value: format_hints.MultiTypeData) -> str: def optional(func: Callable) -> Callable: - @wraps(func) def wrapped(x: Any) -> str: if isinstance(x, interfaces.renderers.BaseAbsentValue): @@ -80,7 +80,6 @@ def optional(func: Callable) -> Callable: def quoted_optional(func: Callable) -> Callable: - @wraps(func) def wrapped(x: Any) -> str: result = optional(func)(x) @@ -102,7 +101,7 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str: disasm: Input disassembly objects Returns: - A string as rendererd by capstone where available, otherwise output as if it were just bytes + A string as rendered by capstone where available, otherwise output as if it were just bytes """ if CAPSTONE_PRESENT: @@ -182,16 +181,28 @@ class QuickTextRenderer(CLIRenderer): outfd.write("\n") +class NoneRenderer(CLIRenderer): + """Outputs no results""" + name = "none" + + def get_render_options(self): + pass + + def render(self, grid: interfaces.renderers.TreeGrid) -> None: + if not grid.populated: + grid.populate(lambda x, y: True, True) + + class CSVRenderer(CLIRenderer): _type_renderers = { - format_hints.Bin: quoted_optional(lambda x: f"0b{x:b}"), - format_hints.Hex: quoted_optional(lambda x: f"0x{x:x}"), - format_hints.HexBytes: quoted_optional(hex_bytes_as_text), - format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), - interfaces.renderers.Disassembly: quoted_optional(display_disassembly), - bytes: quoted_optional(lambda x: " ".join([f"{b:02x}" for b in x])), - datetime.datetime: quoted_optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), - 'default': quoted_optional(lambda x: f"{x}") + format_hints.Bin: optional(lambda x: f"0b{x:b}"), + format_hints.Hex: optional(lambda x: f"0x{x:x}"), + format_hints.HexBytes: optional(hex_bytes_as_text), + format_hints.MultiTypeData: optional(multitypedata_as_text), + interfaces.renderers.Disassembly: optional(display_disassembly), + bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), + 'default': optional(lambda x: f"{x}") } name = "csv" @@ -208,28 +219,28 @@ class CSVRenderer(CLIRenderer): """ outfd = sys.stdout - line = ['"TreeDepth"'] + header_list = ['TreeDepth'] for column in grid.columns: # Ignore the type because namedtuples don't realize they have accessible attributes - line.append("{}".format('"' + column.name + '"')) - outfd.write(f"{','.join(line)}") + header_list.append(f"{column.name}") + + writer = csv.DictWriter(outfd, header_list, lineterminator='\n') + writer.writeheader() def visitor(node: interfaces.renderers.TreeNode, accumulator): - accumulator.write("\n") # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case - accumulator.write(str(max(0, node.path_depth - 1)) + ",") - line = [] + row = {'TreeDepth': str(max(0, node.path_depth - 1))} for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get(column.type, self._type_renderers['default']) - line.append(renderer(node.values[column_index])) - accumulator.write(f"{','.join(line)}") + row[f'{column.name}'] = renderer(node.values[column_index]) + accumulator.writerow(row) return accumulator if not grid.populated: - grid.populate(visitor, outfd) + grid.populate(visitor, writer) else: - grid.visit(node = None, function = visitor, initial_accumulator = outfd) + grid.visit(node = None, function = visitor, initial_accumulator = writer) outfd.write("\n") @@ -263,7 +274,8 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths = dict([(column.name, len(column.name)) for column in grid.columns]) def visitor( - node: interfaces.renderers.TreeNode, accumulator: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]] + node: interfaces.renderers.TreeNode, + accumulator: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]] ) -> List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]]: # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case max_column_widths[tree_indent_column] = max(max_column_widths.get(tree_indent_column, 0), node.path_depth) @@ -272,9 +284,10 @@ class PrettyTextRenderer(CLIRenderer): column = grid.columns[column_index] renderer = self._type_renderers.get(column.type, self._type_renderers['default']) data = renderer(node.values[column_index]) + field_width = max([len(self.tab_stop(x)) for x in f"{data}".split("\n")]) max_column_widths[column.name] = max(max_column_widths.get(column.name, len(column.name)), - len(f"{data}")) - line[column] = data + field_width) + line[column] = data.split("\n") accumulator.append((node.path_depth, line)) return accumulator @@ -296,7 +309,22 @@ class PrettyTextRenderer(CLIRenderer): column_titles = [""] + [column.name for column in grid.columns] outfd.write(format_string.format(*column_titles)) for (depth, line) in final_output: - outfd.write(format_string.format("*" * depth, *[line[column] for column in grid.columns])) + nums_line = max([len(line[column]) for column in line]) + for column in line: + line[column] = line[column] + ([""] * (nums_line - len(line[column]))) + for index in range(nums_line): + if index == 0: + outfd.write(format_string.format("*" * depth, *[self.tab_stop(line[column][index]) for column in grid.columns])) + else: + outfd.write(format_string.format(" " * depth, *[self.tab_stop(line[column][index]) for column in grid.columns])) + + def tab_stop(self, line: str) -> str: + tab_width = 8 + while line.find('\t') >= 0: + i = line.find('\t') + pad = " " * (tab_width - (i % tab_width)) + line = line.replace("\t", pad, 1) + return line class JsonRenderer(CLIRenderer): @@ -317,7 +345,7 @@ class JsonRenderer(CLIRenderer): def output_result(self, outfd, result): """Outputs the JSON data to a file in a particular format""" - outfd.write(json.dumps(result, indent = 2, sort_keys = True)) + outfd.write("{}\n".format(json.dumps(result, indent = 2, sort_keys = True))) def render(self, grid: interfaces.renderers.TreeGrid): outfd = sys.stdout diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index f2375c774..5eeef77cf 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -11,7 +11,7 @@ import sys import volatility3.plugins import volatility3.symbols from volatility3 import cli, framework -from volatility3.cli.volshell import generic, windows, linux, mac +from volatility3.cli.volshell import generic, linux, mac, windows from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins # Make sure we log everything @@ -85,6 +85,10 @@ class VolShell(cli.CommandLine): help = "Write configuration JSON file out to config.json", default = False, action = 'store_true') + parser.add_argument("--save-config", + help = "Save configuration JSON file to a file", + default = None, + type = str) parser.add_argument("--clear-cache", help = "Clears out all short-term cached items", default = False, @@ -137,8 +141,7 @@ class VolShell(cli.CommandLine): console.setLevel(10 - (partial_args.verbosity - 2)) if partial_args.clear_cache: - for cache_filename in glob.glob(os.path.join(constants.CACHE_PATH, '*.cache')): - os.unlink(cache_filename) + framework.clear_cache() # Do the initialization ctx = contexts.Context() # Construct a blank context @@ -235,15 +238,25 @@ class VolShell(cli.CommandLine): self.file_handler_class_factory()) if args.write_config: - vollog.debug("Writing out configuration data to config.json") - with open("config.json", "w") as f: + vollog.warning('Use of --write-config has been deprecated, replaced by --save-config ') + args.save_config = 'config.json' + if args.save_config: + vollog.debug("Writing out configuration data to {args.save_config}") + if os.path.exists(os.path.abspath(args.save_config)): + parser.error(f"Cannot write configuration: file {args.save_config} already exists") + with open(args.save_config, "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) + f.write("\n") + except exceptions.UnsatisfiedException as excp: + self.process_unsatisfied_exceptions(excp) + parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") + try: # Construct and run the plugin - constructed.run() + if constructed: + constructed.run() except exceptions.VolatilityException as excp: self.process_exceptions(excp) - parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") def main(): diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 8f81a0420..19e263a03 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -8,11 +8,11 @@ import random import string import struct import sys -from typing import Any, Dict, List, Optional, Tuple, Union, Type, Iterable -from urllib import request, parse +from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union +from urllib import parse, request from volatility3.cli import text_renderer, volshell -from volatility3.framework import renderers, interfaces, objects, plugins, exceptions +from volatility3.framework import exceptions, interfaces, objects, plugins, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, physical, resources @@ -31,6 +31,8 @@ class Volshell(interfaces.plugins.PluginInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__current_layer: Optional[str] = None + self.__current_symbol_table: Optional[str] = None + self.__current_kernel_name: Optional[str] = None self.__console = None def random_string(self, length: int = 32) -> str: @@ -57,8 +59,6 @@ class Volshell(interfaces.plugins.PluginInterface): Return a TreeGrid but this is always empty since the point of this plugin is to run interactively """ - self.__current_layer = self.config['primary'] - # Try to enable tab completion try: import readline @@ -79,9 +79,11 @@ class Volshell(interfaces.plugins.PluginInterface): banner = f""" Call help() to see available functions - Volshell mode: {mode} - Current Layer: {self.current_layer} - """ + Volshell mode : {mode} + Current Layer : {self.current_layer} + Current Symbol Table : {self.current_symbol_table} + Current Kernel Name : {self.current_kernel_name} +""" sys.ps1 = f"({self.current_layer}) >>> " self.__console = code.InteractiveConsole(locals = self._construct_locals_dict()) @@ -121,7 +123,10 @@ class Volshell(interfaces.plugins.PluginInterface): (['dw', 'display_words'], self.display_words), (['dd', 'display_doublewords'], self.display_doublewords), (['dq', 'display_quadwords'], self.display_quadwords), (['dis', 'disassemble'], self.disassemble), - (['cl', 'change_layer'], self.change_layer), (['context'], self.context), (['self'], self), + (['cl', 'change_layer'], self.change_layer), + (['cs', 'change_symboltable'], self.change_symbol_table), + (['ck', 'change_kernel'], self.change_kernel), + (['context'], self.context), (['self'], self), (['dpo', 'display_plugin_output'], self.display_plugin_output), (['gt', 'generate_treegrid'], self.generate_treegrid), (['rt', 'render_treegrid'], self.render_treegrid), @@ -174,15 +179,58 @@ class Volshell(interfaces.plugins.PluginInterface): @property def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.config['primary'] return self.__current_layer - def change_layer(self, layer_name = None): + @property + def current_symbol_table(self): + if self.__current_symbol_table is None and self.kernel: + self.__current_symbol_table = self.kernel.symbol_table_name + return self.__current_symbol_table + + @property + def current_kernel_name(self): + if self.__current_kernel_name is None: + self.__current_kernel_name = self.config.get('kernel', None) + return self.__current_kernel_name + + @property + def kernel(self): + """Returns the current kernel object""" + if self.current_kernel_name not in self.context.modules: + return None + return self.context.modules[self.current_kernel_name] + + def change_layer(self, layer_name: str = None): """Changes the current default layer""" if not layer_name: - layer_name = self.config['primary'] - self.__current_layer = layer_name + layer_name = self.current_layer + if layer_name not in self.context.layers: + print(f"Layer {layer_name} not present in context") + else: + self.__current_layer = layer_name sys.ps1 = f"({self.current_layer}) >>> " + def change_symbol_table(self, symbol_table_name: str = None): + """Changes the current_symbol_table""" + if not symbol_table_name: + print("No symbol table provided, not changing current symbol table") + if symbol_table_name not in self.context.symbol_space: + print(f"Symbol table {symbol_table_name} not present in context symbol_space") + else: + self.__current_symbol_table = symbol_table_name + print(f"Current Symbol Table: {self.current_symbol_table}") + + def change_kernel(self, kernel_name: str = None): + if not kernel_name: + print("No kernel module name provided, not changing current kernel") + if kernel_name not in self.context.modules: + print(f"Kernel module {kernel_name} not found in the context module list") + else: + self.__current_kernel_name = kernel_name + print(f"Current kernel : {self.current_kernel_name}") + def display_bytes(self, offset, count = 128, layer_name = None): """Displays byte values and ASCII characters""" remaining_data = self._read_data(offset, count = count, layer_name = layer_name) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 4338ae06f..0f2a90c7e 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -5,7 +5,7 @@ from typing import Any, List, Tuple, Union from volatility3.cli.volshell import generic -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.plugins.linux import pslist @@ -15,8 +15,8 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return (super().get_requirements() + [ - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), + return ([ + requirements.ModuleRequirement(name = "kernel", description = "Linux kernel module"), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) @@ -37,14 +37,14 @@ class Volshell(generic.Volshell): def list_tasks(self): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_tasks(self.context, self.config['primary'], self.config['vmlinux'])) + return list(pslist.PsList.list_tasks(self.context, self.current_kernel_name)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (['ct', 'change_task', 'cp'], self.change_task), (['lt', 'list_tasks', 'ps'], self.list_tasks), - (['symbols'], self.context.symbol_space[self.config['vmlinux']]), + (['symbols'], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get('pid', None) is not None: self.change_task(self.config['pid']) @@ -56,11 +56,17 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['vmlinux'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: - symbol_table = self.config['vmlinux'] + symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) + + @property + def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.kernel.layer_name + return self.__current_layer diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 8218848ba..6744f3394 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -15,9 +15,9 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return (super().get_requirements() + [ - requirements.SymbolTableRequirement(name = "darwin", description = "Darwin kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + return ([ + requirements.ModuleRequirement(name = "kernel", description = "Darwin kernel module"), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) @@ -34,17 +34,17 @@ class Volshell(generic.Volshell): return print(f"No task with task ID {pid} found") - def list_tasks(self): + def list_tasks(self, method = None): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_tasks(self.context, self.config['primary'], self.config['darwin'])) + return list(pslist.PsList.get_list_tasks(method)(self.context, self.current_kernel_name)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (['ct', 'change_task', 'cp'], self.change_task), (['lt', 'list_tasks', 'ps'], self.list_tasks), - (['symbols'], self.context.symbol_space[self.config['darwin']]), + (['symbols'], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get('pid', None) is not None: self.change_task(self.config['pid']) @@ -56,11 +56,17 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['darwin'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: - symbol_table = self.config['darwin'] + symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) + + @property + def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.kernel.layer_name + return self.__current_layer diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 6c191ad28..2cc5d3e1d 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -5,7 +5,7 @@ from typing import Any, List, Tuple, Union from volatility3.cli.volshell import generic -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements from volatility3.plugins.windows import pslist @@ -15,8 +15,8 @@ class Volshell(generic.Volshell): @classmethod def get_requirements(cls): - return (super().get_requirements() + [ - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + return ([ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel'), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) @@ -34,14 +34,14 @@ class Volshell(generic.Volshell): def list_processes(self): """Returns a list of EPROCESS objects from the primary layer""" # We always use the main kernel memory and associated symbols - return list(pslist.PsList.list_processes(self.context, self.config['primary'], self.config['nt_symbols'])) + return list(pslist.PsList.list_processes(self.context, self.current_layer, self.current_symbol_table)) def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (['cp', 'change_process'], self.change_process), (['lp', 'list_processes', 'ps'], self.list_processes), - (['symbols'], self.context.symbol_space[self.config['nt_symbols']]), + (['symbols'], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get('pid', None) is not None: self.change_process(self.config['pid']) @@ -53,11 +53,17 @@ class Volshell(generic.Volshell): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): if constants.BANG not in object: - object = self.config['nt_symbols'] + constants.BANG + object + object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: - symbol_table = self.config['nt_symbols'] + symbol_table = self.current_symbol_table return super().display_symbols(symbol_table) + + @property + def current_layer(self): + if self.__current_layer is None: + self.__current_layer = self.kernel.layer_name + return self.__current_layer diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 2c9444130..176eb2242 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -51,7 +51,7 @@ def require_interface_version(*args) -> None: if args[1] > interface_version()[1]: raise RuntimeError( "Framework interface version {} is an older revision than the required version {}".format( - ".".join([str(x) for x in interface_version()[0:1]]), ".".join([str(x) for x in args[0:2]]))) + ".".join([str(x) for x in interface_version()[0:2]]), ".".join([str(x) for x in args[0:2]]))) class NonInheritable(object): diff --git a/volatility3/framework/automagic/__init__.py b/volatility3/framework/automagic/__init__.py index e4d422c99..7567f206d 100644 --- a/volatility3/framework/automagic/__init__.py +++ b/volatility3/framework/automagic/__init__.py @@ -21,14 +21,6 @@ from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) -windows_automagic = [ - 'ConstructionMagic', 'LayerStacker', 'KernelPDBScanner', 'WinSwapLayers', 'KernelModule' -] - -linux_automagic = ['ConstructionMagic', 'LayerStacker', 'LinuxBannerCache', 'LinuxSymbolFinder', 'KernelModule'] - -mac_automagic = ['ConstructionMagic', 'LayerStacker', 'MacBannerCache', 'MacSymbolFinder', 'KernelModule'] - def available(context: interfaces.context.ContextInterface) -> List[interfaces.automagic.AutomagicInterface]: """Returns an ordered list of all subclasses of @@ -58,10 +50,7 @@ def choose_automagic( plugin_category = "None" plugin_categories = plugin.__module__.split('.') lowest_index = len(plugin_categories) - - automagic_categories = {'windows': windows_automagic, 'linux': linux_automagic, 'mac': mac_automagic} - - for os in automagic_categories: + for os in constants.OS_CATEGORIES: try: if plugin_categories.index(os) < lowest_index: lowest_index = plugin_categories.index(os) @@ -70,14 +59,16 @@ def choose_automagic( # The value wasn't found, try the next one pass - if plugin_category not in automagic_categories: + if plugin_category not in constants.OS_CATEGORIES: vollog.info("No plugin category detected") return automagics - vollog.info(f"Detected a {plugin_category} category plugin") + output = [] for amagic in automagics: - if amagic.__class__.__name__ in automagic_categories[plugin_category]: + if plugin_category not in amagic.exclusion_list: + # Only include uncategorized automagic, or platform specific automagic + # (This allows user defined/uncategorized automagic to be included) output += [amagic] return output diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f9fa22c07..9bb2dae9b 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,10 +3,12 @@ # import logging +import os from typing import Optional, Tuple, Type -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder +from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import linux @@ -23,6 +25,13 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify linux within this layer.""" + # Version check the SQlite cache + required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + vollog.info( + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + return None + # Bail out by default unless we can stack properly layer = context.layers[layer_name] join = interfaces.configuration.path_join @@ -32,7 +41,9 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - linux_banners = LinuxBannerCache.load_banners() + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + linux_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( + operating_system = 'linux') # If we have no banners, don't bother scanning if not linux_banners: vollog.info("No Linux banners found - if this is a linux plugin, please check your symbol files location") @@ -43,9 +54,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = linux_banners.get(banner, None) - if symbol_files: - isf_path = symbol_files[0] + isf_path = linux_banners.get(banner, None) + if isf_path: table_name = context.symbol_space.free_table_name('LintelStacker') table = linux.LinuxKernelIntermedSymbols(context, 'temporary.' + table_name, @@ -141,18 +151,11 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): return addr - 0xc0000000 -class LinuxBannerCache(symbol_cache.SymbolBannerCache): - """Caches the banners found in the Linux symbol files.""" - - os = "linux" - symbol_name = "linux_banner" - banner_path = constants.LINUX_BANNERS_PATH - - class LinuxSymbolFinder(symbol_finder.SymbolFinder): """Linux symbol loader based on uname signature strings.""" banner_config_key = "kernel_banner" - banner_cache = LinuxBannerCache + operating_system = 'linux' symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] + exclusion_list = ['mac', 'windows'] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index fb725a234..9bb3ad5f0 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -3,11 +3,13 @@ # import logging +import os import struct from typing import Optional -from volatility3.framework import interfaces, constants, layers, exceptions +from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.automagic import symbol_cache, symbol_finder +from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import mac @@ -24,6 +26,13 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify mac within this layer.""" + # Version check the SQlite cache + required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required(required, symbol_cache.SqliteCache.version): + vollog.info( + f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}") + return None + # Bail out by default unless we can stack properly layer = context.layers[layer_name] new_layer = None @@ -34,7 +43,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - mac_banners = MacBannerCache.load_banners() + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + mac_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( + operating_system = 'mac') # If we have no banners, don't bother scanning if not mac_banners: vollog.info("No Mac banners found - if this is a mac plugin, please check your symbol files location") @@ -46,9 +57,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): dtb = None vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = mac_banners.get(banner, None) - if symbol_files: - isf_path = symbol_files[0] + isf_path = mac_banners.get(banner, None) + if isf_path: table_name = context.symbol_space.free_table_name('MacintelStacker') table = mac.MacKernelIntermedSymbols(context = context, config_path = join('temporary', table_name), @@ -197,17 +207,11 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): yield offset, banner -class MacBannerCache(symbol_cache.SymbolBannerCache): - """Caches the banners found in the Mac symbol files.""" - os = "mac" - symbol_name = "version" - banner_path = constants.MAC_BANNERS_PATH - - class MacSymbolFinder(symbol_finder.SymbolFinder): """Mac symbol loader based on uname signature strings.""" banner_config_key = 'kernel_banner' - banner_cache = MacBannerCache + operating_system = 'mac' find_aslr = MacIntelStacker.find_aslr symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols" + exclusion_list = ['windows', 'linux'] diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index 3d2bb584a..6810a58e2 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from volatility3.framework import interfaces, constants, configuration diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 93d4337da..5cbdbfe0e 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -7,10 +7,11 @@ from loaded PE files. This module contains a standalone scanner, and also a :class:`~volatility3.framework.interfaces.layers.ScannerInterface` based scanner for use within the framework by calling :func:`~volatility3.framework.interfaces.layers.DataLayerInterface.scan`. """ +import contextlib import logging import math import os -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union, Callable +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.configuration import requirements @@ -44,6 +45,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): """ priority = 30 max_pdb_size = 0x400000 + exclusion_list = ['linux', 'mac'] def find_virtual_layers_from_req(self, context: interfaces.context.ContextInterface, config_path: str, requirement: interfaces.configuration.RequirementInterface) -> List[str]: @@ -138,22 +140,29 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]: + def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ + ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): # Rule out kernels that couldn't find a suitable MZ header return None return (virtual_layer_name, kernel['mz_offset'], kernel) + vollog.debug("Kernel base determination - optimized scan virtual layer") + valid_kernel = self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, True, False, progress_callback) + if valid_kernel is not None: + return valid_kernel + vollog.debug("Kernel base determination - slow scan virtual layer") - return self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, False, progress_callback) + return self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, False, False, progress_callback) def method_fixed_mapping(self, context: interfaces.context.ContextInterface, vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - def test_physical_kernel(physical_layer_name:str , virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]: + def test_physical_kernel(physical_layer_name: str, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ + ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): # Rule out kernels that couldn't find a suitable MZ header @@ -174,12 +183,13 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vollog.debug(f"Potential kernel_virtual_offset caused a page fault: {hex(kvo)}") vollog.debug("Kernel base determination - testing fixed base address") - return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, True, progress_callback) + return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, False, True, progress_callback) def _method_layer_pdb_scan(self, context: interfaces.context.ContextInterface, vlayer: layers.intel.Intel, test_kernel: Callable, + optimized: bool = False, physical: bool = True, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: # TODO: Verify this is a windows image @@ -191,9 +201,15 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): if not physical: layer_to_scan = virtual_layer_name + start_scan_address = 0 + if optimized and not physical and context.layers[layer_to_scan].metadata.architecture in ["Intel64"]: + # TODO: change this value accordingly when 5-Level paging is supported. + start_scan_address = (0x1f0 << 39) + kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] kernels = PDBUtility.pdbname_scan(ctx = context, layer_name = layer_to_scan, + start = start_scan_address, page_size = vlayer.page_size, pdb_names = kernel_pdb_names, progress_callback = progress_callback) @@ -261,7 +277,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] virtual_layer_name = vlayer.name - try: + with contextlib.suppress(exceptions.InvalidAddressException): if vlayer.read(address, 0x2) == b'MZ': res = list( PDBUtility.pdbname_scan(ctx = context, @@ -273,8 +289,6 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): end = address + self.max_pdb_size)) if res: valid_kernel = (virtual_layer_name, address, res[0]) - except exceptions.InvalidAddressException: - pass return valid_kernel # List of methods to be run, in order, to determine the valid kernels diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 7b6adf9b4..fe5dfac52 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -2,18 +2,20 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import base64 -import gc +import datetime import json import logging import os -import pickle +import sqlite3 import urllib import urllib.parse import urllib.request -import zipfile -from typing import Dict, List, Optional +from abc import abstractmethod +from typing import Dict, Generator, Iterable, List, Optional, Tuple -from volatility3.framework import constants, exceptions, interfaces +from volatility3 import framework, schemas +from volatility3.framework import constants, interfaces +from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources from volatility3.framework.symbols import intermed @@ -22,164 +24,414 @@ vollog = logging.getLogger(__name__) BannersType = Dict[bytes, List[str]] -class SymbolBannerCache(interfaces.automagic.AutomagicInterface): - """Runs through all symbols tables and caches their banners.""" +### Identifiers - # Since this is necessary for ConstructionMagic, we set a lower priority - # The user would run it eventually either way, but running it first means it can be used that run +class IdentifierProcessor: + operating_system = None + + def __init__(self): + pass + + @classmethod + @abstractmethod + def get_identifier(cls, json) -> Optional[bytes]: + """Method to extract the identifier from a particular operating system's JSON + + Returns: + identifier is valid or None if not found + """ + raise NotImplementedError("This base class has no get_identifier method defined") + + +class WindowsIdentifier(IdentifierProcessor): + operating_system = 'windows' + separator = '|' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + """Returns the identifier for the file if one can be found""" + windows_metadata = json.get('metadata', {}).get('windows', {}).get('pdb', {}) + if windows_metadata: + guid = windows_metadata.get('GUID', None) + age = windows_metadata.get('age', None) + database = windows_metadata.get('database', None) + if guid and age and database: + return cls.generate(database, guid, age) + return None + + @classmethod + def generate(cls, pdb_name: str, guid: str, age: int) -> bytes: + return bytes(cls.separator.join([pdb_name, guid.upper(), str(age)]), 'latin-1') + + +class MacIdentifier(IdentifierProcessor): + operating_system = 'mac' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + mac_banner = json.get('symbols', {}).get('version', {}).get('constant_data', None) + if mac_banner: + return base64.b64decode(mac_banner) + return None + + +class LinuxIdentifier(IdentifierProcessor): + operating_system = 'linux' + + @classmethod + def get_identifier(cls, json) -> Optional[bytes]: + linux_banner = json.get('symbols', {}).get('linux_banner', {}).get('constant_data', None) + if linux_banner: + return base64.b64decode(linux_banner) + return None + + +### CacheManagers + +class CacheManagerInterface(interfaces.configuration.VersionableInterface): + def __init__(self, filename: str): + super().__init__() + self._filename = filename + self._classifiers = {} + for subclazz in framework.class_subclasses(IdentifierProcessor): + self._classifiers[subclazz.operating_system] = subclazz + + def add_identifier(self, location: str, operating_system: str, identifier: str): + """Adds an identifier to the store""" + pass + + def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + """Returns the location of the symbol file given the identifier + + Args: + identifier: string that uniquely identifies a particular symbol table + operating_system: optional string to restrict identifiers to just those for a particular operating system + + Returns: + The location of the symbols file that matches the identifier + """ + pass + + def get_local_locations(self) -> Iterable[str]: + """Returns a list of all the local locations""" + pass + + def update(self): + """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. + This also updates remote locations based on a cache timeout. + + """ + pass + + def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ + Dict[bytes, str]: + """Returns a dictionary of identifiers and locations + + Args: + operating_system: If set, limits responses to a specific operating system + local_only: Returns only local locations + + Returns: + A dictionary of identifiers mapped to a location + """ + pass + + def get_identifier(self, location: str) -> Optional[bytes]: + """Returns an identifier based on a specific location or None""" + pass + + def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: + """Returns all identifiers for a particular operating system""" + pass + + def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + """Returns ISF statistics based on the location + + Returns: + A tuple of base_types, types, enums, symbols, or None is location not found""" + + def get_hash(self, location: str) -> Optional[str]: + """Returns the hash of the JSON from within a location ISF""" + + +class SqliteCache(CacheManagerInterface): + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + + def __init__(self, filename: str): + super().__init__(filename) + self.cache_period = constants.SQLITE_CACHE_PERIOD + try: + self._database = self._connect_storage(filename) + except sqlite3.DatabaseError: + os.unlink(filename) + self._database = self._connect_storage(filename) + + def _connect_storage(self, path: str) -> sqlite3.Connection: + database = sqlite3.connect(path) + database.row_factory = sqlite3.Row + + database.cursor().execute( + f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})') + schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() + if not schema_version: + database.cursor().execute(f'INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCHEMA_VERSION})') + elif schema_version['schema_version'] == constants.CACHE_SQLITE_SCHEMA_VERSION: + # All good, so pass and move on + pass + else: + vollog.info(f"Previous cache schema version found: {schema_version['schema_version']}") + # TODO: Implement code if the schema changes + # Current this should never happen so we start over again + database.close() + os.unlink(path) + return self._connect_storage(path) + database.cursor().execute( + 'CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, hash TEXT,' + 'stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)') + database.commit() + return database + + def find_location(self, identifier: bytes, operating_system: Optional[str]) -> Optional[str]: + """Returns the location of the symbol file given the identifier. + If multiple locations exist for an identifier, the last found is returned + + Args: + identifier: string that uniquely identifies a particular symbol table + operating_system: optional string to restrict identifiers to just those for a particular operating system + + Returns: + The location of the symbols file that matches the identifier or None + """ + statement = 'SELECT location FROM cache WHERE identifier = ?' + parameters = (identifier,) + if operating_system is not None: + statement = 'SELECT location FROM cache WHERE identifier = ? AND operating_system = ?' + parameters = (identifier, operating_system) + results = self._database.cursor().execute(statement, parameters).fetchall() + result = None + for row in results: + result = row['location'] + return result + + def get_local_locations(self) -> Generator[str, None, None]: + result = self._database.cursor().execute('SELECT DISTINCT location FROM cache WHERE local = 1').fetchall() + for row in result: + yield row['location'] + + def is_url_local(self, url: str) -> bool: + """Determines whether an url is local or not""" + parsed = urllib.parse.urlparse(url) + if parsed.scheme in ['file', 'jar']: + return True + + def get_identifier(self, location: str) -> Optional[bytes]: + results = self._database.cursor().execute('SELECT identifier FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['identifier'] + return None + + def get_location_statistics(self, location: str) -> Optional[Tuple[int, int, int, int]]: + results = self._database.cursor().execute( + 'SELECT stats_base_types, stats_types, stats_enums, stats_symbols FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['stats_base_types'], row['stats_types'], row['stats_enums'], row['stats_symbols'] + return None + + def get_hash(self, location: str) -> Optional[str]: + results = self._database.cursor().execute('SELECT hash FROM cache WHERE location = ?', + (location,)).fetchall() + for row in results: + return row['hash'] + + def update(self, progress_callback = None): + """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. + This also updates remote locations based on a cache timeout. + + """ + on_disk_locations = set([filename for filename in intermed.IntermediateSymbolTable.file_symbol_url('')]) + cached_locations = set(self.get_local_locations()) + + new_locations = on_disk_locations.difference(cached_locations) + missing_locations = cached_locations.difference(on_disk_locations) + + cache_update = set() + files_to_timestamp = on_disk_locations.intersection(cached_locations) + if files_to_timestamp: + result = self._database.cursor().execute("SELECT location, cached FROM cache WHERE local = 1 " + f"AND cached < date('now', '{self.cache_period}');") + for row in result: + location = row['location'] + stored_timestamp = datetime.datetime.fromisoformat(row['cached']) + timestamp = stored_timestamp # Default to requiring update + + # See if the file is a local URL type we can handle: + parsed = urllib.parse.urlparse(location) + pathname = None + if parsed.scheme == 'file': + pathname = urllib.request.url2pathname(parsed.path) + if parsed.scheme == 'jar': + inner_url = urllib.parse.urlparse(parsed.path) + if inner_url.scheme == 'file': + pathname = inner_url.path.split('!')[0] + + if pathname: + timestamp = datetime.datetime.fromtimestamp(os.stat(pathname).st_mtime) + else: + vollog.log(constants.LOGLEVEL_VVVV, + "File location in database classed as local but not file/jar URL") + + # If we're supposed to include it, and our last check is older than (or equal to) the file timestamp + if row['location'] in files_to_timestamp and stored_timestamp < timestamp: + cache_update.add(row['location']) + + idextractors = list(framework.class_subclasses(IdentifierProcessor)) + + # New or not recently updated + + files_to_process = new_locations.union(cache_update) + number_files_to_process = len(files_to_process) + cursor = self._database.cursor() + try: + for counter, location in enumerate(files_to_process): + # Open location + progress_callback(counter * 100 / number_files_to_process, + f"Updating caches for {number_files_to_process} files...") + try: + with resources.ResourceAccessor().open(location) as fp: + json_obj = json.load(fp) + hash = schemas.create_json_hash(json_obj) + identifier = None + + # Get stats + stats_base_types = len(json_obj.get('base_types', {})) + stats_types = len(json_obj.get('types', {})) + stats_enums = len(json_obj.get('enums', {})) + stats_symbols = len(json_obj.get('symbols', {})) + + operating_system = None + for idextractor in idextractors: + identifier = idextractor.get_identifier(json_obj) + if identifier is not None: + operating_system = idextractor.operating_system + break + + # We don't try to validate schemas here, we do that on first use + # Store in database + cursor.execute( + "INSERT OR REPLACE INTO cache (location, identifier, operating_system, hash," + "stats_base_types, stats_types, stats_enums, stats_symbols, " + "local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", + ( + location, + identifier, + operating_system, + hash, + stats_base_types, + stats_types, + stats_enums, + stats_symbols, + self.is_url_local(location) + )) + if identifier is not None: + vollog.log(constants.LOGLEVEL_VV, f"Identified {location} as {identifier}") + else: + vollog.log(constants.LOGLEVEL_VVVV, f"No identifier found for {location}") + except Exception as excp: + vollog.log(constants.LOGLEVEL_VVVV, excp) + finally: + self._database.commit() + + # Remote Entries + + if not constants.OFFLINE and constants.REMOTE_ISF_URL: + progress_callback(0, 'Reading remote ISF list') + cursor = self._database.cursor() + cursor.execute( + f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', {self.cache_period})") + remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) + progress_callback(50, 'Reading remote ISF list') + for operating_system in constants.OS_CATEGORIES: + identifiers = remote_identifiers.process({}, operating_system = operating_system) + for identifier, location in identifiers: + cursor.execute( + "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", + (location, identifier, operating_system, False) + ) + progress_callback(100, 'Reading remote ISF list') + self._database.commit() + + # Missing entries + + if missing_locations: + self._database.cursor().execute( + f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})", + [x for x in missing_locations]) + self._database.commit() + + def get_identifier_dictionary(self, operating_system: Optional[str] = None, local_only: bool = False) -> \ + Dict[bytes, str]: + output = {} + additions = [] + statement = 'SELECT location, identifier FROM cache' + if local_only: + additions.append('local = 1') + if operating_system: + additions.append(f"operating_system = '{operating_system}'") + if additions: + statement += f" WHERE {' AND '.join(additions)}" + results = self._database.cursor().execute(statement) + for row in results: + if row['identifier'] in output and row['identifier'] and row['location']: + vollog.debug( + f"Duplicate entry for identifier {row['identifier']}: {row['location']} and {output[row['identifier']]}") + output[row['identifier']] = row['location'] + return output + + def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: + if operating_system: + results = self._database.cursor().execute('SELECT identifier FROM cache WHERE operating_system = ?', + (operating_system,)).fetchall() + else: + results = self._database.cursor().execute('SELECT identifier FROM cache').fetchall() + output = [] + for row in results: + output.append(row['identifier']) + return output + + +### Automagic + +class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): + """Runs through all symbol tables and caches their identifiers""" priority = 0 - os: Optional[str] = None - symbol_name: str = "banner_name" - banner_path: Optional[str] = None - - @classmethod - def load_banners(cls) -> BannersType: - if not cls.banner_path: - raise ValueError("Banner_path not appropriately set") - banners: BannersType = {} - if os.path.exists(cls.banner_path): - with open(cls.banner_path, "rb") as f: - # We use pickle over JSON because we're dealing with bytes objects - banners.update(pickle.load(f)) - - # Remove possibilities that can't exist locally. - remove_banners = [] - for banner in banners: - for path in banners[banner]: - url = urllib.parse.urlparse(path) - if url.scheme == 'file' and not os.path.exists(urllib.request.url2pathname(url.path)): - vollog.log( - constants.LOGLEVEL_VV, "Removing cached path {} for banner {}: file does not exist".format( - path, str(banner or b'', 'latin-1'))) - banners[banner].remove(path) - # This is probably excessive, but it's here if we need it - if url.scheme == 'jar': - zip_file, zip_path = url.path.split("!") - zip_file = urllib.parse.urlparse(zip_file).path - if ((not os.path.exists(zip_file)) or (zip_path not in zipfile.ZipFile(zip_file).namelist())): - vollog.log(constants.LOGLEVEL_VV, - "Removing cached path {} for banner {}: file does not exist".format(path, banner)) - banners[banner].remove(path) - - if not banners[banner]: - remove_banners.append(banner) - for remove_banner in remove_banners: - del banners[remove_banner] - return banners - - @classmethod - def save_banners(cls, banners): - - with open(cls.banner_path, "wb") as f: - pickle.dump(banners, f) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + self._cache = SqliteCache(identifiers_path) def __call__(self, context, config_path, configurable, progress_callback = None): """Runs the automagic over the configurable.""" - - # Bomb out if we're just the generic interface - if self.os is None: - return - - # We only need to be called once, so no recursion necessary - banners = self.load_banners() - - cacheables = self.find_new_banner_files(banners, self.os) - - new_banners = self.read_new_banners(context, config_path, cacheables, self.symbol_name, self.os, - progress_callback) - - # Add in any new banners to the existing list - for new_banner in new_banners: - banner_list = banners.get(new_banner, []) - banners[new_banner] = list(set(banner_list + new_banners[new_banner])) - - # Do remote banners *after* the JSON loading, so that it doesn't pull down all the remote JSON - self.remote_banners(banners, self.os) - - # Rewrite the cached banners each run, since writing is faster than the banner_cache validation portion - self.save_banners(banners) - - if progress_callback is not None: - progress_callback(100, f"Built {self.os} caches") + self._cache.update(progress_callback) @classmethod - def read_new_banners(cls, context: interfaces.context.ContextInterface, config_path: str, new_urls: List[str], - symbol_name: str, operating_system: str = None, - progress_callback = None) -> Optional[Dict[bytes, List[str]]]: - """Reads the any new banners for the OS in question""" - if operating_system is None: - return None - - banners = {} - - total = len(new_urls) - if total > 0: - vollog.info(f"Building {operating_system} caches...") - for current in range(total): - if progress_callback is not None: - progress_callback(current * 100 / total, f"Building {operating_system} caches") - isf_url = new_urls[current] - - isf = None - try: - # Loading the symbol table will be very slow until it's been validated - isf = intermed.IntermediateSymbolTable(context, config_path, "temp", isf_url, validate = False) - - # We should store the banner against the filename - # We don't bother with the hash (it'll likely take too long to validate) - # but we should check at least that the banner matches on load. - banner = isf.get_symbol(symbol_name).constant_data - vollog.log(constants.LOGLEVEL_VV, f"Caching banner {banner} for file {isf_url}") - - bannerlist = banners.get(banner, []) - bannerlist.append(isf_url) - banners[banner] = bannerlist - except exceptions.SymbolError: - pass - except json.JSONDecodeError: - vollog.log(constants.LOGLEVEL_VV, f"Caching file {isf_url} failed due to JSON error") - finally: - # Get rid of the loaded file, in case it sits in memory - if isf: - del isf - gc.collect() - return banners - - @classmethod - def find_new_banner_files(cls, banners: Dict[bytes, List[str]], operating_system: str) -> List[str]: - """Gathers all files and remove existing banners""" - cacheables = list(intermed.IntermediateSymbolTable.file_symbol_url(operating_system)) - for banner in banners: - for json_file in banners[banner]: - if json_file in cacheables: - cacheables.remove(json_file) - return cacheables - - @classmethod - def remote_banners(cls, banners: Dict[bytes, List[str]], operating_system = None, banner_location = None): - """Adds remote URLs to the banner list""" - if operating_system is None: - return None - - if banner_location is None: - banner_location = constants.REMOTE_ISF_URL - - if not constants.OFFLINE and banner_location is not None: - try: - rbf = RemoteBannerFormat(banner_location) - rbf.process(banners, operating_system) - except urllib.error.URLError: - vollog.debug(f"Unable to download remote banner list from {banner_location}") + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + """Returns a list of RequirementInterface objects required by this + object.""" + return [requirements.VersionRequirement(name = 'SQLiteCache', component = SqliteCache, version = (1, 0, 0))] -class RemoteBannerFormat: +class RemoteIdentifierFormat: def __init__(self, location: str): self._location = location with resources.ResourceAccessor().open(url = location) as fp: self._data = json.load(fp) if not self._verify(): - raise ValueError("Unsupported version for remote banner list format") + raise ValueError("Unsupported version for remote identifier list format") def _verify(self) -> bool: version = self._data.get('version', 0) @@ -188,23 +440,22 @@ class RemoteBannerFormat: return True return False - def process(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): - raise ValueError("Banner List version not verified") + def process(self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]) -> Generator[ + Tuple[bytes, str], None, None]: + raise ValueError("Identifier List version not verified") - def process_v1(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): + def process_v1(self, identifiers: Optional[Dict[bytes, List[str]]], operating_system: Optional[str]) -> Generator[ + Tuple[bytes, str], None, None]: if operating_system in self._data: - for banner in self._data[operating_system]: - binary_banner = base64.b64decode(banner) - file_list = banners.get(binary_banner, []) - for value in self._data[operating_system][banner]: - if value not in file_list: - file_list = file_list + [value] - banners[binary_banner] = file_list + for identifier in self._data[operating_system]: + binary_identifier = base64.b64decode(identifier) + for value in self._data[operating_system][identifier]: + yield binary_identifier, value if 'additional' in self._data: for location in self._data['additional']: try: - subrbf = RemoteBannerFormat(location) - subrbf.process(banners, operating_system) + subrbf = RemoteIdentifierFormat(location) + yield from subrbf.process(identifiers, operating_system) except IOError: vollog.debug(f"Remote file not found: {location}") - return banners + return identifiers diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 72dc071a5..7a197dffc 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -3,9 +3,10 @@ # import logging -from typing import Any, Iterable, List, Tuple, Type, Optional, Callable +import os +from typing import Any, Callable, Iterable, List, Optional, Tuple -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces, layers from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners @@ -18,7 +19,7 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): priority = 40 banner_config_key: str = "banner" - banner_cache: Optional[Type[symbol_cache.SymbolBannerCache]] = None + operating_system: Optional[str] = None symbol_class: Optional[str] = None find_aslr: Optional[Callable] = None @@ -27,14 +28,22 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): self._requirements: List[Tuple[str, interfaces.configuration.RequirementInterface]] = [] self._banners: symbol_cache.BannersType = {} + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.VersionRequirement(name = 'SQLiteCache', + component = symbol_cache.SqliteCache, + version = (1, 0, 0)) + ] + @property def banners(self) -> symbol_cache.BannersType: """Creates a cached copy of the results, but only it's been requested.""" if not self._banners: - if not self.banner_cache: - raise RuntimeError(f"Cache has not been properly defined for {self.__class__.__name__}") - self._banners = self.banner_cache.load_banners() + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + cache = symbol_cache.SqliteCache(identifiers_path) + self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system) return self._banners def __call__(self, @@ -103,8 +112,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): vollog.debug(f"Identified banner: {repr(banner)}") symbol_files = self.banners.get(banner, None) if symbol_files: - isf_path = symbol_files[0] - vollog.debug(f"Using symbol library: {symbol_files[0]}") + isf_path = symbol_files + vollog.debug(f"Using symbol library: {symbol_files}") clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join @@ -116,9 +125,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): requirement.construct(context, config_path) break else: - if symbol_files: - vollog.debug(f"Symbol library path not found: {symbol_files[0]}") - # print("Kernel", banner, hex(banner_offset)) + vollog.debug(f"Symbol library path not found for: {banner}") + # print("Kernel", banner, hex(banner_offset)) else: vollog.debug("No existing banners found") # TODO: Fallback to generic regex search? diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index eb63a75e5..aaef3e820 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -28,9 +28,9 @@ The self-referential indices for older versions of windows are listed below: """ import logging import struct -from typing import Generator, List, Optional, Tuple, Type, Iterable +from typing import Generator, Iterable, List, Optional, Tuple, Type -from volatility3.framework import interfaces, layers, constants +from volatility3.framework import constants, interfaces, layers from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel @@ -116,10 +116,27 @@ class DtbSelfRefPae(DtbSelfReferential): mask = 0x3FFFFFFFFFF000, reserved_bits = 0x0) - def __call__(self, *args, **kwargs): - dtb = super().__call__(*args, **kwargs) + @staticmethod + def _and_bytes(abytes, bbytes): + return bytes([a & b for a, b in zip(abytes[::-1], bbytes[::-1])][::-1]) + + def __call__(self, data: bytes, data_offset: int, page_offset: int) -> Optional[Tuple[int, int]]: + dtb = super().__call__(data, data_offset, page_offset) if dtb: - return dtb[0] - 0x4000, dtb[1] + # Find the top page + top_pae_page = dtb[0] - 0x4000 + # The top page should map to the next four pages after it + # Build what we expect the page table to be + expected_table = b''.join([struct.pack(self.ptr_struct, top_pae_page + (i * 0x1000)) for i in range(1, 5)]) + # Mask off the page bits of top level page map + page_table_mask = b"\x00\xf0\xff\xff\xff\xff\xff\xff" * 4 + page_table = data[top_pae_page - data_offset: top_pae_page - data_offset + (4 * self.ptr_size)] + # Compare them + anded_bytes = self._and_bytes(page_table, page_table_mask) + if (anded_bytes == expected_table): + return top_pae_page, dtb[1] + # Return None since the dtb value *isn't* None + return None return dtb @@ -197,35 +214,58 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): context.config[interfaces.configuration.path_join( config_path, "page_map_offset")] = base_layer.metadata['page_map_offset'] layer = layer_type(context, config_path = config_path, name = new_layer_name, metadata = {'os': 'Windows'}) + page_map_offset = context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] + vollog.debug(f"DTB was given to us by base layer: {hex(page_map_offset)}") + return layer # Self Referential finder for description, tests, sections in cls.test_sets: vollog.debug(description) # There is a very high chance that the DTB will live in these very narrow segments, assuming we couldn't find them previously - hits = context.layers[layer_name].scan(context, - PageMapScanner(tests = tests), - sections = sections, - progress_callback = progress_callback) + hits = base_layer.scan(context, + PageMapScanner(tests = tests), + sections = sections, + progress_callback = progress_callback) # Flatten the generator def sort_by_tests(x): + """Key used to sort by tests""" return tests.index(x[0]), x[1] + def get_max_pointer(page_table, test, ptr_size: int): + """Determines a pointer from a page_table""" + max_ptr = 0 + for index in range(0, len(page_table), ptr_size): + pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0] + # Make sure the pointer is valid, ignore large pages which would require more calculation + if pointer & 0x1 and not pointer & 0x80: + max_ptr = max(max_ptr, (pointer ^ (pointer & 0xfff)) % test.layer_type.maximum_address) + return max_ptr + hits = sorted(list(hits), key = sort_by_tests) - if hits: - # TODO: Decide which to use if there are multiple options - test, page_map_offset = hits[0] - vollog.debug(f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}") - new_layer_name = context.layers.free_layer_name("IntelLayer") - config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name) - context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name - context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] = page_map_offset - # TODO: Need to determine the layer type (chances are high it's x64, hence this default) - layer = test.layer_type(context, - config_path = config_path, - name = new_layer_name, - metadata = {'os': 'Windows'}) + for test, page_map_offset in hits: + # Turn the page tables into integers and find the largest one + page_table = base_layer.read(page_map_offset, 0x1000) + ptr_size = struct.calcsize(test.ptr_struct) + max_pointer = get_max_pointer(page_table, test, ptr_size) + + if max_pointer <= base_layer.maximum_address: + vollog.debug(f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}") + new_layer_name = context.layers.free_layer_name("IntelLayer") + config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name) + context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name + context.config[ + interfaces.configuration.path_join(config_path, "page_map_offset")] = page_map_offset + layer = test.layer_type(context, + config_path = config_path, + name = new_layer_name, + metadata = {'os': 'Windows'}) + break + else: + vollog.debug( + f"Max pointer for hit with test {test.__class__.__name__} not met: {hex(max_pointer)} > {hex(base_layer.maximum_address)}") + if layer is not None and config_path: break if layer is not None and config_path: @@ -238,6 +278,8 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): """Class to read swap_layers filenames from single-swap-layers, create the layers and populate the single-layers swap_layers.""" + exclusion_list = ['linux', 'mac'] + def __call__(self, context: interfaces.context.ContextInterface, config_path: str, diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index a0fb186ae..cc4f05ae6 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -10,7 +10,7 @@ expect to be in the context (such as particular layers or symboltables). """ import abc import logging -from typing import Any, ClassVar, List, Optional, Type, Dict, Tuple +from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type from volatility3.framework import constants, interfaces @@ -303,7 +303,8 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if not subreq.optional]): + [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if + not subreq.optional]): return None obj = self._construct_class(context, config_path, args) @@ -358,7 +359,8 @@ class SymbolTableRequirement(interfaces.configuration.ConstructableRequirementIn args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if not subreq.optional]): + [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if + not subreq.optional]): return None # Fill out the parameter for class creation @@ -406,13 +408,19 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type config_path = interfaces.configuration.path_join(config_path, self.name) - if len(self._version) > 0 and self._component.version[0] != self._version[0]: - return {config_path: self} - if len(self._version) > 1 and self._component.version[1] < self._version[1]: + if not self.matches_required(self._version, self._component.version): return {config_path: self} context.config[interfaces.configuration.path_join(config_path, self.name)] = True return {} + @classmethod + def matches_required(cls, required: Tuple[int, ...], version: Tuple[int, int, int]) -> bool: + if len(required) > 0 and version[0] != required[0]: + return False + if len(required) > 1 and version[1] < required[1]: + return False + return True + class PluginRequirement(VersionRequirement): @@ -462,11 +470,20 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa "TypeError - Module Requirement only accepts string labels: {}".format(repr(value))) return {config_path: self} + result = {} + for subreq in self._requirements: + req_unsatisfied = self._requirements[subreq].unsatisfied(context, config_path) + if req_unsatisfied: + result.update(req_unsatisfied) + if not result: + vollog.log(constants.LOGLEVEL_V, f"IndexError - No configuration provided: {config_path}") + result = {config_path: self} + ### NOTE: This validate method has side effects (the dependencies can change)!!! self._validate_class(context, interfaces.configuration.parent_path(config_path)) - vollog.log(constants.LOGLEVEL_V, f"IndexError - No configuration provided: {config_path}") - return {config_path: self} + + return result def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None: """Constructs the appropriate layer and adds it based on the class parameter.""" @@ -482,7 +499,8 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa args = {"context": context, "config_path": config_path, "name": name} if any( - [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if not subreq.optional]): + [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if + not subreq.optional]): return None obj = self._construct_class(context, config_path, args) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 4ce2dbe2d..b19e80472 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,7 +9,7 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys -from typing import Optional, Callable +from typing import Callable, Optional import volatility3.framework.constants.linux import volatility3.framework.constants.windows @@ -39,8 +39,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 4 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature @@ -63,21 +63,26 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" -if sys.platform == 'windows': - CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") +SQLITE_CACHE_PERIOD = '-3 days' +"""SQLite time modifier for how long each item is valid in the cache for""" + +if sys.platform == 'win32': + CACHE_PATH = os.path.realpath(os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3")) os.makedirs(CACHE_PATH, exist_ok = True) -LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") -""""Default location to record information about available linux banners""" +IDENTIFIERS_FILENAME = "identifier.cache" +"""Default location to record information about available identifiers""" -MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") -""""Default location to record information about available mac banners""" +CACHE_SQLITE_SCHEMA_VERSION = 1 +"""Version for the sqlite3 cache schema""" BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues" ProgressCallback = Optional[Callable[[float, str], None]] """Type information for ProgressCallback objects""" +OS_CATEGORIES = ['windows', 'mac', 'linux'] + class Parallelism(enum.IntEnum): """An enumeration listing the different types of parallelism applied to diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index c25ea0e2f..c0f85593f 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -11,3 +11,6 @@ KERNEL_NAME = "__kernel__" # arch/x86/include/asm/page_types.h PAGE_SHIFT = 12 """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" + +# include/linux/sched.h +PF_KTHREAD = 0x00200000 # I'm a kernel thread diff --git a/volatility3/framework/constants/windows/__init__.py b/volatility3/framework/constants/windows/__init__.py index 372897598..7face984a 100644 --- a/volatility3/framework/constants/windows/__init__.py +++ b/volatility3/framework/constants/windows/__init__.py @@ -1,10 +1,12 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""Volatility 3 Linux Constants. +"""Volatility 3 Windows Constants. Windows-specific values that aren't found in debug symbols """ KERNEL_MODULE_NAMES = ["ntkrnlmp", "ntkrnlpa", "ntkrpamp", "ntoskrnl"] """The list of names that kernel modules can have within the windows OS""" + +PE_MAX_EXTRACTION_SIZE = 1024 * 1024 * 256 diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 518215ab4..85a7d32b7 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -141,7 +141,7 @@ class Context(interfaces.context.ContextInterface): layer_name: The layer within the context in which the module exists offset: The offset at which the module exists in the layer native_layer_name: The default native layer for objects constructed by the module - size: The size, in bytes, that the module occupys from offset location within the layer named layer_name + size: The size, in bytes, that the module occupies from offset location within the layer named layer_name """ if size: return SizedModule.create(self, @@ -321,7 +321,7 @@ class SizedModule(Module): The mapping should be sorted and should be quicker than reading the data We turn it into JSON to make a common string and use a - quick hash, because collissions are unlikely + quick hash, because collisions are unlikely """ layer = self._context.layers[self.layer_name] if not isinstance(layer, interfaces.layers.TranslationLayerInterface): diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index c310f5b4a..713f91da0 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -9,9 +9,9 @@ that a user has not filled. """ import logging from abc import ABCMeta -from typing import Any, List, Optional, Tuple, Union, Type +from typing import Any, List, Optional, Tuple, Type, Union -from volatility3.framework import interfaces, constants +from volatility3.framework import constants, interfaces from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -40,13 +40,17 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla priority = 10 """An ordering to indicate how soon this automagic should be run""" + exclusion_list = [] + """A list of plugin categories (typically operating systems) which the plugin will not operate on""" + def __init__(self, context: interfaces.context.ContextInterface, config_path: str, *args, **kwargs) -> None: super().__init__(context, config_path) for requirement in self.get_requirements(): if not isinstance(requirement, (interfaces.configuration.SimpleTypeRequirement, - requirements.ChoiceRequirement, requirements.ListRequirement)): + requirements.ChoiceRequirement, requirements.ListRequirement, + requirements.VersionRequirement)): raise TypeError( - "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement or ListRequirement") + "Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement, ListRequirement or VersionRequirement") def __call__(self, context: interfaces.context.ContextInterface, diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 7dc046a3e..e271ef6d4 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -73,7 +73,7 @@ class HierarchicalDict(collections.abc.Mapping): separator: str = CONFIG_SEPARATOR) -> None: """ Args: - initial_dict: A dictionary to populate the HierachicalDict with initially + initial_dict: A dictionary to populate the HierarchicalDict with initially separator: A custom hierarchy separator (defaults to CONFIG_SEPARATOR) """ if not (isinstance(separator, str) and len(separator) == 1): @@ -523,7 +523,7 @@ class ConstructableRequirementInterface(RequirementInterface): must happen after the class configuration value has been provided). These values are then provided to the object's constructor by name as arguments (as well as the standard `context` and `config_path` - arguments. + arguments). """ def __init__(self, *args, **kwargs) -> None: diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index c52e1aaa5..b8470ae47 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -129,7 +129,7 @@ class ContextInterface(metaclass = ABCMeta): layer_name: The layer the module is associated with (which layer the module lives within) offset: The initial/base offset of the module (used as the offset for relative symbols) native_layer_name: The default native_layer_name to use when the module constructs objects - size: The size, in bytes, that the module occupys from offset location within the layer named layer_name + size: The size, in bytes, that the module occupies from offset location within the layer named layer_name Returns: A module object diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index a42282c39..7ff110c6e 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -307,7 +307,7 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla while length > 0: chunk_size = min(length, scanner.chunk_size + scanner.overlap) yield [(layer_name, mapped_offset, chunk_size)], offset + chunk_size - # It we've got more than the scanner's chunk_size, only move up by the chunk_size + # If we've got more than the scanner's chunk_size, only move up by the chunk_size if chunk_size > scanner.chunk_size: chunk_size -= scanner.overlap length -= chunk_size @@ -517,7 +517,7 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): yield output, chunk_position output = [] chunk_position = chunk_start - # Take from chunk_position as far as far as the block can go, + # Take from chunk_position as far as the block can go, # or as much left of a scanner chunk as we can chunk_size = min(block_end - chunk_position, scanner.chunk_size + scanner.overlap - (chunk_position - chunk_start)) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 811327094..1df13ca81 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -6,6 +6,7 @@ interpreted values of data from a layer.""" import abc import collections import collections.abc +import contextlib import logging from typing import Any, Dict, List, Mapping, Optional @@ -115,7 +116,8 @@ class ObjectInterface(metaclass = abc.ABCMeta): mask = context.layers[object_info.layer_name].address_mask normalized_offset = object_info.offset & mask - self._vol = collections.ChainMap({}, object_info, {'type_name': type_name, 'offset': normalized_offset}, kwargs) + vol_info_dict = {'type_name': type_name, 'offset': normalized_offset} + self._vol = collections.ChainMap({}, vol_info_dict, object_info, kwargs) self._context = context def __getattr__(self, attr: str) -> Any: @@ -156,7 +158,7 @@ class ObjectInterface(metaclass = abc.ABCMeta): """ # TODO: Carefully consider the implications of casting and how it should work if constants.BANG not in new_type_name: - symbol_table = self.vol['type_name'].split(constants.BANG)[0] + symbol_table = self.get_symbol_table_name() new_type_name = symbol_table + constants.BANG + new_type_name object_template = self._context.symbol_space.get_type(new_type_name) object_template = object_template.clone() @@ -186,11 +188,9 @@ class ObjectInterface(metaclass = abc.ABCMeta): """ if self.has_member(member_name): # noinspection PyBroadException - try: + with contextlib.suppress(Exception): _ = getattr(self, member_name) return True - except Exception: - pass return False def has_valid_members(self, member_names: List[str]) -> bool: @@ -240,6 +240,12 @@ class ObjectInterface(metaclass = abc.ABCMeta): the child member.""" raise KeyError(f"Template does not contain any children: {template.vol.type_name}") + @classmethod + @abc.abstractmethod + def child_template(cls, template: 'Template', child: str) -> 'interfaces.objects.Template': + """Returns the template of the child member from the parent.""" + raise KeyError(f"Template does not contain any children: {template.vol.type_name}") + @classmethod @abc.abstractmethod def has_member(cls, template: 'Template', member_name: str) -> bool: @@ -304,6 +310,10 @@ class Template: """Returns the relative offset of the `child` member from its parent offset.""" + @abc.abstractmethod + def child_template(self, child: str) -> 'interfaces.objects.Template': + """Returns the `child` member template from its parent.""" + @abc.abstractmethod def replace_child(self, old_child: 'Template', new_child: 'Template') -> None: """Replaces `old_child` with `new_child` in the list of children.""" diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index 7f80425a4..9368009a9 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""All plugins output a TreeGrid object which must then be rendered (eithe by a +"""All plugins output a TreeGrid object which must then be rendered (either by a GUI, or as text output, html output or in some other form. This module defines both the output format (:class:`TreeGrid`) and the diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index b271de412..1ad30cfdf 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -167,6 +167,20 @@ class BaseSymbolTableInterface: """ raise NotImplementedError("Abstract method set_type_class not implemented yet.") + def optional_set_type_class(self, name: str, clazz: Type[objects.ObjectInterface]) -> bool: + """Calls the set_type_class function but does not throw an exception. + Returns whether setting the type class was successful. + Args: + name: The name of the type to override the class for + clazz: The actual class to override for the provided type name + """ + try: + self.set_type_class(name, clazz) + + return True + except ValueError: + return False + def get_type_class(self, name: str) -> Type[objects.ObjectInterface]: """Returns the class associated with a Symbol type.""" raise NotImplementedError("Abstract method get_type_class not implemented yet.") diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index acc4493f4..f31737232 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + """Functions that read AVML files. The user of the file doesn't have to worry about the compression, diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index 550161e6d..e019bcbcd 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + """Codecs used for encoding or decoding data should live here diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index c690c8d8f..6194501ee 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -1,6 +1,7 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib import logging import struct from typing import Tuple, Optional @@ -202,11 +203,9 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: for layer in [WindowsCrashDump32Layer, WindowsCrashDump64Layer]: - try: + with contextlib.suppress(WindowsCrashDumpFormatException): layer.check_header(context.layers[layer_name]) new_name = context.layers.free_layer_name(layer.__name__) context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name return layer(context, new_name, new_name) - except WindowsCrashDumpFormatException: - pass return None diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index 8c492ca85..fb0442cfe 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import io import logging import urllib.parse diff --git a/volatility3/framework/layers/linear.py b/volatility3/framework/layers/linear.py index c5cb47bdc..383f3d558 100644 --- a/volatility3/framework/layers/linear.py +++ b/volatility3/framework/layers/linear.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import functools from typing import List, Optional, Tuple, Iterable diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 73d46b211..b09055c90 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -5,7 +5,7 @@ import logging import threading from typing import Any, Dict, IO, List, Optional, Union -from volatility3.framework import exceptions, interfaces, constants +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources @@ -88,6 +88,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): self._accessor = resources.ResourceAccessor() self._file_: Optional[IO[Any]] = None self._size: Optional[int] = None + self._maximum_address: Optional[int] = None # Construct the lock now (shared if made before threading) in case we ever need it self._lock: Union[DummyLock, threading.Lock] = DummyLock() if constants.PARALLELISM == constants.Parallelism.Threading: @@ -113,14 +114,15 @@ class FileLayer(interfaces.layers.DataLayerInterface): def maximum_address(self) -> int: """Returns the largest available address in the space.""" # Zero based, so we return the size of the file minus 1 - if self._size: - return self._size + if self._maximum_address: + return self._maximum_address with self._lock: orig = self._file.tell() self._file.seek(0, 2) self._size = self._file.tell() self._file.seek(orig) - return self._size + self._maximum_address = self._size - 1 + return self._maximum_address @property def minimum_address(self) -> int: @@ -189,7 +191,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): """Closes the file handle.""" self._file.close() - def __del__(self) -> None: + def __exit__(self, type, value, traceback) -> None: self.destroy() @classmethod diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 4ce17bb63..907116e99 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -1,16 +1,19 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import bisect import functools import json -import math -from typing import Optional, Dict, Any, Tuple, List, Set +import logging +import re +import struct +from typing import Any, Dict, List, Optional, Set, Tuple -from volatility3.framework import interfaces, exceptions, constants -from volatility3.framework.layers import segmented +from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework.layers import scanners, segmented from volatility3.framework.symbols import intermed +vollog = logging.getLogger(__name__) + class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): """A Qemu suspend-to-disk translation layer.""" @@ -34,6 +37,34 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): SEGMENT_FLAG_XBZRLE = 0x40 SEGMENT_FLAG_HOOK = 0x80 + # See https://qemu.readthedocs.io/en/latest/devel/memory.html for more info + # + # At least the following values could occur for devices using > 3-4 GB RAM: + # +--------------------------------+--------------------------------+------------+-------------+ + # | Architecture | Reference Code | Hole Start | Hole End | + # +--------------------------------+--------------------------------+------------+-------------+ + # | PC i440FX + PIIX "New Default" | qemu/hw/i386/pc_piix.c:98 | 0xc0000000 | 0x100000000 | + # | PC i440FX + PIIX "Old Default" | qemu/hw/i386/pc_piix.c:98 | 0xe0000000 | 0x100000000 | + # | PC Q35 + ICH9 | qemu/hw/i386/pc_q35.c:141 | 0x80000000 | 0x100000000 | + # | MicroVM | qemu/hw/i386/microvm.c:291 | 0xc0000000 | 0x100000000 | + # | Xen | qemu/hw/i386/xen/xen-hvm.c:248 | 0xf0000000 | 0x100000000 | + # +--------------------------------+--------------------------------+------------+-------------+ + # + # For now, we assume that the parameter max-ram-below-4g is not set, since this parameter influences the size + # and location of the memory gap. Deviating hole sizes could eventually be detected for Linux by e.g. scanning + # for dmesg entries with a regex like rb'\[mem (0x[0-9a-f]{4,10})-0x[0-9a-f]{4,10}\] available for PCI devices' + + distro_re = r"(\w+[\d{1,2}\.]*)" + + pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000), + re.compile(r"^pc-i440fx-[01]\.\d$"): (0xe0000000, 0xe0000000, 0x100000000), + re.compile(r"^pc-q35-\d\.\d$"): (0xb0000000, 0x80000000, 0x100000000), + re.compile(r"^microvm$"): (0xc0000000, 0xc0000000, 0x100000000), + re.compile(r"^xen$"): (0xf0000000, 0xf0000000, 0x100000000), + re.compile(r"^pc-i440fx-" + distro_re + r"$"): (0xe0000000, 0xc0000000, 0x100000000), + re.compile(r"^pc-q35-" + distro_re + r"$"): (0xb0000000, 0x80000000, 0x100000000), + } + def __init__(self, context: interfaces.context.ContextInterface, config_path: str, @@ -41,8 +72,12 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): metadata: Optional[Dict[str, Any]] = None) -> None: self._qemu_table_name = intermed.IntermediateSymbolTable.create(context, config_path, 'generic', 'qemu') self._configuration = None + self._architecture = None self._compressed: Set[int] = set() self._current_segment_name = b'' + self._pci_hole_start = 0 + self._pci_hole_end = 0 + self._pci_hole_minimum = 0 super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) @classmethod @@ -52,10 +87,11 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): raise exceptions.LayerException(name, 'No QEMU magic bytes') if header[4:] != b'\x00\x00\x00\x03': raise exceptions.LayerException(name, 'Unsupported QEMU version found') + vollog.debug("QEVM header found") def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, name: str) -> Any: """Reads the JSON configuration from the end of the file""" - chunk_size = 0x4096 + chunk_size = 4096 data = b'' for i in range(base_layer.maximum_address, base_layer.minimum_address, -chunk_size): if i != base_layer.maximum_address: @@ -66,6 +102,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if start_of_json >= 0: data = data[start_of_json:] return json.loads(data) + # No JSON configuration found at the end of the file, return empty dict return dict() raise exceptions.LayerException(name, "Invalid JSON configuration at the end of the file") @@ -74,30 +111,43 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): done = None segments = [] + size_array = {} base_layer = self.context.layers[self._base_layer] while not done: - addr = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', - offset = index, - layer_name = self._base_layer) + # Use struct.unpack here for performance improvements + addr = struct.unpack('>Q', base_layer.read(index, 8))[0] + + # Flags are stored in the n least significant bits, where n equals the bit-length of pagesize flags = addr & (page_size - 1) - page_size_bits = int(math.log(page_size, 2)) - addr = (addr >> page_size_bits) << page_size_bits + # addr equals the highest multiple of pagesize <= offset + # (We assume that page_size is a power of 2) + addr = addr ^ (addr & (page_size - 1)) index += 8 + if addr >= self._pci_hole_start: + addr += self._pci_hole_end - self._pci_hole_start + if flags & self.SEGMENT_FLAG_MEM_SIZE: namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) while namelen != 0: - # if base_layer.read(index + 1, namelen) == b'pc.ram': - # total_size = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', - # offset = index + 1 + namelen, - # layer_name = self._base_layer) + total_size = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', + offset = index + 1 + namelen, + layer_name = self._base_layer) + size_array[base_layer.read(index + 1, namelen)] = total_size index += 1 + namelen + 8 namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) + highest_possible_maximum = max([x[0] for x in self.pci_hole_table.values()]) + 1 + if size_array.get(b'pc.ram', highest_possible_maximum) < self._pci_hole_minimum: + # Turns off the pci_hole if it's not supposed to be there + vollog.debug( + f"QEVM turning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}") + self._pci_hole_start, self._pci_hole_end = 0, 0 + if flags & (self.SEGMENT_FLAG_COMPRESS | self.SEGMENT_FLAG_PAGE): if not (flags & self.SEGMENT_FLAG_CONTINUE): namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char', @@ -127,10 +177,28 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): self._configuration = self._read_configuration(base_layer, self.name) section_byte = -1 index = 8 + section_info = dict() current_section_id = -1 - version_id = -1 - name = None + arch_detected = False while section_byte != self.QEVM_EOF and index <= base_layer.maximum_address: + if index > 20 and not arch_detected: + # We're past where the QEVM_CONFIGURATION might be, so set the values + # If no architecture has been set, try to determine it using fallback mechanisms + if not self._architecture: + self._architecture = self._fallback_determine_architecture() + if self._architecture is None: + vollog.log(constants.LOGLEVEL_VV, f"QEVM architecture could not be determined") + + # Once all segments have been read, determine the PCI hole if any + for regex in self.pci_hole_table: + if regex.match(self._architecture): + self._pci_hole_minimum, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex] + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}") + break + else: + vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}") + arch_detected = True + section_byte = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned char', offset = index, layer_name = self._base_layer) @@ -139,6 +207,9 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', offset = index, layer_name = self._base_layer) + self._architecture = self.context.object(self._qemu_table_name + constants.BANG + 'string', + offset = index + 4, layer_name = self._base_layer, + max_length = section_len) index += 4 + section_len elif section_byte == self.QEVM_SECTION_START or section_byte == self.QEVM_SECTION_FULL: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', @@ -163,6 +234,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): offset = index, layer_name = self._base_layer) index += 4 + # Store section info for handling QEVM_SECTION_PARTs later on + section_info[current_section_id] = {'name': name, 'version_id': version_id} # Read additional data index = self.extract_data(index, name, version_id) elif section_byte == self.QEVM_SECTION_PART or section_byte == self.QEVM_SECTION_END: @@ -172,7 +245,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): current_section_id = section_id index += 4 # Read additional data - index = self.extract_data(index, name, version_id) + index = self.extract_data(index, section_info[current_section_id]['name'], + section_info[current_section_id]['version_id']) elif section_byte == self.QEVM_SECTION_FOOTER: section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', offset = index, @@ -186,11 +260,52 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): else: raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}') + def _fallback_determine_architecture(self) -> str: + architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|\w+[\d{1,2}\.]*)' + default_suffix = "-2.0" + base_layer = self.context.layers[self._base_layer] + + vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used") + + res = scanners.RegExScanner(architecture_pattern) + for offset in base_layer.scan(context = self.context, scanner = res): + line = base_layer.read(offset, 64) + regex_results = re.search(architecture_pattern, line) + architecture = regex_results.group().decode() + return architecture + + # If that does not work, look in configuration JSON for devices specific to a certain architecture + architecture = None + for device in self._configuration.get('devices', []): + device_name = device.get('vmsd_name', '').lower() + if 'i440fx' in device_name or 'piix' in device_name: + architecture = 'pc-i440fx' + default_suffix + break + elif 'ich9' in device_name: + architecture = 'pc-q35' + default_suffix + break + if architecture: + vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}') + return architecture + + # Still haven't found architecture, switch to fallback-method + architecture_pattern = rb'Standard PC \((i440FX|Q35)' + res = scanners.RegExScanner(architecture_pattern) + for offset in base_layer.scan(context = self.context, scanner = res): + line = base_layer.read(offset, 64) + regex_results = re.search(architecture_pattern, line) + architecture = "pc-" + regex_results.groups()[0].decode().lower() + default_suffix + vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}') + return architecture + + vollog.warning("Could not determine QEMU target architecture!") + return None + def extract_data(self, index, name, version_id): if name == 'ram': if version_id != 4: raise exceptions.LayerException(f"QEMU unknown RAM version_id {version_id}") - new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', None) or 4096) + new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', 4096)) self._segments += new_segments elif name == 'spapr/htab': if version_id != 1: @@ -209,6 +324,13 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): layer_name = self._base_layer) htab_index, htab_n_valid, htab_n_invalid = htab index += 8 + (htab_n_valid * self.HASH_PTE_SIZE_64) + elif name == 'dirty-bitmap': + index += 1 + elif name == 'pbs-state': + section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long', + offset = index, + layer_name = self._base_layer) + index += 8 + section_len return index def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes: @@ -218,10 +340,12 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): of the starting data. It is the responsibility of the layer to turn the provided data chunk into the right portion of data necessary. """ - start_offset, _, start_mapped_offset, _ = self._segments[ - bisect.bisect_right(self._segments, (offset, 0xffffffffffffff,)) - 1] - if start_mapped_offset in self._compressed: - data = (data * 0x1000) + page_size = self._configuration.get('page_size', 4096) + # start_offset equals the highest multiple of pagesize <= offset + # (We assume that page_size is a power of 2) + start_offset = offset ^ (offset & (page_size - 1)) + if start_offset in self._compressed: + data = (data * page_size) result = data[offset - start_offset:output_length + offset - start_offset] return result diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 55a6e5186..ec7aed217 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import logging from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union @@ -92,11 +92,9 @@ class RegistryHive(linear.LinearlyMappedLayer): @property def root_cell_offset(self) -> int: """Returns the offset for the root cell in this hive.""" - try: + with contextlib.suppress(InvalidAddressException): if self._base_block.Signature.cast("string", max_length = 4, encoding = "latin-1") == 'regf': return self._base_block.RootCell - except InvalidAddressException: - pass return 0x20 def get_cell(self, cell_offset: int) -> 'objects.StructType': @@ -201,11 +199,11 @@ class RegistryHive(linear.LinearlyMappedLayer): if offset & 0x7fffffff > self._get_hive_maxaddr(volatile): vollog.log(constants.LOGLEVEL_VVV, "Layer {} couldn't translate offset {}, greater than {} in {} store of {}".format( - self.name, - hex(offset & 0x7fffffff), - hex(self._get_hive_maxaddr(volatile)), - "volative" if volatile else "non-volatile", - self.get_name())) + self.name, + hex(offset & 0x7fffffff), + hex(self._get_hive_maxaddr(volatile)), + "volative" if volatile else "non-volatile", + self.get_name())) raise RegistryInvalidIndex(self.name, "Mapping request for value greater than maxaddr") storage = self.hive.Storage[volatile] @@ -252,14 +250,13 @@ class RegistryHive(linear.LinearlyMappedLayer): def is_valid(self, offset: int, length: int = 1) -> bool: """Returns a boolean based on whether the offset is valid or not.""" - try: + with contextlib.suppress(exceptions.InvalidAddressException): # Pass this to the lower layers for now return all([ self.context.layers[layer].is_valid(offset, length) for (_, _, offset, length, layer) in self.mapping(offset, length) ]) - except exceptions.InvalidAddressException: - return False + return False @property def minimum_address(self) -> int: diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index b6ef1b6ba..dca215c85 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -10,10 +10,11 @@ import logging import lzma import os import ssl +import sys import urllib.parse import urllib.request import zipfile -from typing import Optional, Any, IO, List +from typing import Any, IO, List, Optional from urllib import error from volatility3 import framework @@ -82,7 +83,7 @@ class ResourceAccessor(object): """Determines whether a URLs contents should be cached""" parsed_url = urllib.parse.urlparse(url) - return self._enable_cache and not parsed_url.scheme in self._non_cached_schemes() + return self._enable_cache and parsed_url.scheme not in self._non_cached_schemes() @staticmethod def _non_cached_schemes() -> List[str]: @@ -100,6 +101,21 @@ class ResourceAccessor(object): """ urllib.request.install_opener(urllib.request.build_opener(*self._handlers)) + # Python bug 46654 + if sys.platform == 'win32': + # We only need to worry about UNC paths on windows, on linux they'd be smb:// and need pysmb or similar + parsed_url = urllib.parse.urlparse(url, scheme = 'file') + # Only worry about file scheme URLs, make sure that there's either a host or + # the unparsing left an extra slash at the start (which will get lost with urlunparse) + if parsed_url.scheme == 'file' and (parsed_url.netloc or parsed_url.path.startswith('//')): + # Change the netloc to '/' and then prepend the netloc to the path + # Urlunparse will remove extra initial slashes from path, hence setting netloc + new_url = urllib.parse.urlunparse((parsed_url.scheme, '/', + '/' + parsed_url.netloc + parsed_url.path, parsed_url.params, + parsed_url.query, parsed_url.fragment)) + vollog.log(constants.LOGLEVEL_VVVV, f'UNC path detected, converted path {url} to {new_url}') + url = new_url + try: fp = urllib.request.urlopen(url, context = self._context) except error.URLError as excp: @@ -155,6 +171,8 @@ class ResourceAccessor(object): cache_file.write(block) block = fp.read(block_size) cache_file.close() + else: + vollog.debug(f"Using already cached file at: {temp_filename}") # Re-open the cache with a different mode # Since we don't want people thinking they're able to save to the cache file, # open it in read mode only and allow breakages to happen if they wanted to write @@ -166,14 +184,12 @@ class ResourceAccessor(object): stop = False while not stop: detected = None - try: + with contextlib.suppress(AttributeError, IOError): # Detect the content detected = magic.detect_from_fobj(curfile) IMPORTED_MAGIC = True # This is because python-magic and file provide a magic module # Only file's python has magic.detect_from_fobj - except (AttributeError, IOError): - pass if detected: if detected.mime_type == 'application/x-xz': diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index 85c8390d9..ec66f2708 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -31,7 +31,7 @@ class BytesScanner(layers.ScannerInterface): class RegExScanner(layers.ScannerInterface): """A scanner that can be provided with a bytes-object regular expression pattern - The scanner will scqn all blocks for the regular expression and report the absolute offset of any finds + The scanner will scan all blocks for the regular expression and report the absolute offset of any finds The default flags include DOTALL, since the searches are through binary data and the newline character should have no specific significance in such searches""" @@ -95,7 +95,7 @@ class MultiStringScanner(layers.ScannerInterface): else: suffixes.append(re.escape(bytes([entry]))) else: - # If we've fininshed one of the strings at this point, remember it for later + # If we've finished one of the strings at this point, remember it for later finished = True if len(suffixes) == 1: diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 85e961b24..ae4a7d55e 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -1,14 +1,14 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import logging import struct from typing import Any, Dict, List, Optional -from volatility3.framework import interfaces, constants, exceptions +from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.layers import physical, segmented, resources +from volatility3.framework.layers import physical, resources, segmented from volatility3.framework.symbols import native vollog = logging.getLogger(__name__) @@ -87,13 +87,13 @@ class VmwareLayer(segmented.SegmentedLayer): offset = offset + name_len + 2 + (index * index_len), layer_name = self._meta_layer)) data_len = flags & 0x3f - + if data_len in [62, 63]: # Handle special data sizes that indicate a longer data stream data_len = 4 if version == 0 else 8 # Read the size of the data data_size = self._context.object(self._choose_type(data_len), - layer_name = self._meta_layer, - offset = offset + 2 + name_len + (indices_len * index_len)) + layer_name = self._meta_layer, + offset = offset + 2 + name_len + (indices_len * index_len)) # Skip two bytes of padding (as it seems?) # Read the actual data data = self._context.object("vmware!bytes", @@ -113,9 +113,9 @@ class VmwareLayer(segmented.SegmentedLayer): if tags[("regionsCount", ())][1] == 0: raise VmwareFormatException(self.name, "VMware VMEM is not split into regions") for region in range(tags[("regionsCount", ())][1]): - offset = tags[("regionPPN", (region, ))][1] * self._page_size - mapped_offset = tags[("regionPageNum", (region, ))][1] * self._page_size - length = tags[("regionSize", (region, ))][1] * self._page_size + offset = tags[("regionPPN", (region,))][1] * self._page_size + mapped_offset = tags[("regionPageNum", (region,))][1] * self._page_size + length = tags[("regionSize", (region,))][1] * self._page_size self._segments.append((offset, mapped_offset, length, length)) @property @@ -153,23 +153,19 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): current_layer_name) vmss_success = False - try: + with contextlib.suppress(IOError): _ = resources.ResourceAccessor().open(vmss).read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmss_success = True - except IOError: - pass vmsn_success = False if not vmss_success: - try: + with contextlib.suppress(IOError): _ = resources.ResourceAccessor().open(vmsn).read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmsn context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmsn_success = True - except IOError: - pass vollog.log(constants.LOGLEVEL_VVVV, f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})") diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 6ee24407f..eedd22bb2 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -6,10 +6,10 @@ import collections import collections.abc import logging import struct -from typing import Any, ClassVar, Dict, List, Iterable, Optional, Tuple, Type, Union as TUnion, overload +from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload -from volatility3.framework import interfaces, constants -from volatility3.framework.objects import templates, utility +from volatility3.framework import constants, interfaces +from volatility3.framework.objects import templates vollog = logging.getLogger(__name__) @@ -136,12 +136,15 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): if k not in ["context", "data_format", "object_info", "type_name"]: kwargs[k] = v kwargs['new_value'] = self.__new_value - return (self._context, self._vol.maps[-2]['type_name'], self._vol.maps[-3], self._data_format), kwargs + return (self._context, self._vol.maps[-3]['type_name'], self._vol.maps[-2], self._data_format), kwargs @classmethod def _unmarshall(cls, context: interfaces.context.ContextInterface, data_format: DataFormatInfo, object_info: interfaces.objects.ObjectInformation) -> TUnion[int, float, bool, bytes, str]: - data = context.layers.read(object_info.layer_name, object_info.offset, data_format.length) + # Don't try to lookup a 0 length data format, incase it's at an invalid offset. Length 0 means b'' + data = b'' + if data_format.length > 0: + data = context.layers.read(object_info.layer_name, object_info.offset, data_format.length) return convert_data_to_value(data, cls._struct_type, data_format) class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): @@ -203,7 +206,7 @@ class Bytes(PrimitiveObject, bytes): length: int = 1, **kwargs) -> 'Bytes': """Creates the appropriate class and returns it so that the native type - is inherritted. + is inherited. The only reason the kwargs is added, is so that the inheriting types can override __init__ without needing to @@ -599,6 +602,14 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): return 0 raise IndexError(f"Member not present in array template: {child}") + @classmethod + def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: + """Returns the template of the child member.""" + if 'subtype' in template.vol and child == 'subtype': + return template.vol.subtype + raise IndexError(f"Member not present in array template: {child}") + + @overload def __getitem__(self, i: int) -> interfaces.objects.Template: ... @@ -701,7 +712,7 @@ class AggregateType(interfaces.objects.ObjectInterface): tmp_list[member] = (relative_offset, new_child) # If there's trouble with mutability, consider making update_vol return a clone with the changes # (there will be a few other places that will be necessary) and/or making these part of the - # permanent dictionaries rather than the non-clonable ones + # permanent dictionaries rather than the non-cloneable ones template.update_vol(members = tmp_list) @classmethod @@ -712,6 +723,15 @@ class AggregateType(interfaces.objects.ObjectInterface): raise IndexError(f"Member not present in template: {child}") return retlist[0] + @classmethod + def child_template(cls, template: interfaces.objects.Template, child: str) -> interfaces.objects.Template: + """Returns the template of a child to its parent.""" + retlist = template.vol.members.get(child, None) + if retlist is None: + raise IndexError(f"Member not present in template: {child}") + return retlist[1] + + @classmethod def has_member(cls, template: interfaces.objects.Template, member_name: str) -> bool: """Returns whether the object would contain a member called diff --git a/volatility3/framework/objects/templates.py b/volatility3/framework/objects/templates.py index b544d117f..e8b523373 100644 --- a/volatility3/framework/objects/templates.py +++ b/volatility3/framework/objects/templates.py @@ -48,6 +48,12 @@ class ObjectTemplate(interfaces.objects.Template): plateProxy`)""" return self.vol.object_class.VolTemplateProxy.relative_child_offset(self, child) + def child_template(self, child: str) -> interfaces.objects.Template: + """Returns the template of a child of the templated object (see + :class:`~volatility3.framework.interfaces.objects.ObjectInterface.VolTem + plateProxy`)""" + return self.vol.object_class.VolTemplateProxy.child_template(self, child) + def replace_child(self, old_child: interfaces.objects.Template, new_child: interfaces.objects.Template) -> None: """Replaces `old_child` for `new_child` in the templated object's child list (see :class:`~volatility3.framework.interfaces.objects.ObjectInterf @@ -63,7 +69,7 @@ class ObjectTemplate(interfaces.objects.Template): object_info: interfaces.objects.ObjectInformation) -> interfaces.objects.ObjectInterface: """Constructs the object. - Returns: an object adhereing to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface` + Returns: an object adhering to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface` """ arguments: Dict[str, Any] = {} for arg in self.vol: @@ -99,6 +105,7 @@ class ReferenceTemplate(interfaces.objects.Template): size: ClassVar[Any] = property(_unresolved) replace_child: ClassVar[Any] = _unresolved relative_child_offset: ClassVar[Any] = _unresolved + child_template: ClassVar[Any] = _unresolved has_member: ClassVar[Any] = _unresolved def __call__(self, context: interfaces.context.ContextInterface, object_info: interfaces.objects.ObjectInformation): diff --git a/volatility3/framework/plugins/frameworkinfo.py b/volatility3/framework/plugins/frameworkinfo.py index b7c887d5c..63ba24d09 100644 --- a/volatility3/framework/plugins/frameworkinfo.py +++ b/volatility3/framework/plugins/frameworkinfo.py @@ -1,3 +1,7 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from typing import List from volatility3 import framework diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 575f25426..efffa9b87 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -1,17 +1,16 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import base64 import json import logging import os import pathlib import zipfile -from typing import List, Type, Any, Generator +from typing import Generator, List from volatility3 import schemas, symbols -from volatility3.framework import interfaces, renderers, constants -from volatility3.framework.automagic import mac, linux, symbol_cache +from volatility3.framework import constants, interfaces, renderers +from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import resources @@ -23,7 +22,7 @@ class IsfInfo(plugins.PluginInterface): """Determines information about the currently available ISF files, or a specific one""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -39,6 +38,13 @@ class IsfInfo(plugins.PluginInterface): requirements.BooleanRequirement(name = 'validate', description = 'Validate against schema if possible', default = False, + optional = True), + requirements.VersionRequirement(name = 'SQLiteCache', + component = symbol_cache.SqliteCache, + version = (1, 0, 0)), + requirements.BooleanRequirement(name = 'live', + description = 'Traverse all files, rather than use the cache', + default = False, optional = True) ] @@ -62,14 +68,6 @@ class IsfInfo(plugins.PluginInterface): if filename.endswith(extension): yield pathlib.Path(base_name).as_uri() - def _get_banner(self, clazz: Type[symbol_cache.SymbolBannerCache], data: Any) -> str: - """Gets a banner from an ISF file""" - banner_symbol = data.get('symbols', {}).get(clazz.symbol_name, {}).get('constant_data', - renderers.NotAvailableValue()) - if not isinstance(banner_symbol, interfaces.renderers.BaseAbsentValue): - banner_symbol = str(base64.b64decode(banner_symbol), encoding = 'latin-1') - return banner_symbol - def _generator(self): if self.config.get('isf', None) is not None: file_list = [self.config['isf']] @@ -98,33 +96,54 @@ class IsfInfo(plugins.PluginInterface): def check_valid(data): return "Unknown" - # Process the filtered list - for entry in filtered_list: - num_types = num_enums = num_bases = num_symbols = 0 - windows_info = linux_banner = mac_banner = renderers.NotAvailableValue() - valid = "Unknown" - with resources.ResourceAccessor().open(url = entry) as fp: - try: - data = json.load(fp) - num_symbols = len(data.get('symbols', [])) - num_types = len(data.get('user_types', [])) - num_enums = len(data.get('enums', [])) - num_bases = len(data.get('base_types', [])) + if self.config['live']: + # Process the filtered list + for entry in filtered_list: + num_types = num_enums = num_bases = num_symbols = 0 + valid = "Unknown" + with resources.ResourceAccessor().open(url = entry) as fp: + try: + data = json.load(fp) + num_symbols = len(data.get('symbols', [])) + num_types = len(data.get('user_types', [])) + num_enums = len(data.get('enums', [])) + num_bases = len(data.get('base_types', [])) - linux_banner = self._get_banner(linux.LinuxBannerCache, data) - mac_banner = self._get_banner(mac.MacBannerCache, data) - if not linux_banner and not mac_banner: - windows_info = os.path.splitext(os.path.basename(entry))[0] - valid = check_valid(data) - except (UnicodeDecodeError, json.decoder.JSONDecodeError): - vollog.warning(f"Invalid ISF: {entry}") - yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, windows_info, linux_banner, - mac_banner)) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + identifier_cache = symbol_cache.SqliteCache(identifiers_path) + identifier = identifier_cache.get_identifier(location = entry) + if identifier: + identifier = identifier.decode('utf-8', errors = 'replace') + else: + identifier = renderers.NotAvailableValue() + valid = check_valid(data) + except (UnicodeDecodeError, json.decoder.JSONDecodeError): + vollog.warning(f"Invalid ISF: {entry}") + yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) + else: + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + cache = symbol_cache.SqliteCache(identifiers_path) + valid = 'Unknown' + for identifier, location in cache.get_identifier_dictionary().items(): + num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) + if identifier: + json_hash = cache.get_hash(location) + if json_hash and json_hash in schemas.cached_validations: + valid = 'True (cached)' + if self.config['validate']: + # Even if we're not live, if we've been explicitly asked to validate, then do-so + with resources.ResourceAccessor().open(url = location) as fp: + try: + data = json.load(fp) + valid = check_valid(data) + except (UnicodeDecodeError, json.decoder.JSONDecodeError): + vollog.warning(f"Invalid ISF: {location}") + + yield (0, (location, valid, num_bases, num_types, num_symbols, num_enums, str(identifier))) # Try to open the file, load it as JSON, read the data from it def run(self): return renderers.TreeGrid([("URI", str), ("Valid", str), ("Number of base_types", int), ("Number of types", int), ("Number of symbols", int), - ("Number of enums", int), ("Windows info", str), ("Linux banner", str), - ("Mac banner", str)], self._generator()) + ("Number of enums", int), ("Identifying information", str)], self._generator()) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index b2a02116e..0068ec224 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -110,9 +110,9 @@ class LayerWriter(plugins.PluginInterface): def _generate_layers(self): """List layer names from this run""" for name in self.context.layers: - yield (0, (name, )) + yield (0, (name, self.context.layers[name].__class__.__name__)) def run(self): if self.config['list']: - return renderers.TreeGrid([("Layer name", str)], self._generate_layers()) + return renderers.TreeGrid([("Layer name", str), ('Layer type', str)], self._generate_layers()) return renderers.TreeGrid([("Status", str)], self._generator()) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 613469eed..9bc1a067d 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -44,7 +44,7 @@ class Check_creds(interfaces.plugins.PluginInterface): cred_addr = task.cred.dereference().vol.offset - if not cred_addr in creds: + if cred_addr not in creds: creds[cred_addr] = [] creds[cred_addr].append(task.pid) diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 729a0bec6..6ec5fd354 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -3,11 +3,11 @@ # """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" +import contextlib import logging from typing import List -from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers, constants +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.renderers import format_hints @@ -40,11 +40,9 @@ class Check_syscall(plugins.PluginInterface): symbol_list = [] for sn in vmlinux.symbols: - try: + with contextlib.suppress(exceptions.SymbolError): # When requesting the symbol from the module, a full resolve is performed symbol_list.append((vmlinux.get_symbol(sn).address, sn)) - except exceptions.SymbolError: - pass sorted_symbols = sorted(symbol_list) sym_address = 0 @@ -80,7 +78,7 @@ class Check_syscall(plugins.PluginInterface): def _get_table_info_disassembly(self, ptr_sz, vmlinux): """Find the size of the system call table by disassembling functions - that immediately reference it in their first isntruction This is in the + that immediately reference it in their first instruction This is in the form 'cmp reg,NR_syscalls'.""" table_size = 0 @@ -152,7 +150,7 @@ class Check_syscall(plugins.PluginInterface): except exceptions.SymbolError: ia32_symbol = None - if ia32_symbol != None: + if ia32_symbol is not None: ia32_info = self._get_table_info(vmlinux, "ia32_sys_call_table", ptr_sz) tables.append(("32bit", ia32_info)) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index a729c1695..8f4540766 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -4,9 +4,9 @@ import logging from abc import ABC, abstractmethod from enum import Enum -from typing import List, Iterator, Tuple, Generator +from typing import Generator, Iterator, List, Tuple -from volatility3.framework import renderers, interfaces, constants, contexts, class_subclasses +from volatility3.framework import class_subclasses, constants, contexts, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -15,39 +15,39 @@ vollog = logging.getLogger(__name__) class DescStateEnum(Enum): - desc_miss = -1 # ID mismatch (pseudo state) - desc_reserved = 0x0 # reserved, in use by writer - desc_committed = 0x1 # committed by writer, could get reopened - desc_finalized = 0x2 # committed, no further modification allowed - desc_reusable = 0x3 # free, not yet used by any writer + desc_miss = -1 # ID mismatch (pseudo state) + desc_reserved = 0x0 # reserved, in use by writer + desc_committed = 0x1 # committed by writer, could get reopened + desc_finalized = 0x2 # committed, no further modification allowed + desc_reusable = 0x3 # free, not yet used by any writer class ABCKmsg(ABC): """Kernel log buffer reader""" LEVELS = ( - "emerg", # system is unusable - "alert", # action must be taken immediately - "crit", # critical conditions - "err", # error conditions - "warn", # warning conditions - "notice", # normal but significant condition - "info", # informational - "debug", # debug-level messages + "emerg", # system is unusable + "alert", # action must be taken immediately + "crit", # critical conditions + "err", # error conditions + "warn", # warning conditions + "notice", # normal but significant condition + "info", # informational + "debug", # debug-level messages ) FACILITIES = ( - "kern", # kernel messages - "user", # random user-level messages - "mail", # mail system - "daemon", # system daemons - "auth", # security/authorization messages - "syslog", # messages generated internally by syslogd - "lpr", # line printer subsystem - "news", # network news subsystem - "uucp", # UUCP subsystem - "cron", # clock daemon + "kern", # kernel messages + "user", # random user-level messages + "mail", # mail system + "daemon", # system daemons + "auth", # security/authorization messages + "syslog", # messages generated internally by syslogd + "lpr", # line printer subsystem + "news", # network news subsystem + "uucp", # UUCP subsystem + "cron", # clock daemon "authpriv", # security/authorization messages (private) - "ftp" # FTP daemon + "ftp" # FTP daemon ) def __init__( @@ -247,12 +247,20 @@ class KmsgFiveTen(ABCKmsg): The data block ring 'text_data_ring' contains the records' text strings. A pointer to the high level structure is kept in the prb pointer which is initialized to a static ringbuffer. + + .. code-block:: c + static struct printk_ringbuffer *prb = &printk_rb_static; + In SMP systems with more than 64 CPUs this ringbuffer size is dynamically allocated according the number of CPUs based on the value of CONFIG_LOG_CPU_MAX_BUF_SHIFT. The prb pointer is updated consequently to this dynamic ringbuffer in setup_log_buf(). + + .. code-block:: c + prb = &printk_rb_dynamic; + Behind scenes, log_buf is still used as external buffer. When the static printk_ringbuffer struct is initialized, _DEFINE_PRINTKRB sets text_data_ring.data pointer to the address in log_buf which points to @@ -262,12 +270,14 @@ class KmsgFiveTen(ABCKmsg): buffer via the prb_init function. In that case, the original external static buffer in __log_buf and printk_rb_static are unused. - ... + + .. code-block:: c + new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN); prb_init(&printk_rb_dynamic, new_log_buf, ...); log_buf = new_log_buf; prb = &printk_rb_dynamic; - ... + See printk.c and printk_ringbuffer.c in kernel/printk/ folder for more details. """ diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py new file mode 100644 index 000000000..551d128ad --- /dev/null +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -0,0 +1,219 @@ +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from collections import namedtuple +from typing import Tuple, List, Iterable, Union + +from volatility3.framework import renderers, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + +MountInfoData = namedtuple("MountInfoData", ("mnt_id", "parent_id", "st_dev", "mnt_root_path", "path_root", + "mnt_opts", "fields", "mnt_type", "devname", "sb_opts")) + +class MountInfo(plugins.PluginInterface): + """Lists mount points on processes mount namespaces""" + + _required_framework_version = (2, 2, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name="kernel", description="Linux kernel", + architectures=["Intel32", "Intel64"]), + requirements.PluginRequirement(name="pslist", + plugin=pslist.PsList, version=(2, 0, 0)), + requirements.ListRequirement(name="pids", + description="Filter on specific process IDs.", + element_type=int, + optional=True), + requirements.ListRequirement(name="mntns", + description="Filter results by mount namespace. " + "Otherwise, all of them are shown.", + element_type=int, + optional=True), + requirements.BooleanRequirement(name="mount-format", + description="Shows a brief summary of the mount points information " + "with similar output format to the older /proc/[pid]/mounts or the " + "user-land command 'mount -l'.", + optional=True, + default=False), + ] + + @classmethod + def _do_get_path(cls, mnt, fs_root) -> Union[None, str]: + """It mimics the Linux kernel prepend_path function.""" + vfsmnt = mnt.mnt + dentry = vfsmnt.get_mnt_root() + + path_reversed = [] + while dentry != fs_root.dentry or vfsmnt.vol.offset != fs_root.mnt: + if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): + parent = mnt.get_mnt_parent().dereference() + # Escaped? + if dentry != vfsmnt.get_mnt_root(): + return None + + # Global root? + if mnt.vol.offset != parent.vol.offset: + dentry = mnt.get_mnt_mountpoint() + mnt = parent + vfsmnt = mnt.mnt + continue + + return None + + parent = dentry.d_parent + dname = dentry.d_name.name_as_str() + path_reversed.append(dname.strip("/")) + dentry = parent + + path = "/" + "/".join(reversed(path_reversed)) + return path + + @classmethod + def get_mountinfo(cls, mnt, task) -> Union[None, Tuple[int, int, str, str, str, List[str], + List[str], str, str, List[str]]]: + """Extract various information about a mount point. + It mimics the Linux kernel show_mountinfo function. + """ + mnt_root = mnt.get_mnt_root() + if not mnt_root: + return None + + path_root = cls._do_get_path(mnt, task.fs.root) + if path_root is None: + return None + + mnt_root_path = mnt_root.path() + superblock = mnt.get_mnt_sb() + + mnt_id: int = mnt.mnt_id + parent_id: int = mnt.mnt_parent.mnt_id + + st_dev = f"{superblock.major}:{superblock.minor}" + + mnt_opts: List[str] = [] + mnt_opts.append(mnt.get_flags_access()) + mnt_opts.extend(mnt.get_flags_opts()) + + # Tagged fields + fields: List[str] = [] + if mnt.is_shared(): + fields.append(f"shared:{mnt.mnt_group_id}") + + if mnt.is_slave(): + master = mnt.mnt_master.mnt_group_id + fields.append(f"master:{master}") + dominating_id = mnt.get_dominating_id(task.fs.root) + if dominating_id and dominating_id != master: + fields.append(f"propagate_from:{dominating_id}") + + if mnt.is_unbindable(): + fields.append("unbindable") + + mnt_type = superblock.get_type() + + devname = mnt.get_devname() + if not devname: + devname = "none" + + sb_opts: List[str] = [] + sb_opts.append(superblock.get_flags_access()) + sb_opts.extend(superblock.get_flags_opts()) + + return MountInfoData(mnt_id, parent_id, st_dev, mnt_root_path, path_root, mnt_opts, fields, + mnt_type, devname, sb_opts) + + def _get_tasks_mountpoints(self, tasks: Iterable[interfaces.objects.ObjectInterface], per_namespace: bool): + seen_namespaces = set() + for task in tasks: + if not (task and task.fs and task.fs.root and task.nsproxy and task.nsproxy.mnt_ns): + # This task doesn't have all the information required + continue + + mnt_namespace = task.nsproxy.mnt_ns + mnt_ns_id = mnt_namespace.get_inode() + + if per_namespace: + if mnt_ns_id in seen_namespaces: + continue + else: + seen_namespaces.add(mnt_ns_id) + + for mount in mnt_namespace.get_mount_points(): + yield task, mount, mnt_ns_id + + def _generator( + self, + tasks: Iterable[interfaces.objects.ObjectInterface], + mnt_ns_ids: List[int], + mount_format: bool, + per_namespace: bool) -> Iterable[Tuple[int, Tuple]]: + + for task, mnt, mnt_ns_id in self._get_tasks_mountpoints(tasks, per_namespace): + if mnt_ns_ids and mnt_ns_id not in mnt_ns_ids: + continue + + mnt_info = self.get_mountinfo(mnt, task) + if mnt_info is None: + continue + + if mount_format: + all_opts = set() + all_opts.update(mnt_info.mnt_opts) + all_opts.update(mnt_info.sb_opts) + all_opts_str = ",".join(all_opts) + + extra_fields_values = [mnt_info.devname, mnt_info.path_root, mnt_info.mnt_type, all_opts_str] + else: + mnt_opts_str = ",".join(mnt_info.mnt_opts) + fields_str = " ".join(mnt_info.fields) + sb_opts_str = ",".join(mnt_info.sb_opts) + + extra_fields_values = [mnt_info.mnt_id, mnt_info.parent_id, mnt_info.st_dev, mnt_info.mnt_root_path, + mnt_info.path_root, mnt_opts_str, fields_str, mnt_info.mnt_type, + mnt_info.devname, sb_opts_str] + + fields_values = [mnt_ns_id] + if not per_namespace: + fields_values.append(task.pid) + fields_values.extend(extra_fields_values) + + yield (0, fields_values) + + def run(self): + pids = self.config.get('pids') + mount_ns_ids = self.config.get('mntns') + mount_format = self.config.get('mount-format') + + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks(self.context, self.config['kernel'], filter_func=pid_filter) + + columns = [("MNT_NS_ID", int)] + # The PID column does not make sense when a PID filter is not specified. In that case, the default behavior is + # to displays the mountpoints per namespace. + if pids: + columns.append(("PID", int)) + per_namespace = False + else: + per_namespace = True + + if self.config.get('mount-format'): + extra_columns = [("DEVNAME", str), ("PATH", str), ("FSTYPE", str), ("MNT_OPTS", str)] + else: + # /proc/[pid]/mountinfo output format + extra_columns = [("MOUNT ID", int), ("PARENT_ID", int), ("MAJOR:MINOR", str), ("ROOT", str), + ("MOUNT_POINT", str), ("MOUNT_OPTIONS", str), ("FIELDS", str), ("FSTYPE", str), + ("MOUNT_SRC", str), ("SB_OPTIONS", str)] + + columns.extend(extra_columns) + + return renderers.TreeGrid(columns, self._generator(tasks, mount_ns_ids, mount_format, per_namespace)) diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py new file mode 100644 index 000000000..d8b844ca4 --- /dev/null +++ b/volatility3/framework/plugins/linux/psaux.py @@ -0,0 +1,111 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from typing import Optional + +from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist + + +class PsAux(plugins.PluginInterface): + """ Lists processes with their command line arguments """ + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + description = 'Filter on specific process IDs', + element_type = int, + optional = True) + ] + + def _get_command_line_args(self, task: interfaces.objects.ObjectInterface, + name: str) -> Optional[str]: + """ + Reads the command line arguments of a process + These are stored on the userland stack + Kernel threads re-use the process data structure, but do not have a valid 'mm' pointer + + Parameters: + task: task_struct object of the process + name: string name of the process (from task.comm) + """ + + # kernel threads never have an mm as they do not have userland mappings + try: + mm = task.mm + except exceptions.InvalidAddressException: + mm = None + + if mm: + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + return renderers.UnreadableValue() + + proc_layer = self.context.layers[proc_layer_name] + + # read argv from userland + start = task.mm.arg_start + + # get the size of the arguments with sanity checking + size_to_read = task.mm.arg_end - task.mm.arg_start + if not (0 < size_to_read <= 4096): + return renderers.UnreadableValue() + + # attempt to read it all as partial values are invalid and misleading + try: + argv = proc_layer.read(start, size_to_read) + except exceptions.InvalidAddressException: + return renderers.UnreadableValue() + + # the arguments are null byte terminated, replace the nulls with spaces + s = argv.decode().split('\x00') + args = " ".join(s) + else: + # kernel thread + # [ ] mimics ps on a live system + # also helps identify malware masquerading as a kernel thread, which is fairly common + args = "[" + name + "]" + + # remove trailing space, if present + if len(args) > 1 and args[-1] == " ": + args = args[:-1] + + return args + + def _generator(self, tasks): + """ Generates a listing of processes along with command line arguments """ + + # walk the process list and report the arguments + for task in tasks: + pid = task.pid + + try: + ppid = task.parent.pid + except exceptions.InvalidAddressException: + ppid = 0 + + name = utility.array_to_string(task.comm) + + args = self._get_command_line_args(task, name) + + yield (0, (pid, ppid, name, args)) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + + return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str), ("ARGS", str)], + self._generator( + pslist.PsList.list_tasks(self.context, + self.config['kernel'], + filter_func = filter_func))) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 5672bb56e..f9a1fe2a0 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -1,11 +1,12 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Callable, Iterable, List, Any +from typing import Callable, Iterable, List, Any, Tuple from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints class PsList(interfaces.plugins.PluginInterface): @@ -13,7 +14,7 @@ class PsList(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -23,7 +24,15 @@ class PsList(interfaces.plugins.PluginInterface): requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, - optional = True) + optional = True), + requirements.BooleanRequirement(name="threads", + description="Include user threads", + optional=True, + default=False), + requirements.BooleanRequirement(name="decorate_comm", + description="Show `user threads` comm in curly brackets, and `kernel threads` comm in square brackets", + optional=True, + default=False), ] @classmethod @@ -48,31 +57,76 @@ class PsList(interfaces.plugins.PluginInterface): else: return lambda _: False - def _generator(self): + def _get_task_fields( + self, + task: interfaces.objects.ObjectInterface, + decorate_comm: bool = False) -> Tuple[int, int, int, str]: + """Extract the fields needed for the final output + + Args: + task: A task object from where to get the fields. + decorate_comm: If True, it decorates the comm string of + - User threads: in curly brackets, + - Kernel threads: in square brackets + Defaults to False. + Returns: + A tuple with the fields to show in the plugin output. + """ + pid = task.tgid + tid = task.pid + ppid = task.parent.tgid if task.parent else 0 + name = utility.array_to_string(task.comm) + if decorate_comm: + if task.is_kernel_thread: + name = f"[{name}]" + elif task.is_user_thread: + name = f"{{{name}}}" + + task_fields = (format_hints.Hex(task.vol.offset), pid, tid, ppid, name) + return task_fields + + def _generator( + self, + pid_filter: Callable[[Any], bool], + include_threads: bool = False, + decorate_comm: bool = False): + """Generates the tasks list. + + Args: + pid_filter: A function which takes a process object and returns True if the process should be ignored/filtered + include_threads: If True, the output will also show the user threads + If False, only the thread group leaders will be shown + Defaults to False. + decorate_comm: If True, it decorates the comm string of + - User threads: in curly brackets, + - Kernel threads: in square brackets + Defaults to False. + Yields: + Each rows + """ for task in self.list_tasks(self.context, self.config['kernel'], - filter_func = self.create_pid_filter(self.config.get('pid', None))): - pid = task.pid - ppid = 0 - if task.parent: - ppid = task.parent.pid - name = utility.array_to_string(task.comm) - yield (0, (pid, ppid, name)) + pid_filter, + include_threads): + row = self._get_task_fields(task, decorate_comm) + yield (0, row) @classmethod def list_tasks( cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, - filter_func: Callable[[int], bool] = lambda _: False) -> Iterable[interfaces.objects.ObjectInterface]: + filter_func: Callable[[int], bool] = lambda _: False, + include_threads: bool = False) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer. Args: context: The context to retrieve required elements (layers, symbol tables) from vmlinux_module_name: The name of the kernel module on which to operate - + filter_func: A function which takes a process object and returns True if the process should be ignored/filtered + include_threads: If True, it will also return user threads. Yields: - Process objects + Task objects """ vmlinux = context.modules[vmlinux_module_name] @@ -80,8 +134,19 @@ class PsList(interfaces.plugins.PluginInterface): # Note that the init_task itself is not yielded, since "ps" also never shows it. for task in init_task.tasks: - if not filter_func(task): - yield task + if filter_func(task): + continue + + yield task + + if include_threads: + yield from task.get_threads() def run(self): - return renderers.TreeGrid([("PID", int), ("PPID", int), ("COMM", str)], self._generator()) + pids = self.config.get('pid') + include_threads = self.config.get('threads') + decorate_comm = self.config.get('decorate_comm') + filter_func = self.create_pid_filter(pids) + + columns = [("OFFSET (V)", format_hints.Hex), ("PID", int), ("TID", int), ("PPID", int), ("COMM", str)] + return renderers.TreeGrid(columns, self._generator(filter_func, include_threads, decorate_comm)) diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 3b95a344c..3ad5f3e19 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -1,8 +1,7 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework.objects import utility from volatility3.plugins.linux import pslist @@ -12,44 +11,74 @@ class PsTree(pslist.PsList): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._processes = {} + self._tasks = {} self._levels = {} self._children = {} - def find_level(self, pid): - """Finds how deep the pid is in the processes list.""" - seen = set([]) - seen.add(pid) - level = 0 - proc = self._processes.get(pid, None) - while proc is not None and proc.parent != 0 and proc.parent.pid not in seen: - ppid = int(proc.parent.pid) + def find_level(self, pid: int) -> None: + """Finds how deep the PID is in the tasks hierarchy. - child_list = self._children.get(ppid, set([])) + Args: + pid: PID to find the level in the hierarchy + """ + seen = set([pid]) + level = 0 + proc = self._tasks.get(pid) + while proc and proc.parent and proc.parent.pid not in seen: + if proc.is_thread_group_leader: + parent_pid = proc.parent.pid + else: + parent_pid = proc.tgid + + child_list = self._children.setdefault(parent_pid, set()) child_list.add(proc.pid) - self._children[ppid] = child_list - proc = self._processes.get(ppid, None) + + proc = self._tasks.get(parent_pid) level += 1 + self._levels[pid] = level - def _generator(self): - """Generates the.""" + def _generator( + self, + pid_filter, + include_threads: bool = False, + decorate_com: bool = False): + """Generates the tasks hierarchy tree. + + Args: + pid_filter: A function which takes a process object and returns True if the process should be ignored/filtered + include_threads: If True, the output will also show the user threads + If False, only the thread group leaders will be shown + Defaults to False. + decorate_comm: If True, it decorates the comm string of + - User threads: in curly brackets, + - Kernel threads: in square brackets + Defaults to False. + Yields: + Each rows + """ vmlinux = self.context.modules[self.config['kernel']] - for proc in self.list_tasks(self.context, vmlinux.name): - self._processes[proc.pid] = proc + for proc in self.list_tasks(self.context, + vmlinux.name, + filter_func=pid_filter, + include_threads=include_threads): + self._tasks[proc.pid] = proc # Build the child/level maps - for pid in self._processes: + for pid in self._tasks: self.find_level(pid) def yield_processes(pid): - proc = self._processes[pid] - row = (proc.pid, proc.parent.pid, utility.array_to_string(proc.comm)) + task = self._tasks[pid] - yield (self._levels[pid] - 1, row) - for child_pid in self._children.get(pid, []): + row = self._get_task_fields(task, decorate_com) + + tid = task.pid + yield (self._levels[tid] - 1, row) + + for child_pid in sorted(self._children.get(tid, [])): yield from yield_processes(child_pid) - for pid in self._levels: - if self._levels[pid] == 1: + for pid, level in self._levels.items(): + if level == 1: yield from yield_processes(pid) diff --git a/volatility3/framework/plugins/mac/__init__.py b/volatility3/framework/plugins/mac/__init__.py index e69de29bb..ef6762bee 100644 --- a/volatility3/framework/plugins/mac/__init__.py +++ b/volatility3/framework/plugins/mac/__init__.py @@ -0,0 +1,8 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +"""All core mac plugins. + +These modules should only be imported from volatility3.plugins NOT +volatility3.framework.plugins +""" diff --git a/volatility3/framework/plugins/mac/ifconfig.py b/volatility3/framework/plugins/mac/ifconfig.py index c366a19f0..330c13f07 100644 --- a/volatility3/framework/plugins/mac/ifconfig.py +++ b/volatility3/framework/plugins/mac/ifconfig.py @@ -9,7 +9,7 @@ from volatility3.framework.symbols import mac class Ifconfig(plugins.PluginInterface): - """Lists loaded kernel modules""" + """Lists network interface information for all devices""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/mac/kauth_listeners.py b/volatility3/framework/plugins/mac/kauth_listeners.py index 7002d88e2..fba6a8e0a 100644 --- a/volatility3/framework/plugins/mac/kauth_listeners.py +++ b/volatility3/framework/plugins/mac/kauth_listeners.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # diff --git a/volatility3/framework/plugins/mac/kauth_scopes.py b/volatility3/framework/plugins/mac/kauth_scopes.py index 910de35fd..f1a2ad345 100644 --- a/volatility3/framework/plugins/mac/kauth_scopes.py +++ b/volatility3/framework/plugins/mac/kauth_scopes.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 16fa51fa4..4a82d81cd 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # @@ -74,7 +74,7 @@ class Kevents(interfaces.plugins.PluginInterface): @classmethod def _walk_klist_array(cls, kernel, fdp, array_pointer_member, array_size_member): """ - Convience wrapper for walking an array of lists of kernel events + Convenience wrapper for walking an array of lists of kernel events Handles invalid address references """ try: diff --git a/volatility3/framework/plugins/mac/list_files.py b/volatility3/framework/plugins/mac/list_files.py index 19f28b18f..8bae986b7 100644 --- a/volatility3/framework/plugins/mac/list_files.py +++ b/volatility3/framework/plugins/mac/list_files.py @@ -72,7 +72,7 @@ class List_Files(plugins.PluginInterface): key = vnode.vol.offset added = False - if not key in loop_vnodes: + if key not in loop_vnodes: # We can't do anything with a no-name vnode v_name = cls._vnode_name(vnode) if v_name is None: @@ -108,7 +108,7 @@ class List_Files(plugins.PluginInterface): added = True parent = cls._get_parent(context, vnode) - while parent and not parent in loop_vnodes: + while parent and parent not in loop_vnodes: if not cls._walk_vnode(context, parent, loop_vnodes): break diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index 398559446..ba3ab83c8 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -12,7 +12,7 @@ from volatility3.framework.symbols import mac class Mount(plugins.PluginInterface): """A module containing a collection of plugins that produce data typically - foundin Mac's mount command""" + found in Mac's mount command""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/mac/vfsevents.py b/volatility3/framework/plugins/mac/vfsevents.py index 38f4172ce..bc5668495 100644 --- a/volatility3/framework/plugins/mac/vfsevents.py +++ b/volatility3/framework/plugins/mac/vfsevents.py @@ -1,4 +1,4 @@ -# This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 6bf592504..c1d29062d 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -12,7 +12,7 @@ import traceback from typing import Generator, Iterable, List, Optional, Tuple, Type from volatility3 import framework -from volatility3.framework import renderers, automagic, interfaces, plugins, exceptions +from volatility3.framework import automagic, exceptions, interfaces, plugins, renderers from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -74,10 +74,6 @@ class Timeliner(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.StringRequirement(name = 'plugins', - description = "Comma separated list of plugins to run", - optional = True, - default = None), requirements.BooleanRequirement( name = 'record-config', description = "Whether to record the state of all the plugins once complete", @@ -105,14 +101,23 @@ class Timeliner(interfaces.plugins.PluginInterface): return [sortable(timestamp) for timestamp in data[2:]] - def _generator(self, runable_plugins: List[TimeLinerInterface]) -> Optional[Iterable[Tuple[int, Tuple]]]: + def _generator(self, runnable_plugins: List[TimeLinerInterface]) -> Optional[Iterable[Tuple[int, Tuple]]]: """Takes a timeline, sorts it and output the data from each relevant row from each plugin.""" # Generate the results for each plugin data = [] - for plugin in runable_plugins: + + # Open the bodyfile now, so we can start outputting to it immediately + if self.config.get('create-bodyfile', True): + file_data = self.open("volatility.body") + fp = io.TextIOWrapper(file_data, write_through = True) + else: + file_data = None + fp = None + + for plugin in runnable_plugins: plugin_name = plugin.__class__.__name__ - self._progress_callback((runable_plugins.index(plugin) * 100) // len(runable_plugins), + self._progress_callback((runnable_plugins.index(plugin) * 100) // len(runnable_plugins), f"Running plugin {plugin_name}...") try: vollog.log(logging.INFO, f"Running {plugin_name}") @@ -130,27 +135,31 @@ class Timeliner(interfaces.plugins.PluginInterface): times.get(TimeLinerType.ACCESSED, renderers.NotApplicableValue()), times.get(TimeLinerType.CHANGED, renderers.NotApplicableValue()) ])) - except Exception: - vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}") - vollog.log(logging.DEBUG, traceback.format_exc()) - for data_item in sorted(data, key = self._sort_function): - yield data_item - # Write out a body file if necessary - if self.config.get('create-bodyfile', True): - with self.open("volatility.body") as file_data: - with io.TextIOWrapper(file_data, write_through = True) as fp: - for (plugin_name, item) in self.timeline: + # Write each entry because the body file doesn't need to be sorted + if fp: times = self.timeline[(plugin_name, item)] # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime if self._any_time_present(times): - fp.write("|{} - {}||||||{}|{}|{}|{}\n".format( + fp.write("|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format( plugin_name, self._sanitize_body_format(item), self._text_format(times.get(TimeLinerType.ACCESSED, "")), self._text_format(times.get(TimeLinerType.MODIFIED, "")), self._text_format(times.get(TimeLinerType.CHANGED, "")), self._text_format(times.get(TimeLinerType.CREATED, "")))) + except Exception: + vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}") + vollog.log(logging.DEBUG, traceback.format_exc()) + + for data_item in sorted(data, key = self._sort_function): + yield data_item + + # Write out a body file if necessary + if self.config.get('create-bodyfile', True): + if fp: + fp.close() + file_data.close() def _sanitize_body_format(self, value): return value.replace("|", "_") @@ -164,7 +173,7 @@ class Timeliner(interfaces.plugins.PluginInterface): def _text_format(self, value): """Formats a value as text, in case it is an AbsentValue""" if isinstance(value, interfaces.renderers.BaseAbsentValue): - return "" + return "0" if isinstance(value, datetime.datetime): return int(value.timestamp()) return value @@ -202,7 +211,7 @@ class Timeliner(interfaces.plugins.PluginInterface): if isinstance(plugin, TimeLinerInterface): if not len(filter_list) or any( - [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]): + [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]): plugins_to_run.append(plugin) except exceptions.UnsatisfiedException as excp: # Remove the failed plugin from the list and continue diff --git a/volatility3/framework/plugins/windows/bigpools.py b/volatility3/framework/plugins/windows/bigpools.py index c81125f07..9e120446f 100644 --- a/volatility3/framework/plugins/windows/bigpools.py +++ b/volatility3/framework/plugins/windows/bigpools.py @@ -21,7 +21,7 @@ class BigPools(interfaces.plugins.PluginInterface): """List big page pools.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -32,7 +32,11 @@ class BigPools(interfaces.plugins.PluginInterface): requirements.StringRequirement(name = 'tags', description = "Comma separated list of pool tags to filter pools returned", optional = True, - default = None) + default = None), + requirements.BooleanRequirement(name = 'show-free', + description = 'Show freed regions (otherwise only show allocations in use)', + default = False, + optional = True) ] @classmethod @@ -40,7 +44,8 @@ class BigPools(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - tags: Optional[list] = None): + tags: Optional[list] = None, + show_free: bool = False): """Returns the big page pool objects from the kernel PoolBigPageTable array. Args: @@ -97,7 +102,7 @@ class BigPools(interfaces.plugins.PluginInterface): for big_pool in big_pools: if big_pool.is_valid(): - if tags is None or big_pool.get_key() in tags: + if (tags is None or big_pool.get_key() in tags) and (show_free or not big_pool.is_free()): yield big_pool def _generator(self) -> Iterator[Tuple[int, Tuple[int, str]]]: # , str, int]]]: @@ -110,13 +115,19 @@ class BigPools(interfaces.plugins.PluginInterface): for big_pool in self.list_big_pools(context = self.context, layer_name = kernel.layer_name, symbol_table = kernel.symbol_table_name, - tags = tags): + tags = tags, + show_free = self.config.get("show-free")): num_bytes = big_pool.get_number_of_bytes() if not isinstance(num_bytes, interfaces.renderers.BaseAbsentValue): num_bytes = format_hints.Hex(num_bytes) - yield (0, (format_hints.Hex(big_pool.Va), big_pool.get_key(), big_pool.get_pool_type(), num_bytes)) + if big_pool.is_free(): + status = "Free" + else: + status = "Allocated" + + yield (0, (format_hints.Hex(big_pool.Va), big_pool.get_key(), big_pool.get_pool_type(), num_bytes, status)) def run(self): return renderers.TreeGrid([ @@ -124,4 +135,5 @@ class BigPools(interfaces.plugins.PluginInterface): ('Tag', str), ('PoolType', str), ('NumberOfBytes', format_hints.Hex), + ('Status', str), ], self._generator()) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index ddfa856b9..f77c6257b 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -46,7 +46,7 @@ class Cachedump(interfaces.plugins.PluginInterface): rc4 = ARC4.new(rc4key) data = rc4.encrypt(edata) # lgtm [py/weak-cryptographic-algorithm] else: - # based on Based on code from http://lab.mediaservice.net/code/cachedump.rb + # Based on code from http://lab.mediaservice.net/code/cachedump.rb aes = AES.new(nlkm[16:32], AES.MODE_CBC, ch) data = b"" for i in range(0, len(edata), 16): diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 352dba448..2195671df 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -11,7 +11,6 @@ from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.plugins.windows import ssdt -from volatility3.plugins.windows import svcscan vollog = logging.getLogger(__name__) @@ -28,7 +27,6 @@ class Callbacks(interfaces.plugins.PluginInterface): requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'ssdt', plugin = ssdt.SSDT, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'svcscan', plugin = svcscan.SvcScan, version = (1, 0, 0)) ] @staticmethod @@ -111,30 +109,19 @@ class Callbacks(interfaces.plugins.PluginInterface): yield symbol_name, callback.Callback, None @classmethod - def list_registry_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - callback_table_name: str) -> Iterable[Tuple[str, int, None]]: - """Lists all registry callbacks. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - symbol_table: The name of the table containing the kernel symbols - callback_table_name: The nae of the table containing the callback symbols - - Yields: - A name, location and optional detail string + def _list_registry_callbacks_legacy(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, + callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + """ + Lists all registry callbacks from the old format via the CmpCallBackVector. """ kvo = context.layers[layer_name].config['kernel_virtual_offset'] ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) full_type_name = callback_table_name + constants.BANG + "_EX_CALLBACK_ROUTINE_BLOCK" - try: - symbol_offset = ntkrnlmp.get_symbol("CmpCallBackVector").address - symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address - except exceptions.SymbolError: - vollog.debug("Cannot find CmpCallBackVector or CmpCallBackCount") - return + symbol_offset = ntkrnlmp.get_symbol("CmpCallBackVector").address + symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address + callback_count = ntkrnlmp.object(object_type = "unsigned int", offset = symbol_count_offset) @@ -155,6 +142,62 @@ class Callbacks(interfaces.plugins.PluginInterface): if callback.Function != 0: yield "CmRegisterCallback", callback.Function, None + @classmethod + def _list_registry_callbacks_new(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, + callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + """ + Lists all registry callbacks via the CallbackListHead. + """ + + kvo = context.layers[layer_name].config['kernel_virtual_offset'] + ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + full_type_name = callback_table_name + constants.BANG + "_CM_CALLBACK_ENTRY" + + symbol_offset = ntkrnlmp.get_symbol("CallbackListHead").address + symbol_count_offset = ntkrnlmp.get_symbol("CmpCallBackCount").address + + callback_count = ntkrnlmp.object(object_type = "unsigned int", offset = symbol_count_offset) + + if callback_count == 0: + return + + callback_list = ntkrnlmp.object(object_type = "_LIST_ENTRY", offset = symbol_offset) + for callback in callback_list.to_list(full_type_name, "Link"): + yield "CmRegisterCallbackEx", callback.Function, f"Altitude: {callback.Altitude.String}" + + @classmethod + def list_registry_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, + callback_table_name: str) -> Iterable[Tuple[str, int, None]]: + """Lists all registry callbacks. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + callback_table_name: The nae of the table containing the callback symbols + + Yields: + A name, location and optional detail string + """ + + kvo = context.layers[layer_name].config['kernel_virtual_offset'] + ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) + + if ntkrnlmp.has_symbol("CmpCallBackVector") and ntkrnlmp.has_symbol("CmpCallBackCount"): + yield from cls._list_registry_callbacks_legacy(context, layer_name, symbol_table, callback_table_name) + elif ntkrnlmp.has_symbol("CallbackListHead") and ntkrnlmp.has_symbol("CmpCallBackCount"): + yield from cls._list_registry_callbacks_new(context, layer_name, symbol_table, callback_table_name) + else: + symbols_to_check = ["CmpCallBackVector", "CmpCallBackCount", "CallbackListHead"] + vollog.debug("Failed to get registry callbacks!") + for symbol_name in symbols_to_check: + symbol_status = "does not exist" + if ntkrnlmp.has_symbol(symbol_name): + symbol_status = "exists" + vollog.debug(f"symbol {symbol_name} {symbol_status}.") + + return + @classmethod def list_bugcheck_reason_callbacks(cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, callback_table_name: str) -> Iterable[Tuple[str, int, str]]: diff --git a/volatility3/framework/plugins/windows/devicetree.py b/volatility3/framework/plugins/windows/devicetree.py new file mode 100644 index 000000000..1b6b55cb7 --- /dev/null +++ b/volatility3/framework/plugins/windows/devicetree.py @@ -0,0 +1,167 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterator, List, Tuple + +from volatility3.framework import constants, renderers, exceptions, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import driverscan + +DEVICE_CODES = { + 0x00000027 : "FILE_DEVICE_8042_PORT", + 0x00000032 : "FILE_DEVICE_ACPI", + 0x00000029 : "FILE_DEVICE_BATTERY", + 0x00000001 : "FILE_DEVICE_BEEP", + 0x0000002a : "FILE_DEVICE_BUS_EXTENDER", + 0x00000002 : "FILE_DEVICE_CD_ROM", + 0x00000003 : "FILE_DEVICE_CD_ROM_FILE_SYSTEM", + 0x00000030 : "FILE_DEVICE_CHANGER", + 0x00000004 : "FILE_DEVICE_CONTROLLER", + 0x00000005 : "FILE_DEVICE_DATALINK", + 0x00000006 : "FILE_DEVICE_DFS", + 0x00000035 : "FILE_DEVICE_DFS_FILE_SYSTEM", + 0x00000036 : "FILE_DEVICE_DFS_VOLUME", + 0x00000007 : "FILE_DEVICE_DISK", + 0x00000008 : "FILE_DEVICE_DISK_FILE_SYSTEM", + 0x00000033 : "FILE_DEVICE_DVD", + 0x00000009 : "FILE_DEVICE_FILE_SYSTEM", + 0x0000003a : "FILE_DEVICE_FIPS", + 0x00000034 : "FILE_DEVICE_FULLSCREEN_VIDEO", + 0x0000000a : "FILE_DEVICE_INPORT_PORT", + 0x0000000b : "FILE_DEVICE_KEYBOARD", + 0x0000002f : "FILE_DEVICE_KS", + 0x00000039 : "FILE_DEVICE_KSEC", + 0x0000000c : "FILE_DEVICE_MAILSLOT", + 0x0000002d : "FILE_DEVICE_MASS_STORAGE", + 0x0000000d : "FILE_DEVICE_MIDI_IN", + 0x0000000e : "FILE_DEVICE_MIDI_OUT", + 0x0000002b : "FILE_DEVICE_MODEM", + 0x0000000f : "FILE_DEVICE_MOUSE", + 0x00000010 : "FILE_DEVICE_MULTI_UNC_PROVIDER", + 0x00000011 : "FILE_DEVICE_NAMED_PIPE", + 0x00000012 : "FILE_DEVICE_NETWORK", + 0x00000013 : "FILE_DEVICE_NETWORK_BROWSER", + 0x00000014 : "FILE_DEVICE_NETWORK_FILE_SYSTEM", + 0x00000028 : "FILE_DEVICE_NETWORK_REDIRECTOR", + 0x00000015 : "FILE_DEVICE_NULL", + 0x00000016 : "FILE_DEVICE_PARALLEL_PORT", + 0x00000017 : "FILE_DEVICE_PHYSICAL_NETCARD", + 0x00000018 : "FILE_DEVICE_PRINTER", + 0x00000019 : "FILE_DEVICE_SCANNER", + 0x0000001c : "FILE_DEVICE_SCREEN", + 0x00000037 : "FILE_DEVICE_SERENUM", + 0x0000001a : "FILE_DEVICE_SERIAL_MOUSE_PORT", + 0x0000001b : "FILE_DEVICE_SERIAL_PORT", + 0x00000031 : "FILE_DEVICE_SMARTCARD", + 0x0000002e : "FILE_DEVICE_SMB", + 0x0000001d : "FILE_DEVICE_SOUND", + 0x0000001e : "FILE_DEVICE_STREAMS", + 0x0000001f : "FILE_DEVICE_TAPE", + 0x00000020 : "FILE_DEVICE_TAPE_FILE_SYSTEM", + 0x00000038 : "FILE_DEVICE_TERMSRV", + 0x00000021 : "FILE_DEVICE_TRANSPORT", + 0x00000022 : "FILE_DEVICE_UNKNOWN", + 0x0000002c : "FILE_DEVICE_VDM", + 0x00000023 : "FILE_DEVICE_VIDEO", + 0x00000024 : "FILE_DEVICE_VIRTUAL_DISK", + 0x00000025 : "FILE_DEVICE_WAVE_IN", + 0x00000026 : "FILE_DEVICE_WAVE_OUT", +} + +vollog = logging.getLogger(__name__) + +class DeviceTree(interfaces.plugins.PluginInterface): + """Listing tree based on drivers and attached devices in a particular windows memory image.""" + + _required_framework_version = (2, 0, 3) + _version = (1, 0, 1) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name = "kernel", description = "Windows kernel", + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = "driverscan", plugin = driverscan.DriverScan, version = (1, 0, 0)), + ] + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config["kernel"]] + + # Scan the Layer for drivers + for driver in driverscan.DriverScan.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): + try: + try: + driver_name = driver.get_driver_name() + except (ValueError, exceptions.InvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, + f"Failed to get Driver name : {driver.vol.offset:x}") + driver_name = renderers.UnparsableValue() + + yield (0, ( + format_hints.Hex(driver.vol.offset), + "DRV", + driver_name, + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue() + )) + + # Scan to get the device information of driver. + for device in driver.get_devices(): + try: + device_name = device.get_device_name() + except (ValueError, exceptions.InvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, + f"Failed to get Device name : {device.vol.offset:x}") + device_name = renderers.UnparsableValue() + + device_type = DEVICE_CODES.get(device.DeviceType, "UNKNOWN") + + yield (1, ( + format_hints.Hex(driver.vol.offset), + "DEV", + driver_name, + device_name, + renderers.NotApplicableValue(), + device_type + )) + + # Scan to get the attached devices information of device. + for level, attached_device in enumerate(device.get_attached_devices(), start=2): + try: + device_name = attached_device.get_device_name() + except (ValueError, exceptions.InvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, + f"Failed to get Attached Device Name: {attached_device.vol.offset:x}") + device_name = renderers.UnparsableValue() + + attached_device_driver_name = attached_device.DriverObject.DriverName.get_string() + attached_device_type = DEVICE_CODES.get(attached_device.DeviceType, "UNKNOWN") + + yield (level, ( + format_hints.Hex(driver.vol.offset), + "ATT", + driver_name, + device_name, + attached_device_driver_name, + attached_device_type + )) + + except(exceptions.InvalidAddressException): + vollog.log(constants.LOGLEVEL_VVVV, + f"Invalid address identified in drivers and devices: {driver.vol.offset:x}") + continue + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid([ + ("Offset", format_hints.Hex), + ("Type", str), + ("DriverName", str), + ("DeviceName", str), + ("DriverNameOfAttDevice", str), + ("DeviceType", str), + ], self._generator()) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index b24f2c0ca..cb7626dfa 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -1,18 +1,19 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import contextlib import datetime import logging import ntpath from typing import List, Optional, Type -from volatility3.framework import exceptions, renderers, interfaces, constants +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins import timeliner -from volatility3.plugins.windows import pslist, info +from volatility3.plugins.windows import info, pslist vollog = logging.getLogger(__name__) @@ -28,7 +29,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), requirements.ListRequirement(name = 'pid', @@ -65,7 +66,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): try: name = dll_entry.FullDllName.get_string() except exceptions.InvalidAddressException: - name = 'UnreadbleDLLName' + name = 'UnreadableDLLName' if layer_name is None: layer_name = dll_entry.vol.layer_name @@ -107,12 +108,10 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for entry in proc.load_order_modules(): BaseDllName = FullDllName = renderers.UnreadableValue() - try: + with contextlib.suppress(exceptions.InvalidAddressException): BaseDllName = entry.BaseDllName.get_string() # We assume that if the BaseDllName points to an invalid buffer, so will FullDllName FullDllName = entry.FullDllName.get_string() - except exceptions.InvalidAddressException: - pass if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 58166ee7f..b20efa45d 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -5,6 +5,7 @@ import logging import ntpath from typing import List, Tuple, Type, Optional, Generator + from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -32,8 +33,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.IntRequirement(name = 'pid', description = "Process ID to include (all other processes are excluded)", optional = True), @@ -63,29 +65,28 @@ class DumpFiles(interfaces.plugins.PluginInterface): :return: result status """ filedata = open_method(desired_file_name) - try: - # Description of these variables: - # memoffset: offset in the specified layer where the page begins - # fileoffset: write to this offset in the destination file - # datasize: size of the page + # Description of these variables: + # memoffset: offset in the specified layer where the page begins + # fileoffset: write to this offset in the destination file + # datasize: size of the page - # track number of bytes written so we don't write empty files to disk - bytes_written = 0 + # track number of bytes written so we don't write empty files to disk + bytes_written = 0 + try: for memoffset, fileoffset, datasize in memory_object.get_available_pages(): data = layer.read(memoffset, datasize, pad = True) bytes_written += len(data) filedata.seek(fileoffset) filedata.write(data) - - if not bytes_written: - vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") - return None - else: - vollog.debug(f"Stored {filedata.preferred_filename}") - return filedata except exceptions.InvalidAddressException: vollog.debug(f"Unable to dump file at {file_object.vol.offset:#x}") return None + if not bytes_written: + vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") + return None + + vollog.debug(f"Stored {filedata.preferred_filename}") + return filedata @classmethod def process_file_object(cls, context: interfaces.context.ContextInterface, primary_layer_name: str, @@ -98,12 +99,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): :param open_method: class for constructing output files :param file_obj: the FILE_OBJECT """ - # Filtering by these types of devices prevents us from processing other types of devices that # use the "File" object type, such as \Device\Tcp and \Device\NamedPipe. if file_obj.DeviceObject.DeviceType not in [FILE_DEVICE_DISK, FILE_DEVICE_NETWORK_FILE_SYSTEM]: - vollog.log(constants.LOGLEVEL_VVV, - f"The file object at {file_obj.vol.offset:#x} is not a file on disk") + vollog.log(constants.LOGLEVEL_VVV, f"The file object at {file_obj.vol.offset:#x} is not a file on disk") return # Depending on the type of object (DataSection, ImageSection, SharedCacheMap) we may need to @@ -120,7 +119,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): # layer to read from, # file extension to apply, # ) - dump_parameters = [] + dump_parameters = list() # The DataSectionObject and ImageSectionObject caches are handled in basically the same way. # We carve these "pages" from the memory_layer. @@ -131,8 +130,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if control_area.is_valid(): dump_parameters.append((control_area, memory_layer, extension)) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") # The SharedCacheMap is handled differently than the caches above. # We carve these "pages" from the primary_layer. @@ -142,8 +140,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if shared_cache_map.is_valid(): dump_parameters.append((shared_cache_map, primary_layer, "vacb")) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] @@ -151,7 +148,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): memory_object.vol.offset, cache_name, ntpath.basename(obj_name), extension) - file_handle = DumpFiles.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) + file_handle = cls.dump_file_producer(file_obj, memory_object, open_method, layer, desired_file_name) file_output = "Error dumping file" if file_handle: @@ -185,8 +182,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): try: object_table = proc.ObjectTable except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") continue for entry in handles_plugin.handles(object_table): @@ -218,12 +214,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue - for result in self.process_file_object(self.context, kernel.layer_name, self.open, - file_obj): + for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - f"Cannot extract file from VAD at {vad.vol.offset:#x}") + vollog.log(constants.LOGLEVEL_VVV, f"Cannot extract file from VAD at {vad.vol.offset:#x}") elif offsets: # Now process any offsets explicitly requested by the user. @@ -234,10 +228,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not is_virtual: layer_name = self.context.layers[layer_name].config["memory_layer"] - file_obj = self.context.object( - kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", + file_obj = self.context.object(kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", layer_name = layer_name, - native_layer_name = kernel.layer_name, + native_layer_name = kernel.layer_name, offset = offset) for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) @@ -246,9 +239,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): def run(self): # a list of tuples (, ) where is the address and is True for virtual. - offsets = [] + offsets = list() # a list of processes matching the pid filter. all files for these process(es) will be dumped. - procs = [] + procs = list() kernel = self.context.modules[self.config['kernel']] if self.config.get("virtaddr", None) is not None: diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index 9791fa580..e9015280a 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -1,9 +1,10 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +import contextlib import logging from typing import List -from volatility3.framework import renderers, interfaces, objects, exceptions, constants +from volatility3.framework import constants, exceptions, interfaces, objects, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.plugins.windows import pslist @@ -23,7 +24,7 @@ class Envars(interfaces.plugins.PluginInterface): # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -61,13 +62,11 @@ class Envars(interfaces.plugins.PluginInterface): key = hive.get_key('CurrentControlSet\\Control\\Session Manager\\Environment') sys = True except KeyError: - try: + with contextlib.suppress(KeyError): key = hive.get_key('ControlSet001\\Control\\Session Manager\\Environment') sys = True - except KeyError: - pass if sys: - try: + with contextlib.suppress(KeyError): for node in key.get_values(): try: value_node_name = node.get_name() @@ -78,17 +77,13 @@ class Envars(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVV, "Error while parsing global environment variables keys (some keys might be excluded)") continue - except KeyError: - pass ## The user-specific variables - try: + with contextlib.suppress(KeyError): key = hive.get_key('Environment') ntuser = True - except KeyError: - pass if ntuser: - try: + with contextlib.suppress(KeyError): for node in key.get_values(): try: value_node_name = node.get_name() @@ -99,8 +94,6 @@ class Envars(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVV, "Error while parsing user environment variables keys (some keys might be excluded)") continue - except KeyError: - pass ## The volatile user variables try: diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index ab11d30d6..bdff88075 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -323,7 +323,7 @@ class Handles(interfaces.plugins.PluginInterface): obj_name = item.file_name_with_device() elif obj_type == "Process": item = entry.Body.cast("_EPROCESS") - obj_name = f"{utility.array_to_string(proc.ImageFileName)} Pid {item.UniqueProcessId}" + obj_name = f"{utility.array_to_string(item.ImageFileName)} Pid {item.UniqueProcessId}" elif obj_type == "Thread": item = entry.Body.cast("_ETHREAD") obj_name = f"Tid {item.Cid.UniqueThread} Pid {item.Cid.UniqueProcess}" diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index 4d0b25b5c..e9f8047e0 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -4,10 +4,10 @@ import binascii import hashlib import logging -from struct import unpack, pack -from typing import List, Tuple, Optional +from struct import pack, unpack +from typing import List, Optional, Tuple -from Crypto.Cipher import ARC4, DES, AES +from Crypto.Cipher import AES, ARC4, DES from Crypto.Hash import MD5 from volatility3.framework import interfaces, renderers @@ -28,7 +28,7 @@ class Hashdump(interfaces.plugins.PluginInterface): def get_requirements(cls): return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) ] @@ -63,7 +63,8 @@ class Hashdump(interfaces.plugins.PluginInterface): def get_hive_key(cls, hive: registry.RegistryHive, key: str): result = None try: - result = hive.get_key(key) + if hive: + result = hive.get_key(key) except KeyError: vollog.info( f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image") @@ -132,7 +133,7 @@ class Hashdump(interfaces.plugins.PluginInterface): rc4_key = md5.digest() rc4 = ARC4.new(rc4_key) - hbootkey = rc4.encrypt(sam_data[0x80:0xA0]) # lgtm [py/weak-cryptographic-algorithm] + hbootkey = rc4.encrypt(sam_data[0x80:0xA0]) # lgtm [py/weak-cryptographic-algorithm] return hbootkey elif revision == 3: # AES encrypted @@ -151,7 +152,7 @@ class Hashdump(interfaces.plugins.PluginInterface): des2 = DES.new(des_k2, DES.MODE_ECB) cipher = AES.new(hbootkey[:16], AES.MODE_CBC, salt) obfkey = cipher.decrypt(enc_hash) - return des1.decrypt(obfkey[:8]) + des2.decrypt(obfkey[8:16]) # lgtm [py/weak-cryptographic-algorithm] + return des1.decrypt(obfkey[:8]) + des2.decrypt(obfkey[8:16]) # lgtm [py/weak-cryptographic-algorithm] @classmethod def get_user_hashes(cls, user: registry.CM_KEY_NODE, samhive: registry.RegistryHive, @@ -229,9 +230,9 @@ class Hashdump(interfaces.plugins.PluginInterface): md5.update(hbootkey[:0x10] + pack(" Optional[bytes]: @@ -253,13 +254,9 @@ class Hashdump(interfaces.plugins.PluginInterface): # replaces the dump_hashes method in vol2 def _generator(self, syshive: registry.RegistryHive, samhive: registry.RegistryHive): if syshive is None: - vollog.debug("SYSTEM address is None: Did you use the correct profile?") - yield (0, (renderers.NotAvailableValue(), renderers.NotAvailableValue(), renderers.NotAvailableValue(), - renderers.NotAvailableValue())) + vollog.debug("SYSTEM address is None: No system hive found") if samhive is None: - vollog.debug("SAM address is None: Did you use the correct profile?") - yield (0, (renderers.NotAvailableValue(), renderers.NotAvailableValue(), renderers.NotAvailableValue(), - renderers.NotAvailableValue())) + vollog.debug("SAM address is None: No SAM hive found") bootkey = self.get_bootkey(syshive) hbootkey = self.get_hbootkey(samhive, bootkey) if hbootkey: diff --git a/volatility3/framework/plugins/windows/joblinks.py b/volatility3/framework/plugins/windows/joblinks.py new file mode 100644 index 000000000..40d09b9ea --- /dev/null +++ b/volatility3/framework/plugins/windows/joblinks.py @@ -0,0 +1,75 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterator, List, Tuple + +from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + + +class JobLinks(interfaces.plugins.PluginInterface): + """Print process job link information""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.BooleanRequirement(name = 'physical', + description = "Display physical offset instead of virtual", + default = False, + optional = True), + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)) + ] + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config['kernel']] + memory = self.context.layers[kernel.layer_name] + + for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, kernel.symbol_table_name): + try: + if not self.config['physical']: + offset = proc.vol.offset + else: + (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + + job = proc.Job.dereference() + + yield (0, (format_hints.Hex(offset), utility.array_to_string(proc.ImageFileName), proc.UniqueProcessId, + proc.InheritedFromUniqueProcessId, proc.get_session_id(), job.SessionId, proc.get_is_wow64(), + job.TotalProcesses, job.ActiveProcesses, job.TotalTerminatedProcesses, + renderers.NotApplicableValue(), "(Original Process)")) + + for entry in job.ProcessListHead.to_list(proc.vol.type_name, "JobLinks"): + if not self.config['physical']: + offset = entry.vol.offset + else: + (_, _, offset, _, _) = list(memory.mapping(offset = entry.vol.offset, length = 0))[0] + + yield (1, (format_hints.Hex(offset), utility.array_to_string(entry.ImageFileName), + entry.UniqueProcessId, entry.InheritedFromUniqueProcessId, entry.get_session_id(), 0, + entry.get_is_wow64(), 0, 0, 0, "Yes", + entry.get_peb().ProcessParameters.ImagePathName.get_string())) + + except (exceptions.InvalidAddressException): + continue + + def run(self) -> renderers.TreeGrid: + offsettype = "(V)" if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT) else "(P)" + + return renderers.TreeGrid([(f"Offset{offsettype}", format_hints.Hex), ("Name", str), + ("PID", int), ("PPID", int), ("Sess", int), ("JobSess", int), ("Wow64", bool), + ("Total", int), ("Active", int), ("Term", int), ("JobLink", str), ("Process", str)], + self._generator()) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py new file mode 100644 index 000000000..ba8d049a6 --- /dev/null +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -0,0 +1,97 @@ +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, vadinfo + +class LdrModules(interfaces.plugins.PluginInterface): + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), + requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True), + ] + + def _generator(self, procs): + + pe_table_name = intermed.IntermediateSymbolTable.create(self.context, + self.config_path, + "windows", + "pe", + class_types = pe.class_types) + + def filter_function(x: interfaces.objects.ObjectInterface) -> bool: + try: + return not (x.get_private_memory() == 0 and x.ControlArea) + except AttributeError: + return False + + filter_func = filter_function + + for proc in procs: + proc_layer_name = proc.add_process_layer() + + # Build dictionaries from different module lists, where the DllBase address is the key and value is the module object + load_order_mod = dict((mod.DllBase, mod) + for mod in proc.load_order_modules()) + init_order_mod = dict((mod.DllBase, mod) + for mod in proc.init_order_modules()) + mem_order_mod = dict((mod.DllBase, mod) + for mod in proc.mem_order_modules()) + + # Build dictionary of mapped files, where the VAD start address is the key and value is the file name of the mapped file + mapped_files = {} + for vad in vadinfo.VadInfo.list_vads(proc, filter_func = filter_func): + dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset = vad.get_start(), + layer_name = proc_layer_name) + try: + # Filter out VADs that do not start with a MZ header + if dos_header.e_magic != 0x5A4D: + continue + except exceptions.PagedInvalidAddressException: + continue + + mapped_files[vad.get_start()] = vad.get_file_name() + + for base in mapped_files.keys(): + # Does the base address exist in the PEB DLL lists? + load_mod = load_order_mod.get(base, None) + init_mod = init_order_mod.get(base, None) + mem_mod = mem_order_mod.get(base, None) + + yield (0, [int(proc.UniqueProcessId), + str(proc.ImageFileName.cast("string", + max_length = proc.ImageFileName.vol.count, + errors = 'replace')), + format_hints.Hex(base), + load_mod is not None, + init_mod is not None, + mem_mod is not None, + mapped_files[base]]) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] + + return renderers.TreeGrid([("Pid", int), + ("Process", str), + ("Base", format_hints.Hex), + ("InLoad", bool), + ("InInit", bool), + ("InMem", bool), + ("MappedPath", str)], + self._generator( + pslist.PsList.list_processes(context = self.context, + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, + filter_func = filter_func))) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index bfd29a254..9b5fab3f5 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -17,7 +17,7 @@ vollog = logging.getLogger(__name__) class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) @classmethod def get_requirements(cls): @@ -53,10 +53,10 @@ class Malfind(interfaces.plugins.PluginInterface): """ CHUNK_SIZE = 0x1000 - all_zero_page = "\x00" * CHUNK_SIZE + all_zero_page = b"\x00" * CHUNK_SIZE offset = 0 - vad_length = vad.get_end() - vad.get_start() + vad_length = vad.get_size() while offset < vad_length: next_addr = vad.get_start() + offset diff --git a/volatility3/framework/plugins/windows/mbrscan.py b/volatility3/framework/plugins/windows/mbrscan.py new file mode 100644 index 000000000..d064e7d29 --- /dev/null +++ b/volatility3/framework/plugins/windows/mbrscan.py @@ -0,0 +1,200 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import hashlib + +from typing import Iterator, List, Tuple + +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mbr + +vollog = logging.getLogger(__name__) + +class MBRScan(interfaces.plugins.PluginInterface): + """Scans for and parses potential Master Boot Records (MBRs)""" + + _required_framework_version = (2, 0, 1) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls)-> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.BooleanRequirement(name = 'full', + description ="It analyzes and provides all the information in the partition entry and bootcode hexdump. (It returns a lot of information, so we recommend you render it in CSV.)", + default = False, + optional = True) + ] + + @classmethod + def get_hash(cls, data:bytes) -> str: + return hashlib.md5(data).hexdigest() + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config['kernel']] + physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) + + # Decide of Memory Dump Architecture + layer = self.context.layers[physical_layer_name] + architecture = "intel" if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) else "intel64" + + # Read in the Symbol File + symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, + config_path = self.config_path, + sub_path = "windows", + filename = "mbr", + class_types = { + 'PARTITION_TABLE': mbr.PARTITION_TABLE, + 'PARTITION_ENTRY': mbr.PARTITION_ENTRY + }) + + partition_table_object = symbol_table + constants.BANG + "PARTITION_TABLE" + + # Define Signature and Data Length + mbr_signature = b"\x55\xAA" + mbr_length = 0x200 + bootcode_length = 0x1B8 + + # Scan the Layer for Raw Master Boot Record (MBR) and parse the fields + for offset, _value in layer.scan(context = self.context, scanner = scanners.MultiStringScanner(patterns = [mbr_signature])): + try: + mbr_start_offset = offset - (mbr_length - len(mbr_signature)) + partition_table = self.context.object(partition_table_object, offset = mbr_start_offset, layer_name = layer.name) + + # Extract only BootCode + full_mbr = layer.read(mbr_start_offset, mbr_length, pad = True) + bootcode = full_mbr[:bootcode_length] + + all_zeros = None + + if bootcode: + all_zeros = bootcode.count(b"\x00") == len(bootcode) + + if not all_zeros: + + partition_entries = [ + partition_table.FirstEntry, partition_table.SecondEntry, + partition_table.ThirdEntry, partition_table.FourthEntry + ] + + if not self.config.get("full", True): + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + interfaces.renderers.Disassembly(bootcode, 0, architecture) + )) + else: + yield (0, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + renderers.NotApplicableValue(), + interfaces.renderers.Disassembly(bootcode, 0, architecture), + format_hints.HexBytes(bootcode) + )) + + for partition_index, partition_entry_object in enumerate(partition_entries, start=1): + + if not self.config.get("full", True): + yield (1, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_index, + partition_entry_object.is_bootable(), + partition_entry_object.get_partition_type(), + format_hints.Hex(partition_entry_object.get_size_in_sectors()), + renderers.NotApplicableValue() + )) + else: + yield (1, ( + format_hints.Hex(offset), + partition_table.get_disk_signature(), + self.get_hash(bootcode), + self.get_hash(full_mbr), + partition_index, + partition_entry_object.is_bootable(), + format_hints.Hex(partition_entry_object.get_bootable_flag()), + partition_entry_object.get_partition_type(), + format_hints.Hex(partition_entry_object.PartitionType), + format_hints.Hex(partition_entry_object.get_starting_lba()), + partition_entry_object.get_starting_cylinder(), + partition_entry_object.get_starting_chs(), + partition_entry_object.get_starting_sector(), + partition_entry_object.get_ending_cylinder(), + partition_entry_object.get_ending_chs(), + partition_entry_object.get_ending_sector(), + format_hints.Hex(partition_entry_object.get_size_in_sectors()), + renderers.NotApplicableValue(), + renderers.NotApplicableValue() + )) + else: + vollog.log(constants.LOGLEVEL_VVVV, f"Not a valid MBR: Data all zeroed out : {format_hints.Hex(offset)}") + continue + + except exceptions.PagedInvalidAddressException as excp: + vollog.log(constants.LOGLEVEL_VVVV, f"Invalid address identified in guessed MBR: {hex(excp.invalid_address)}") + continue + + def run(self)-> renderers.TreeGrid: + if not self.config.get("full", True): + return renderers.TreeGrid([ + ("Potential MBR at Physical Offset", format_hints.Hex), + ("Disk Signature", str), + ("Bootcode MD5", str), + ("Full MBR MD5", str), + ("PartitionIndex", int), + ("Bootable", bool), + ("PartitionType", str), + ("SectorInSize", format_hints.Hex), + ("Disasm", interfaces.renderers.Disassembly) + ], self._generator()) + else: + return renderers.TreeGrid([ + ("Potential MBR at Physical Offset", format_hints.Hex), + ("Disk Signature", str), + ("Bootcode MD5", str), + ("Full MBR MD5", str), + ("PartitionIndex", int), + ("Bootable", bool), + ("BootFlag", format_hints.Hex), + ("PartitionType", str), + ("PartitionTypeRaw", format_hints.Hex), + ("StartingLBA", format_hints.Hex), + ("StartingCylinder", int), + ("StartingCHS", int), + ("StartingSector", int), + ("EndingCylinder", int), + ("EndingCHS", int), + ("EndingSector", int), + ("SectorInSize", format_hints.Hex), + ("Disasm", interfaces.renderers.Disassembly), + ("Bootcode", format_hints.HexBytes) + ], self._generator()) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py new file mode 100644 index 000000000..c96fd9522 --- /dev/null +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -0,0 +1,161 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import contextlib +import datetime +import logging + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import conversion, format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import mft +from volatility3.plugins import timeliner, yarascan + +vollog = logging.getLogger(__name__) + + +class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """Scans for MFT FILE objects present in a particular windows memory image.""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner, + version = (2, 0, 0)), + ] + + def _generator(self): + layer = self.context.layers[self.config['primary']] + + # Yara Rule to scan for MFT Header Signatures + rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'}) + + # Read in the Symbol File + symbol_table = intermed.IntermediateSymbolTable.create(context = self.context, + config_path = self.config_path, + sub_path = "windows", + filename = "mft", + class_types = { + 'FILE_NAME_ENTRY': mft.MFTFileName, + 'MFT_ENTRY': mft.MFTEntry + }) + + # get each of the individual Field Sets + mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" + header_object = symbol_table + constants.BANG + "ATTR_HEADER" + si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Scan the layer for Raw MFT records and parse the fields + for offset, _rule_name, _name, _value in layer.scan(context = self.context, + scanner = yarascan.YaraScanner(rules = rules)): + with contextlib.suppress(exceptions.PagedInvalidAddressException): + mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name) + # We will update this on each pass in the next loop and use it as the new offset. + attr_base_offset = mft_record.FirstAttrOffset + + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) + + # There is no field that has a count of Attributes + # Keep Attempting to read attributes until we get an invalid attr_header.AttrType + + while attr_header.AttrType.is_valid_choice: + vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") + + # Offset past the headers to the attribute data + attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type( + attribute_object).relative_child_offset("Attr_Data") + + # MFT Flags determine the file type or dir + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + mft_flag = mft_record.Flags.lookup() + except ValueError: + mft_flag = hex(mft_record.Flags) + + # Standard Information Attribute + if attr_header.AttrType.lookup() == 'STANDARD_INFORMATION': + attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name) + + yield 0, ( + format_hints.Hex(attr_data_offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + mft_flag, + renderers.NotApplicableValue(), + attr_header.AttrType.lookup(), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + renderers.NotApplicableValue(), + ) + + # File Name Attribute + if attr_header.AttrType.lookup() == 'FILE_NAME': + attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name) + file_name = attr_data.get_full_name() + + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + permissions = attr_data.Flags.lookup() + except ValueError: + permissions = hex(attr_data.Flags) + + yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(), + mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, + attr_header.AttrType.lookup(), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), file_name) + + # If there's no advancement the loop will never end, so break it now + if attr_header.Length == 0: + break + + # Update the base offset to point to the next attribute + attr_base_offset += attr_header.Length + # Get the next attribute + attr_header = self.context.object(header_object, + offset = offset + attr_base_offset, + layer_name = layer.name) + + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + + # Only Output FN Records + if row_data[6] == 'FILE_NAME': + filename = row_data[-1] + description = f"MFT FILE_NAME entry for {filename}" + yield (description, timeliner.TimeLinerType.CREATED, row_data[7]) + yield (description, timeliner.TimeLinerType.MODIFIED, row_data[8]) + yield (description, timeliner.TimeLinerType.CHANGED, row_data[9]) + yield (description, timeliner.TimeLinerType.ACCESSED, row_data[10]) + + def run(self): + return renderers.TreeGrid([ + ('Offset', format_hints.Hex), + ('Record Type', str), + ('Record Number', int), + ('Link Count', int), + ('MFT Type', str), + ('Permissions', str), + ('Attribute Type', str), + ('Created', datetime.datetime), + ('Modified', datetime.datetime), + ('Updated', datetime.datetime), + ('Accessed', datetime.datetime), + ('Filename', str), + ], self._generator()) diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index b661d71d7..e352c21fe 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -25,7 +25,7 @@ class ModScan(interfaces.plugins.PluginInterface): return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.VersionRequirement(name = 'poolerscanner', + requirements.VersionRequirement(name = 'poolscanner', component = poolscanner.PoolScanner, version = (1, 0, 0)), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 33b5a7fbc..16301bdd8 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -14,7 +14,7 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.framework.symbols.windows.extensions import network from volatility3.plugins import timeliner -from volatility3.plugins.windows import info, poolscanner +from volatility3.plugins.windows import info, poolscanner, verinfo vollog = logging.getLogger(__name__) @@ -34,6 +34,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): component = poolscanner.PoolScanner, version = (1, 0, 0)), requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), + requirements.VersionRequirement(name = 'verinfo', component = verinfo.VerInfo, version = (1, 0, 0)), requirements.BooleanRequirement( name = 'include-corrupt', description = @@ -141,49 +142,55 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # furthermore, it is easy to append new versions. if arch == "x86": version_dict = { - (6, 0, 6000): "netscan-vista-x86", - (6, 0, 6001): "netscan-vista-x86", - (6, 0, 6002): "netscan-vista-x86", - (6, 0, 6003): "netscan-vista-x86", - (6, 1, 7600): "netscan-win7-x86", - (6, 1, 7601): "netscan-win7-x86", - (6, 1, 8400): "netscan-win7-x86", - (6, 2, 9200): "netscan-win8-x86", - (6, 3, 9600): "netscan-win81-x86", - (10, 0, 10240): "netscan-win10-10240-x86", - (10, 0, 10586): "netscan-win10-10586-x86", - (10, 0, 14393): "netscan-win10-14393-x86", - (10, 0, 15063): "netscan-win10-15063-x86", - (10, 0, 16299): "netscan-win10-15063-x86", - (10, 0, 17134): "netscan-win10-17134-x86", - (10, 0, 17763): "netscan-win10-17134-x86", - (10, 0, 18362): "netscan-win10-17134-x86", - (10, 0, 18363): "netscan-win10-17134-x86" + (6, 0, 6000, 0): "netscan-vista-x86", + (6, 0, 6001, 0): "netscan-vista-x86", + (6, 0, 6002, 0): "netscan-vista-x86", + (6, 0, 6003, 0): "netscan-vista-x86", + (6, 1, 7600, 0): "netscan-win7-x86", + (6, 1, 7601, 0): "netscan-win7-x86", + (6, 1, 8400, 0): "netscan-win7-x86", + (6, 2, 9200, 0): "netscan-win8-x86", + (6, 3, 9600, 0): "netscan-win81-x86", + (10, 0, 10240, 0): "netscan-win10-10240-x86", + (10, 0, 10586, 0): "netscan-win10-10586-x86", + (10, 0, 14393, 0): "netscan-win10-14393-x86", + (10, 0, 15063, 0): "netscan-win10-15063-x86", + (10, 0, 16299, 0): "netscan-win10-15063-x86", + (10, 0, 17134, 0): "netscan-win10-17134-x86", + (10, 0, 17763, 0): "netscan-win10-17134-x86", + (10, 0, 18362, 0): "netscan-win10-17134-x86", + (10, 0, 18363, 0): "netscan-win10-17134-x86" } else: version_dict = { - (6, 0, 6000): "netscan-vista-x64", - (6, 0, 6001): "netscan-vista-sp12-x64", - (6, 0, 6002): "netscan-vista-sp12-x64", - (6, 0, 6003): "netscan-vista-sp12-x64", - (6, 1, 7600): "netscan-win7-x64", - (6, 1, 7601): "netscan-win7-x64", - (6, 1, 8400): "netscan-win7-x64", - (6, 2, 9200): "netscan-win8-x64", - (6, 3, 9600): "netscan-win81-x64", - (10, 0, 10240): "netscan-win10-x64", - (10, 0, 10586): "netscan-win10-x64", - (10, 0, 14393): "netscan-win10-x64", - (10, 0, 15063): "netscan-win10-15063-x64", - (10, 0, 16299): "netscan-win10-16299-x64", - (10, 0, 17134): "netscan-win10-17134-x64", - (10, 0, 17763): "netscan-win10-17763-x64", - (10, 0, 18362): "netscan-win10-18362-x64", - (10, 0, 18363): "netscan-win10-18363-x64", - (10, 0, 19041): "netscan-win10-19041-x64" + (6, 0, 6000, 0): "netscan-vista-x64", + (6, 0, 6001, 0): "netscan-vista-sp12-x64", + (6, 0, 6002, 0): "netscan-vista-sp12-x64", + (6, 0, 6003, 0): "netscan-vista-sp12-x64", + (6, 1, 7600, 0): "netscan-win7-x64", + (6, 1, 7601, 0): "netscan-win7-x64", + (6, 1, 8400, 0): "netscan-win7-x64", + (6, 2, 9200, 0): "netscan-win8-x64", + (6, 3, 9600, 0): "netscan-win81-x64", + (6, 3, 9600, 19935): "netscan-win81-19935-x64", + (10, 0, 10240, 0): "netscan-win10-x64", + (10, 0, 10586, 0): "netscan-win10-x64", + (10, 0, 14393, 0): "netscan-win10-x64", + (10, 0, 15063, 0): "netscan-win10-15063-x64", + (10, 0, 16299, 0): "netscan-win10-16299-x64", + (10, 0, 17134, 0): "netscan-win10-17134-x64", + (10, 0, 17763, 0): "netscan-win10-17763-x64", + (10, 0, 18362, 0): "netscan-win10-18362-x64", + (10, 0, 18363, 0): "netscan-win10-18363-x64", + (10, 0, 19041, 0): "netscan-win10-19041-x64" } - # special use case: Win10_18363 is not recognized by windows.info as 18363 + # we do not need to check for tcpip's specific FileVersion in every case + tcpip_mod_version = 0 # keep it 0 as a default + + # special use cases + + # Win10_18363 is not recognized by windows.info as 18363 # because all kernel file headers and debug structures report 18363 as # "10.0.18362.1198" with the last part being incremented. However, we can use # os_distinguisher to differentiate between 18362 and 18363 @@ -191,18 +198,36 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.debug("Detected 18363 data structures: working with 18363 symbol table.") vers_minor_version = 18363 + # we need to define additional version numbers (which are then found via tcpip.sys's FileVersion header) in case there is + # ambiguity _within_ an OS version. If such a version number (last number of the tuple) is defined for the current OS + # we need to inspect tcpip.sys's headers to see if we can grab the precise version + if [ (a,b,c,d) for a, b, c, d in version_dict if (a,b,c) == (nt_major_version, nt_minor_version, vers_minor_version) and d != 0]: + vollog.debug("Requiring further version inspection due to OS version by checking tcpip.sys's FileVersion header") + # the following is IntelLayer specific and might need to be adapted to other architectures. + physical_layer_name = context.layers[layer_name].config.get('memory_layer', None) + if physical_layer_name: + ver = verinfo.VerInfo.find_version_info(context, physical_layer_name, "tcpip.sys") + if ver: + tcpip_mod_version = ver[3] + vollog.debug("Determined tcpip.sys's FileVersion: {}".format(tcpip_mod_version)) + else: + vollog.debug("Could not determine tcpip.sys's FileVersion.") + else: + vollog.debug("Unable to retrieve physical memory layer, skipping FileVersion check.") + # when determining the symbol file we have to consider the following cases: # the determined version's symbol file is found by intermed.create -> proceed # the determined version's symbol file is not found by intermed -> intermed will throw an exc and abort # the determined version has no mapped symbol file -> if win10 use latest, otherwise throw exc # windows version cannot be determined -> throw exc - filename = version_dict.get((nt_major_version, nt_minor_version, vers_minor_version)) + + filename = version_dict.get((nt_major_version, nt_minor_version, vers_minor_version, tcpip_mod_version)) if not filename: # no match on filename means that we possibly have a version newer than those listed here. # try to grab the latest supported version of the current image NT version. If that symbol # version does not work, support has to be added manually. current_versions = [ - key for key in list(version_dict.keys()) if key[0] == nt_major_version and key[1] == nt_minor_version + (nt_maj, nt_min, vers_min, tcpip_ver) for nt_maj, nt_min, vers_min, tcpip_ver in version_dict if nt_maj == nt_major_version and nt_min == nt_minor_version and tcpip_ver <= tcpip_mod_version ] current_versions.sort() @@ -210,7 +235,9 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): latest_version = current_versions[-1] filename = version_dict.get(latest_version) + vollog.debug(f"Unable to find exact matching symbol file, going with latest: {filename}") + else: raise NotImplementedError("This version of Windows is not supported: {}.{} {}.{}!".format( nt_major_version, nt_minor_version, vers.MajorVersion, vers_minor_version)) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index e71683c10..93ac3af93 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -12,7 +12,7 @@ from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.windows import pdbutil from volatility3.framework.symbols.windows.extensions import network from volatility3.plugins import timeliner -from volatility3.plugins.windows import netscan, modules +from volatility3.plugins.windows import netscan, modules, info, verinfo vollog = logging.getLogger(__name__) @@ -31,6 +31,8 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement(name = 'netscan', component = netscan.NetScan, version = (1, 0, 0)), requirements.VersionRequirement(name = 'modules', component = modules.Modules, version = (1, 0, 0)), requirements.VersionRequirement(name = 'pdbutil', component = pdbutil.PDBUtility, version = (1, 0, 0)), + requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), + requirements.VersionRequirement(name = 'verinfo', component = verinfo.VerInfo, version = (1, 0, 0)), requirements.BooleanRequirement( name = 'include-corrupt', description = @@ -431,7 +433,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self.context, interfaces.configuration.path_join(self.config_path, 'tcpip'), kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) except exceptions.VolatilityException: - vollog.warning("Unable to locate symbols for the memory image's tcpip module") + vollog.error("Unable to locate symbols for the memory image's tcpip module") for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): diff --git a/volatility3/framework/plugins/windows/privileges.py b/volatility3/framework/plugins/windows/privileges.py index eaafbeae6..2d48a30f7 100644 --- a/volatility3/framework/plugins/windows/privileges.py +++ b/volatility3/framework/plugins/windows/privileges.py @@ -55,14 +55,14 @@ class Privs(interfaces.plugins.PluginInterface): try: process_token = task.Token.dereference().cast("_TOKEN") except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, 'Skeep invalid token.') + vollog.log(constants.LOGLEVEL_VVV, 'Skip invalid token.') continue for value, present, enabled, default in process_token.privileges(): # Skip privileges whose bit positions cannot be # translated to a privilege name if not self.privilege_info.get(int(value)): - vollog.log(constants.LOGLEVEL_VVV, f'Skeep invalid privilege ({value}).') + vollog.log(constants.LOGLEVEL_VVV, f'Skip invalid privilege ({value}).') continue name, desc = self.privilege_info.get(int(value)) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 237c0edcd..335624672 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -22,7 +22,7 @@ vollog = logging.getLogger(__name__) class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for processes present in a particular windows memory image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 3, 1) _version = (1, 1, 0) @classmethod @@ -85,7 +85,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): context: The context to retrieve required elements (layers, symbol tables) from layer_name: The name of the layer on which to operate symbol_table: The name of the table containing the kernel symbols - proc: the process object with phisical address + proc: the process object with physical address Returns: A process object on virtual address layer diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index a788f058f..30b5db695 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -3,17 +3,18 @@ # import codecs +import contextlib import datetime import json import logging import os -from typing import Any, List, Tuple, Generator +from typing import Any, Generator, List, Tuple -from volatility3.framework import exceptions, renderers, constants, interfaces +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers.physical import BufferDataLayer from volatility3.framework.layers.registry import RegistryHive -from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.plugins.windows.registry import hivelist @@ -38,7 +39,7 @@ class UserAssist(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + architectures = ["Intel32", "Intel64"]), requirements.IntRequirement(name = 'offset', description = "Hive Offset", default = None, optional = True), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) ] @@ -126,11 +127,9 @@ class UserAssist(interfaces.plugins.PluginInterface): hive_name = hive.hive.cast(kernel.symbol_table_name + constants.BANG + "_CMHIVE").get_name() if self._win7 is None: - try: + with contextlib.suppress(exceptions.SymbolError): self._win7 = self._win7_or_later() - except exceptions.SymbolError: # self._win7 will be None and only registry value rawdata will be output - pass self._determine_userassist_type() @@ -163,7 +162,6 @@ class UserAssist(interfaces.plugins.PluginInterface): # output any subkeys under Count for subkey in countkey.get_subkeys(): - subkey_name = subkey.get_name() result = (1, ( renderers.format_hints.Hex(hive.hive_offset), @@ -185,10 +183,8 @@ class UserAssist(interfaces.plugins.PluginInterface): for value in countkey.get_values(): value_name = value.get_name() - try: + with contextlib.suppress(UnicodeDecodeError): value_name = codecs.encode(value_name, "rot_13") - except UnicodeDecodeError: - pass if self._win7: guid = value_name.split("\\")[0] diff --git a/volatility3/framework/plugins/windows/sessions.py b/volatility3/framework/plugins/windows/sessions.py new file mode 100644 index 000000000..6745e95ec --- /dev/null +++ b/volatility3/framework/plugins/windows/sessions.py @@ -0,0 +1,103 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import datetime +import logging + +from volatility3.framework import renderers, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.plugins.windows import pslist +from volatility3.plugins import timeliner + +vollog = logging.getLogger(__name__) + + +class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """lists Processes with Session information extracted from Environmental Variables""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement(name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ListRequirement(name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True) + ] + + def _generator(self): + kernel = self.context.modules[self.config['kernel']] + filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + + # Collect all the values as we will want to group them later + sessions = {} + + for proc in pslist.PsList.list_processes(self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func = filter_func): + + session_id = proc.get_session_id() + + # Detect RDP, Console or set default value + session_type = renderers.NotAvailableValue() + + # Construct Username from Process Env + user_domain = '' + user_name = '' + + for var, val in proc.environment_variables(): + if var.lower() == 'username': + user_name = val + elif var.lower() == 'userdomain': + user_domain = val + if var.lower() == 'sessionname': + session_type = val + + # Concat Domain and User + full_user = f'{user_domain}/{user_name}' + if full_user == '/': + full_user = renderers.NotAvailableValue() + + # Collect all the values in to a row we can yield after sorting. + row = { + "session_id": session_id, + "process_id": proc.UniqueProcessId, + "process_name": utility.array_to_string(proc.ImageFileName), + "user_name": full_user, + "process_start": proc.get_create_time(), + "session_type": session_type + } + + # Add row to correct session so we can sort it later + if session_id in sessions: + sessions[session_id].append(row) + else: + sessions[session_id] = [row] + + # Group and yield each row + for rows in sessions.values(): + for row in rows: + yield 0, (row.get('session_id'), row.get('session_type'), row.get('process_id'), + row.get('process_name'), row.get('user_name'), row.get('process_start')) + + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + # Only add to timeline if we have the username + # Without the user context PSList output is identical + if isinstance(row_data[4], str): + description = f"Process: {row_data[2]} {row_data[3]} started by user {row_data[4]}" + yield (description, timeliner.TimeLinerType.CREATED, row_data[5]) + + def run(self): + + return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str), + ("User Name", str), ("Create Time", datetime.datetime)], self._generator()) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index cd4a5baec..f6f41864a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -9,7 +9,7 @@ # For a thorough walkthrough on how the R&D was performed to develop this plugin, # please see our blogpost here: # -# +# https://volatility-labs.blogspot.com/2021/10/memory-forensics-r-illustrated.html import io import logging @@ -41,7 +41,7 @@ vollog = logging.getLogger(__name__) class Skeleton_Key_Check(interfaces.plugins.PluginInterface): """ Looks for signs of Skeleton Key malware """ - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) @classmethod def get_requirements(cls): @@ -262,7 +262,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): if isinstance(filename, str) and filename.lower().endswith("cryptdll.dll"): base = vad.get_start() - return base, vad.get_end() - base + return base, vad.get_size() return None, None diff --git a/volatility3/framework/plugins/windows/ssdt.py b/volatility3/framework/plugins/windows/ssdt.py index 0d921535d..78fd72630 100644 --- a/volatility3/framework/plugins/windows/ssdt.py +++ b/volatility3/framework/plugins/windows/ssdt.py @@ -95,10 +95,10 @@ class SSDT(plugins.PluginInterface): if is_kernel_64: array_subtype = "long" - def kvo_calulator(func: int) -> int: + def kvo_calculator(func: int) -> int: return kvo + service_table_address + (func >> 4) - find_address = kvo_calulator + find_address = kvo_calculator else: array_subtype = "unsigned long" diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 9fa1458d1..d3997c8c8 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -33,7 +33,7 @@ winnt_protections = { class VadInfo(interfaces.plugins.PluginInterface): """Lists process memory ranges.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) _version = (2, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @@ -132,7 +132,7 @@ class VadInfo(interfaces.plugins.PluginInterface): vollog.debug("Unable to find the starting/ending VPN member") return None - if maxsize > 0 and (vad_end - vad_start) > maxsize: + if 0 < maxsize < vad.get_size(): vollog.debug(f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit") return None @@ -151,8 +151,9 @@ class VadInfo(interfaces.plugins.PluginInterface): file_handle = open_method(file_name) chunk_size = 1024 * 1024 * 10 offset = vad_start - while offset < vad_end: - to_read = min(chunk_size, vad_end - offset) + vad_size = vad.get_size() + while offset < vad_start + vad_size: + to_read = min(chunk_size, vad_start + vad_size - offset) data = proc_layer.read(offset, to_read, pad = True) if not data: break diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 06a87d003..b71e2f605 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -17,7 +17,7 @@ vollog = logging.getLogger(__name__) class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 4, 0) _version = (1, 0, 0) @classmethod @@ -82,9 +82,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """ vad_root = task.get_vad_root() for vad in vad_root.traverse(): - end = vad.get_end() - start = vad.get_start() - yield (start, end - start) + yield (vad.get_start(), vad.get_size()) def run(self): return renderers.TreeGrid([('Offset', format_hints.Hex), ('PID', int), ('Rule', str), ('Component', str), diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 762832e39..70d473d2e 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -202,3 +202,4 @@ class VerInfo(interfaces.plugins.PluginInterface): return renderers.TreeGrid([("PID", int), ("Process", str), ("Base", format_hints.Hex), ("Name", str), ("Major", int), ("Minor", int), ("Product", int), ("Build", int)], self._generator(procs, mods, session_layers)) + diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 94b3cba45..0ef55ff4b 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -3,7 +3,7 @@ # import logging -from typing import Iterable, Tuple, List, Dict, Any +from typing import Any, Dict, Iterable, List, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -15,8 +15,11 @@ vollog = logging.getLogger(__name__) try: import yara + + if tuple([int(x) for x in yara.__version__.split('.')]) < (3, 8): + raise ImportError except ImportError: - vollog.info("Python Yara module not found, plugin (and dependent plugins) not available") + vollog.info("Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available") raise @@ -40,7 +43,10 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) + + # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string + # or something that makes more sense @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -83,6 +89,8 @@ class YaraScan(plugins.PluginInterface): if config.get('wide', False): rule += " wide ascii" rules = yara.compile(sources = {'n': f'rule r1 {{strings: $a = {rule} condition: $a}}'}) + elif config.get('yara_source', None) is not None: + rules = yara.compile(source = config['yara_source']) elif config.get('yara_file', None) is not None: rules = yara.compile(file = resources.ResourceAccessor().open(config['yara_file'], "rb")) elif config.get('yara_compiled_file', None) is not None: diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index 996cf03a5..3ce49bbde 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import datetime import ipaddress import socket @@ -27,10 +27,8 @@ def unixtime_to_datetime(unixtime: int) -> Union[interfaces.renderers.BaseAbsent ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = renderers.UnparsableValue() if unixtime > 0: - try: + with contextlib.suppress(ValueError): ret = datetime.datetime.utcfromtimestamp(unixtime) - except ValueError: - pass return ret diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index c48fcc8a7..1fceb1bcc 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -11,13 +11,13 @@ import os import pathlib import zipfile from abc import ABCMeta -from typing import Any, Dict, Generator, Iterable, List, Optional, Type, Tuple, Mapping +from typing import Any, Dict, Generator, Iterable, List, Mapping, Optional, Tuple, Type from volatility3 import schemas, symbols from volatility3.framework import class_subclasses, constants, exceptions, interfaces, objects from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources -from volatility3.framework.symbols import native, metadata +from volatility3.framework.symbols import metadata, native vollog = logging.getLogger(__name__) @@ -113,6 +113,9 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): metadata = json_object.get('metadata', None) + if not metadata: + raise exceptions.SymbolSpaceError(f"Invalid ISF file attempted to be parsed: {isf_url}") + # Determine the delegate or throw an exception self._delegate = self._closest_version(metadata.get('format', "0.0.0"), self._versions)(context, config_path, name, json_object, native_types, @@ -199,8 +202,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): pass # Finally try looking in zip files - zip_path = os.path.join(path, sub_path + ".zip") - if os.path.exists(zip_path): + for zip_path in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + '.zip'): # We have a zipfile, so run through it and look for sub files that match the filename with zipfile.ZipFile(zip_path) as zfile: for name in zfile.namelist(): @@ -540,7 +542,8 @@ class Version3Format(Version2Format): if 'type' in symbol: symbol_type = self._interdict_to_template(symbol['type']) - self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address, type = symbol_type) + self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address, + type = symbol_type) return self._symbol_cache[name] diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 36e23a35d..d59a95db5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,10 +1,10 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import List, Tuple, Iterator +from typing import Iterator, List, Tuple from volatility3 import framework -from volatility3.framework import exceptions, constants, interfaces, objects +from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -30,6 +30,9 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('vfsmount', extensions.vfsmount) self.set_type_class('kobject', extensions.kobject) + if 'mnt_namespace' in self.types: + self.set_type_class('mnt_namespace', extensions.mnt_namespace) + if 'module' in self.types: self.set_type_class('module', extensions.module) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0edd60608..73f31115a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -201,6 +201,48 @@ class task_struct(generic.GenericIntelProcess): yield (start, end - start) + @property + def is_kernel_thread(self) -> bool: + """Checks if this task is a kernel thread. + + Returns: + bool: True, if this task is a kernel thread. Otherwise, False. + """ + return (self.flags & constants.linux.PF_KTHREAD) != 0 + + @property + def is_thread_group_leader(self) -> bool: + """Checks if this task is a thread group leader. + + Returns: + bool: True, if this task is a thread group leader. Otherwise, False. + """ + return self.tgid == self.pid + + @property + def is_user_thread(self) -> bool: + """Checks if this task is a user thread. + + Returns: + bool: True, if this task is a user thread. Otherwise, False. + """ + return not self.is_kernel_thread and self.tgid != self.pid + + def get_threads(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns a list of the task_struct based on the list_head + thread_node structure.""" + + task_symbol_table_name = self.get_symbol_table_name() + + # iterating through the thread_list from thread_group + # this allows iterating through pointers to grab the + # threads and using the thread_group offset to get the + # corresponding task_struct + for task in self.thread_group.to_list( + f"{task_symbol_table_name}{constants.BANG}task_struct", + "thread_group" + ): + yield task class fs_struct(objects.StructType): @@ -246,6 +288,29 @@ class super_block(objects.StructType): # include/linux/kdev_t.h MINORBITS = 20 + # Superblock flags + SB_RDONLY = 1 # Mount read-only + SB_NOSUID = 2 # Ignore suid and sgid bits + SB_NODEV = 4 # Disallow access to device special files + SB_NOEXEC = 8 # Disallow program execution + SB_SYNCHRONOUS = 16 # Writes are synced at once + SB_MANDLOCK = 64 # Allow mandatory locks on an FS + SB_DIRSYNC = 128 # Directory modifications are synchronous + SB_NOATIME = 1024 # Do not update access times + SB_NODIRATIME = 2048 # Do not update directory access times + SB_SILENT = 32768 + SB_POSIXACL = (1 << 16) # VFS does not apply the umask + SB_KERNMOUNT = (1 << 22) # this is a kern_mount call + SB_I_VERSION = (1 << 23) # Update inode I_version field + SB_LAZYTIME = (1 << 25) # Update the on-disk [acm]times lazily + + SB_OPTS = { + SB_SYNCHRONOUS: "sync", + SB_DIRSYNC: "dirsync", + SB_MANDLOCK: "mand", + SB_LAZYTIME: "lazytime" + } + @property def major(self) -> int: return self.s_dev >> self.MINORBITS @@ -254,6 +319,20 @@ class super_block(objects.StructType): def minor(self) -> int: return self.s_dev & ((1 << self.MINORBITS) - 1) + def get_flags_access(self) -> str: + return 'ro' if self.s_flags & self.SB_RDONLY else 'rw' + + def get_flags_opts(self) -> Iterable[str]: + sb_opts = [self.SB_OPTS[sb_opt] for sb_opt in self.SB_OPTS if sb_opt & self.s_flags] + return sb_opts + + def get_type(self): + mnt_sb_type = utility.pointer_to_string(self.s_type.name, count=255) + if self.s_subtype: + mnt_sb_subtype = utility.pointer_to_string(self.s_subtype, count=255) + mnt_sb_type += "." + mnt_sb_subtype + return mnt_sb_type + class vm_area_struct(objects.StructType): perm_flags = { @@ -330,7 +409,7 @@ class vm_area_struct(objects.StructType): fname = linux.LinuxUtilities.path_for_file(context, task, self.vm_file) elif self.vm_start <= task.mm.start_brk and self.vm_end >= task.mm.brk: fname = "[heap]" - elif self.vm_start <= task.mm.start_stack and self.vm_end >= task.mm.start_stack: + elif self.vm_start <= task.mm.start_stack <= self.vm_end: fname = "[stack]" elif self.vm_mm.context.has_member("vdso") and self.vm_start == self.vm_mm.context.vdso: fname = "[vdso]" @@ -373,7 +452,50 @@ class qstr(objects.StructType): class dentry(objects.StructType): def path(self) -> str: - return self.d_name.name_as_str() + """Based on __dentry_path Linux kernel function""" + reversed_path = [] + dentry_seen = set() + current_dentry = self + while (not current_dentry.is_root() and + current_dentry.vol.offset not in dentry_seen): + parent = current_dentry.d_parent + reversed_path.append(current_dentry.d_name.name_as_str()) + dentry_seen.add(current_dentry.vol.offset) + current_dentry = parent + return "/" + "/".join(reversed(reversed_path)) + + def is_root(self) -> bool: + return self.vol.offset == self.d_parent + + def is_subdir(self, old_dentry): + """Is this dentry a subdirectory of old_dentry? + + Returns true if this dentry is a subdirectory of the parent (at any depth). + Otherwise, it returns false. + """ + if self.vol.offset == old_dentry: + return True + + return self.d_ancestor(old_dentry) + + def d_ancestor(self, ancestor_dentry): + """Search for an ancestor + + Returns the ancestor dentry which is a child of "ancestor_dentry", + if "ancestor_dentry" is an ancestor of "child_dentry", else None. + """ + + dentry_seen = set() + current_dentry = self + while (not current_dentry.is_root() and + current_dentry.vol.offset not in dentry_seen): + if current_dentry.d_parent == ancestor_dentry.vol.offset: + return current_dentry + + dentry_seen.add(current_dentry.vol.offset) + current_dentry = current_dentry.d_parent + + return None class struct_file(objects.StructType): @@ -468,6 +590,27 @@ class files_struct(objects.StructType): class mount(objects.StructType): + MNT_NOSUID = 0x01 + MNT_NODEV = 0x02 + MNT_NOEXEC = 0x04 + MNT_NOATIME = 0x08 + MNT_NODIRATIME = 0x10 + MNT_RELATIME = 0x20 + MNT_READONLY = 0x40 + MNT_SHRINKABLE = 0x100 + MNT_WRITE_HOLD = 0x200 + MNT_SHARED = 0x1000 + MNT_UNBINDABLE = 0x2000 + + MNT_FLAGS = { + MNT_NOSUID: "nosuid", + MNT_NODEV: "nodev", + MNT_NOEXEC: "noexec", + MNT_NOATIME: "noatime", + MNT_NODIRATIME: "nodiratime", + MNT_RELATIME: "relatime", + } + def get_mnt_sb(self): if self.has_member("mnt"): return self.mnt.mnt_sb @@ -498,6 +641,82 @@ class mount(objects.StructType): def get_mnt_mountpoint(self): return self.mnt_mountpoint + def get_flags_access(self) -> str: + return "ro" if self.get_mnt_flags() & self.MNT_READONLY else "rw" + + def get_flags_opts(self) -> Iterable[str]: + flags = [self.MNT_FLAGS[mntflag] for mntflag in self.MNT_FLAGS if mntflag & self.get_mnt_flags()] + return flags + + def is_shared(self) -> bool: + return self.get_mnt_flags() & self.MNT_SHARED + + def is_unbindable(self) -> bool: + return self.get_mnt_flags() & self.MNT_UNBINDABLE + + def is_slave(self) -> bool: + return self.mnt_master and self.mnt_master.vol.offset != 0 + + def get_devname(self) -> str: + return utility.pointer_to_string(self.mnt_devname, count=255) + + def has_parent(self) -> bool: + return self.vol.offset != self.mnt_parent + + def get_dominating_id(self, root) -> int: + """Get ID of closest dominating peer group having a representative under the given root.""" + mnt_seen = set() + current_mnt = self.mnt_master + while (current_mnt and + current_mnt.vol.offset != 0 and + current_mnt.vol.offset not in mnt_seen): + peer = current_mnt.get_peer_under_root(self.mnt_ns, root) + if peer and peer.vol.offset != 0: + return peer.mnt_group_id + + mnt_seen.add(current_mnt.vol.offset) + current_mnt = current_mnt.mnt_master + return 0 + + def get_peer_under_root(self, ns, root): + """Return true if path is reachable from root. + It mimics the kernel function is_path_reachable(), ref: fs/namespace.c + """ + mnt_seen = set() + current_mnt = self + while current_mnt.vol.offset not in mnt_seen: + if current_mnt.mnt_ns == ns and current_mnt.is_path_reachable(current_mnt.mnt.mnt_root, root): + return current_mnt + + mnt_seen.add(current_mnt.vol.offset) + current_mnt = current_mnt.next_peer() + if current_mnt.vol.offset == self.vol.offset: + break + + return None + + def is_path_reachable(self, current_dentry, root): + """Return true if path is reachable. + It mimics the kernel function with same name, ref fs/namespace.c: + """ + mnt_seen = set() + current_mnt = self + while (current_mnt.mnt.vol.offset != root.mnt and + current_mnt.has_parent() and + current_mnt.vol.offset not in mnt_seen): + + current_dentry = current_mnt.mnt_mountpoint + mnt_seen.add(current_mnt.vol.offset) + current_mnt = current_mnt.mnt_parent + + return current_mnt.mnt.vol.offset == root.mnt and current_dentry.is_subdir(root.dentry) + + def next_peer(self): + table_name = self.vol.type_name.split(constants.BANG)[0] + mount_struct = "{0}{1}mount".format(table_name, constants.BANG) + offset = self._context.symbol_space.get_type(mount_struct).relative_child_offset("mnt_share") + + return self._context.object(mount_struct, self.vol.layer_name, offset=self.mnt_share.next.vol.offset - offset) class vfsmount(objects.StructType): @@ -539,3 +758,22 @@ class kobject(objects.StructType): ret = refcnt.refs.counter return ret + +class mnt_namespace(objects.StructType): + def get_inode(self): + if self.has_member("proc_inum"): + return self.proc_inum + elif self.ns.has_member("inum"): + return self.ns.inum + else: + raise AttributeError("Unable to find mnt_namespace inode") + + def get_mount_points(self): + table_name = self.vol.type_name.split(constants.BANG)[0] + mnt_type = table_name + constants.BANG + "mount" + if not self._context.symbol_space.has_type(mnt_type): + # Old kernels ~ 2.6 + mnt_type = table_name + constants.BANG + "vfsmount" + + for mount in self.list.to_list(mnt_type, "mnt_list"): + yield mount diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 94045d2e7..45dc1db70 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -1,19 +1,18 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib +import logging from typing import Generator, Iterable, Optional, Set, Tuple -import logging - -from volatility3.framework import constants, objects, renderers -from volatility3.framework import exceptions, interfaces +from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic vollog = logging.getLogger(__name__) + class proc(generic.GenericIntelProcess): def get_task(self): @@ -32,6 +31,7 @@ class proc(generic.GenericIntelProcess): try: dtb = self.get_task().map.pmap.pm_cr3 except exceptions.InvalidAddressException: + # Bail out because we couldn't find the DTB return None if preferred_name is None: @@ -43,10 +43,6 @@ class proc(generic.GenericIntelProcess): def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: try: task = self.get_task() - except exceptions.InvalidAddressException: - return - - try: current_map = task.map.hdr.links.next except exceptions.InvalidAddressException: return @@ -55,9 +51,9 @@ class proc(generic.GenericIntelProcess): for i in range(task.map.hdr.nentries): if (not current_map or - current_map.vol.offset in seen or - not self._context.layers[task.vol.native_layer_name].is_valid(current_map.dereference().vol.offset, current_map.dereference().vol.size)): - + current_map.vol.offset in seen or + not self._context.layers[task.vol.native_layer_name].is_valid(current_map.dereference().vol.offset, + current_map.dereference().vol.size)): vollog.log(constants.LOGLEVEL_VVV, "Breaking process maps iteration due to invalid state.") break @@ -102,10 +98,8 @@ class fileglob(objects.StructType): if self.has_member("fg_type"): ret = self.fg_type elif self.fg_ops != 0: - try: + with contextlib.suppress(exceptions.InvalidAddressException): ret = self.fg_ops.fo_type - except exceptions.InvalidAddressException: - pass if ret: ret = str(ret.description).replace("DTYPE_", "") @@ -456,7 +450,7 @@ class queue_entry(objects.StructType): seen = set() for attr in ['next', 'prev']: - try: + with contextlib.suppress(exceptions.InvalidAddressException): n = getattr(self, attr).dereference().cast(type_name) while n is not None and n.vol.offset != list_head: @@ -473,9 +467,6 @@ class queue_entry(objects.StructType): n = getattr(n.member(attr = member_name), attr).dereference().cast(type_name) - except exceptions.InvalidAddressException: - pass - class ifnet(objects.StructType): diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 7cde686ee..350bb0a53 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -38,4 +38,4 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): class LinuxMetadata(interfaces.symbols.MetadataInterface): - """Class to handle the etadata from a Linux symbol table.""" + """Class to handle the metadata from a Linux symbol table.""" diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index 3aa607574..cfac87e2c 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -1,10 +1,11 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import volatility3.framework.symbols.windows.extensions.pool +import contextlib + from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import extensions -from volatility3.framework.symbols.windows.extensions import registry, pool +from volatility3.framework.symbols.windows.extensions import pe, pool, registry class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -38,30 +39,24 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class('_SHARED_CACHE_MAP', extensions.SHARED_CACHE_MAP) self.set_type_class('_VACB', extensions.VACB) self.set_type_class('_POOL_TRACKER_BIG_PAGES', pool.POOL_TRACKER_BIG_PAGES) + self.set_type_class('_IMAGE_DOS_HEADER', pe.IMAGE_DOS_HEADER) + + # Might not necessarily defined in every version of windows + self.optional_set_type_class('_IMAGE_NT_HEADERS', pe.IMAGE_NT_HEADERS) + self.optional_set_type_class('_IMAGE_NT_HEADERS64', pe.IMAGE_NT_HEADERS) # This doesn't exist in very specific versions of windows - try: + with contextlib.suppress(ValueError): if self.get_type("_POOL_TRACKER_BIG_PAGES").has_member("PoolType"): self.set_type_class('_POOL_HEADER', pool.POOL_HEADER_VISTA) else: self.set_type_class('_POOL_HEADER', pool.POOL_HEADER) - except ValueError: - pass # these don't exist in windows XP - try: - self.set_type_class('_MMADDRESS_NODE', extensions.MMVAD_SHORT) - except ValueError: - pass + self.optional_set_type_class('_MMADDRESS_NODE', extensions.MMVAD_SHORT) # these were introduced starting in windows 8 - try: - self.set_type_class('_MM_AVL_NODE', extensions.MMVAD_SHORT) - except ValueError: - pass + self.optional_set_type_class('_MM_AVL_NODE', extensions.MMVAD_SHORT) # these were introduced starting in windows 7 - try: - self.set_type_class('_RTL_BALANCED_NODE', extensions.MMVAD_SHORT) - except ValueError: - pass + self.optional_set_type_class('_RTL_BALANCED_NODE', extensions.MMVAD_SHORT) diff --git a/volatility3/framework/symbols/windows/callbacks-x64.json b/volatility3/framework/symbols/windows/callbacks-x64.json index dbb6086df..87682cb92 100644 --- a/volatility3/framework/symbols/windows/callbacks-x64.json +++ b/volatility3/framework/symbols/windows/callbacks-x64.json @@ -8,6 +8,12 @@ "signed": false, "endian": "little" }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, "unsigned char": { "kind": "char", "size": 1, @@ -137,6 +143,43 @@ }, "kind": "struct", "size": 64 + }, + "_CM_CALLBACK_ENTRY": { + "fields": { + "Link": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Cookie": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 24 + }, + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "Altitude": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 64 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/callbacks-x86.json b/volatility3/framework/symbols/windows/callbacks-x86.json index cf0cb8b65..702b68a65 100644 --- a/volatility3/framework/symbols/windows/callbacks-x86.json +++ b/volatility3/framework/symbols/windows/callbacks-x86.json @@ -8,6 +8,12 @@ "signed": false, "endian": "little" }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, "unsigned char": { "kind": "char", "size": 1, @@ -137,6 +143,43 @@ }, "kind": "struct", "size": 28 + }, + "_CM_CALLBACK_ENTRY": { + "fields": { + "Link": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Cookie": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 16 + }, + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 28 + }, + "Altitude": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 40 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ae7c45d04..a86f5b3cb 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -3,20 +3,23 @@ # import collections.abc +import contextlib import datetime import functools import logging import math -from typing import Iterable, Iterator, Optional, Union, Tuple, List +from typing import Generator, Iterable, Iterator, List, Optional, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols +from volatility3.framework.interfaces.objects import ObjectInterface from volatility3.framework.layers import intel from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic -from volatility3.framework.symbols.windows.extensions import pool, pe, kdbg +from volatility3.framework.symbols.windows.extensions import kdbg, pe, pool vollog = logging.getLogger(__name__) + # Keep these in a basic module, to prevent import cycles when symbol providers require them @@ -84,7 +87,7 @@ class MMVAD_SHORT(objects.StructType): if tag in ["VadS", "VadF"]: target = "_MMVAD_SHORT" - elif tag != None and tag.startswith("Vad"): + elif tag is not None and tag.startswith("Vad"): target = "_MMVAD" elif depth == 0: # the root node at depth 0 is allowed to not have a tag @@ -194,8 +197,8 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the parent member") - def get_start(self): - """Get the VAD's starting virtual address.""" + def get_start(self) -> int: + """Get the VAD's starting virtual address. This is the first accessible byte in the range.""" if self.has_member("StartingVpn"): @@ -213,8 +216,8 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the starting VPN member") - def get_end(self): - """Get the VAD's ending virtual address.""" + def get_end(self) -> int: + """Get the VAD's ending virtual address. This is the last accessible byte in the range.""" if self.has_member("EndingVpn"): @@ -231,6 +234,10 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the ending VPN member") + def get_size(self) -> int: + """Get the size of the VAD region. The OS ensures page granularity.""" + return (self.get_end() - self.get_start()) + 1 + def get_commit_charge(self): """Get the VAD's commit charge (number of committed pages)""" @@ -303,18 +310,18 @@ class MMVAD(MMVAD_SHORT): file_name = renderers.NotApplicableValue() - try: + with contextlib.suppress(exceptions.InvalidAddressException): # this is for xp and 2003 if self.has_member("ControlArea"): - file_name = self.ControlArea.FilePointer.FileName.get_string() + filename_obj = self.ControlArea.FilePointer.FileName # this is for vista through windows 7 else: - file_name = self.Subsection.ControlArea.FilePointer.dereference().cast( - "_FILE_OBJECT").FileName.get_string() + filename_obj = self.Subsection.ControlArea.FilePointer.dereference().cast( + "_FILE_OBJECT").FileName - except exceptions.InvalidAddressException: - pass + if filename_obj.Length > 0: + file_name = filename_obj.get_string() return file_name @@ -348,17 +355,33 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel device objects.""" def get_device_name(self) -> str: + """Get device's name from the object header.""" header = self.get_object_header() return header.NameInfo.Name.String # type: ignore + def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: + """Enumerate the attached device's objects""" + device = self.AttachedDevice.dereference() + while device: + yield device + device = device.AttachedDevice.dereference() + class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" def get_driver_name(self) -> str: + """Get driver's name from the object header.""" header = self.get_object_header() return header.NameInfo.Name.String # type: ignore + def get_devices(self) -> Generator[ObjectInterface, None, None]: + """Enumerate the driver's device objects""" + device = self.DeviceObject.dereference() + while device: + yield device + device = device.NextDevice.dereference() + def is_valid(self) -> bool: """Determine if the object is valid.""" return True @@ -393,15 +416,11 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. if self._context.layers[self.vol.native_layer_name].is_valid(self.DeviceObject): - try: + with contextlib.suppress(ValueError): name = f"\\Device\\{self.DeviceObject.get_device_name()}" - except ValueError: - pass - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): name += self.FileName.String - except (TypeError, exceptions.InvalidAddressException): - pass return name @@ -428,9 +447,17 @@ class KMUTANT(objects.StructType, pool.ExecutiveObject): class ETHREAD(objects.StructType): """A class for executive thread objects.""" - def owning_process(self, kernel_layer: str = None) -> interfaces.objects.ObjectInterface: + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" - return self.ThreadsProcess.dereference(kernel_layer) + + # For Windows XPs + if(self.has_member("ThreadsProcess")): + return self.ThreadsProcess.dereference().cast("_EPROCESS") + # For Windows Vista and later versions + elif(self.has_member("Tcb") and self.Tcb.has_member("Process")): + return self.Tcb.Process.dereference().cast("_EPROCESS") + else: + raise AttributeError("Unable to find the owning process of ethread") def get_cross_thread_flags(self) -> str: dictCrossThreadFlags = { @@ -461,10 +488,13 @@ class UNICODE_STRING(objects.StructType): # We explicitly do *not* catch errors here, we allow an exception to be thrown # (otherwise there's no way to determine anything went wrong) # It's up to the user of this method to catch exceptions - return self.Buffer.dereference().cast("string", - max_length = self.Length, - errors = "replace", - encoding = "utf16") + + # We manually construct an object rather than casting a dereferenced pointer in case + # the buffer length is 0 and the pointer is a NULL pointer + return self._context.object(self.vol.type_name.split(constants.BANG)[0] + constants.BANG + 'string', + layer_name = self.Buffer.vol.native_layer_name, + offset = self.Buffer, + max_length = self.Length, errors = 'replace', encoding = 'utf16') String = property(get_string) @@ -551,9 +581,9 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): proc_layer = self._context.layers[proc_layer_name] if not proc_layer.is_valid(self.Peb): raise exceptions.InvalidAddressException(proc_layer_name, self.Peb, - f"Invalid address at {self.Peb:0x}") + f"Invalid Peb address at {self.Peb:0x}") - sym_table = self.vol.type_name.split(constants.BANG)[0] + sym_table = self.get_symbol_table_name() peb = self._context.object(f"{sym_table}{constants.BANG}_PEB", layer_name = proc_layer_name, offset = self.Peb) @@ -651,7 +681,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): except AttributeError: return False - return value != 0 and value != None + if value: + return True + + return False def get_vad_root(self): @@ -693,7 +726,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): env = envar[:split_index] var = envar[split_index + 1:] - # Exlude parse problem with some types of env + # Exclude parse problem with some types of env if env and var: yield env, var @@ -720,7 +753,10 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): trans_layer = self._context.layers[layer] try: - trans_layer.is_valid(self.vol.offset) + is_valid = trans_layer.is_valid(self.vol.offset) + if not is_valid: + return + link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: return @@ -735,9 +771,7 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): while link.vol.offset not in seen: obj_offset = link.vol.offset - relative_offset - try: - trans_layer.is_valid(obj_offset) - except exceptions.InvalidAddressException: + if not trans_layer.is_valid(obj_offset): return obj = self._context.object(symbol_type, @@ -895,8 +929,8 @@ class CONTROL_AREA(objects.StructType): return False # The first SubsectionBase should not be page aligned - #subsection = self.get_subsection() - #if subsection.SubsectionBase & self.PAGE_MASK == 0: + # subsection = self.get_subsection() + # if subsection.SubsectionBase & self.PAGE_MASK == 0: # return False except exceptions.InvalidAddressException: return False @@ -945,7 +979,7 @@ class CONTROL_AREA(objects.StructType): subsection_offset = starting_sector * 0x200 # Similar to the check in is_valid(), make sure the SubsectionBase is not page aligned. - #if subsection.SubsectionBase & self.PAGE_MASK == 0: + # if subsection.SubsectionBase & self.PAGE_MASK == 0: # break ptecount = 0 @@ -976,8 +1010,8 @@ class CONTROL_AREA(objects.StructType): # Currently just a temporary workaround to deal with custom bit flag # in the PFN field for pages in transition state. # See https://github.com/volatilityfoundation/volatility3/pull/475 - physoffset = (mmpte.u.Trans.PageFrameNumber & (( 1 << 33 ) - 1 ) ) << 12 - + physoffset = (mmpte.u.Trans.PageFrameNumber & ((1 << 33) - 1)) << 12 + yield physoffset, file_offset, self.PAGE_SIZE # Go to the next PTE entry @@ -1087,12 +1121,10 @@ class SHARED_CACHE_MAP(objects.StructType): iterval = 0 while (iterval < full_blocks) and (full_blocks <= 4): vacb_obj = self.InitialVacbs[iterval] - try: + with contextlib.suppress(exceptions.InvalidAddressException): # Make sure that the SharedCacheMap member of the VACB points back to the parent object. if vacb_obj.SharedCacheMap == self.vol.offset: self.save_vacb(vacb_obj, vacb_list) - except exceptions.InvalidAddressException: - pass iterval += 1 # We also have to account for the spill over data that is not found in the full blocks. diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py new file mode 100644 index 000000000..fc7996c52 --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -0,0 +1,62 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework import objects + +class PARTITION_TABLE(objects.StructType): + + def get_disk_signature(self) -> str: + """Get Disk Signature (GUID).""" + return "{0:02x}-{1:02x}-{2:02x}-{3:02x}".format( + self.DiskSignature[0], + self.DiskSignature[1], + self.DiskSignature[2], + self.DiskSignature[3] + ) + +class PARTITION_ENTRY(objects.StructType): + + def get_bootable_flag(self) -> int: + """Get Bootable Flag.""" + return self.BootableFlag + + def is_bootable(self) -> bool: + """Check Bootable Partition.""" + return False if not (self.get_bootable_flag() == 0x80) else True + + def get_partition_type(self) -> str: + """Get Partition Type.""" + return self.PartitionType.lookup() if self.PartitionType.is_valid_choice else "Not Defined PartitionType" + + def get_starting_chs(self) -> int: + """Get Starting CHS (Cylinder Header Sector) Address.""" + return self.StartingCHS[0] + + def get_ending_chs(self) -> int: + """Get Ending CHS (Cylinder Header Sector) Address.""" + return self.EndingCHS[0] + + def get_starting_sector(self) -> int: + """Get Starting Sector.""" + return self.StartingCHS[1] % 64 + + def get_ending_sector(self) -> int: + """Get Ending Sector.""" + return self.EndingCHS[1] % 64 + + def get_starting_cylinder(self) -> int: + """Get Starting Cylinder.""" + return (self.StartingCHS[1] - self.get_starting_sector()) * 4 + self.StartingCHS[2] + + def get_ending_cylinder(self) -> int: + """Get Ending Cylinder.""" + return (self.EndingCHS[1] - self.get_ending_sector()) * 4 + self.EndingCHS[2] + + def get_starting_lba(self) -> int: + """Get Starting LBA (Logical Block Addressing).""" + return self.StartingLBA + + def get_size_in_sectors(self) -> int: + """Get Size in Sectors.""" + return self.SizeInSectors diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py new file mode 100644 index 000000000..ba79b7c8b --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -0,0 +1,21 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework import objects + + +class MFTEntry(objects.StructType): + """This represents the base MFT Record""" + + def get_signature(self) -> str: + signature = self.Signature.cast('string', max_length = 4, encoding = 'latin-1') + return signature + + +class MFTFileName(objects.StructType): + """This represents an MFT $FILE_NAME Attribute""" + + def get_full_name(self) -> str: + output = self.Name.cast("string", encoding = "utf16", max_length = self.NameLength * 2, errors = "replace") + return output diff --git a/volatility3/framework/symbols/windows/extensions/pe.py b/volatility3/framework/symbols/windows/extensions/pe.py index df461318f..2f271da5d 100644 --- a/volatility3/framework/symbols/windows/extensions/pe.py +++ b/volatility3/framework/symbols/windows/extensions/pe.py @@ -2,15 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Generator, Tuple import logging +from typing import Generator, Tuple -from volatility3.framework import constants -from volatility3.framework import objects, interfaces +from volatility3.framework import constants, interfaces, objects from volatility3.framework.renderers import conversion vollog = logging.getLogger(__name__) + class IMAGE_DOS_HEADER(objects.StructType): def get_nt_header(self) -> interfaces.objects.ObjectInterface: @@ -77,12 +77,13 @@ class IMAGE_DOS_HEADER(objects.StructType): image_base_type = nt_header.OptionalHeader.ImageBase.vol.type_name member_size = self._context.symbol_space.get_type(image_base_type).size try: - newval = objects.convert_value_to_data(self.vol.offset, int, nt_header.OptionalHeader.ImageBase.vol.data_format) + newval = objects.convert_value_to_data(self.vol.offset, int, + nt_header.OptionalHeader.ImageBase.vol.data_format) new_pe = raw_data[:image_base_offset] + newval + raw_data[image_base_offset + member_size:] except OverflowError: vollog.warning("Volatility was unable to fix the image base for the PE file at base address {:#x}. " \ - "This will cause issues with many static analysis tools if you do not inform the " \ - "tool of the in-memory load address.".format(self.vol.offset)) + "This will cause issues with many static analysis tools if you do not inform the " \ + "tool of the in-memory load address.".format(self.vol.offset)) new_pe = raw_data return new_pe @@ -109,7 +110,7 @@ class IMAGE_DOS_HEADER(objects.StructType): size_of_image = nt_header.OptionalHeader.SizeOfImage # no legitimate PE is going to be larger than this - if size_of_image > (1024 * 1024 * 100): + if size_of_image > constants.windows.PE_MAX_EXTRACTION_SIZE: raise ValueError(f"The claimed SizeOfImage is too large: {size_of_image}") read_layer = self._context.layers[layer_name] diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index f75c6c417..79ea60027 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -1,12 +1,14 @@ +import contextlib import functools import logging import struct -from typing import Optional, Tuple, List, Dict, Union +from typing import Dict, List, Optional, Tuple, Union -from volatility3.framework import objects, interfaces, constants, symbols, exceptions, renderers -from volatility3.framework.renderers import conversion from volatility3.plugins.windows.poolscanner import PoolConstraint +from volatility3.framework import constants, exceptions, interfaces, objects, renderers, symbols +from volatility3.framework.renderers import conversion + vollog = logging.getLogger(__name__) @@ -128,8 +130,8 @@ class POOL_HEADER(objects.StructType): # --------------- if addr - optional_headers_length < 0: continue - padding_length = struct.unpack( - "= padding_length > addr: continue - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): mem_object = self._context.object(symbol_table_name + constants.BANG + type_name, layer_name = self.vol.layer_name, offset = addr + body_offset + start_offset, @@ -147,15 +149,13 @@ class POOL_HEADER(objects.StructType): if mem_object.is_valid(): yield mem_object - except (TypeError, exceptions.InvalidAddressException): - pass - # use the bottom up approach for windows 7 and earlier else: type_size = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + type_name).size if constraint.additional_structures: for additional_structure in constraint.additional_structures: - type_size += self._context.symbol_space.get_type(symbol_table_name + constants.BANG + additional_structure).size + type_size += self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + additional_structure).size rounded_size = conversion.round(type_size, alignment, up = True) @@ -164,11 +164,9 @@ class POOL_HEADER(objects.StructType): offset = self.vol.offset + self.BlockSize * alignment - rounded_size, native_layer_name = native_layer_name) - try: + with contextlib.suppress(TypeError, exceptions.InvalidAddressException): if mem_object.is_valid(): yield mem_object - except (TypeError, exceptions.InvalidAddressException): - pass @classmethod @functools.lru_cache() @@ -177,20 +175,18 @@ class POOL_HEADER(objects.StructType): headers = [] sizes = [] for header in [ - 'CREATOR_INFO', 'NAME_INFO', 'HANDLE_INFO', 'QUOTA_INFO', 'PROCESS_INFO', 'AUDIT_INFO', 'EXTENDED_INFO', - 'HANDLE_REVOCATION_INFO', 'PADDING_INFO' + 'CREATOR_INFO', 'NAME_INFO', 'HANDLE_INFO', 'QUOTA_INFO', 'PROCESS_INFO', 'AUDIT_INFO', 'EXTENDED_INFO', + 'HANDLE_REVOCATION_INFO', 'PADDING_INFO' ]: - try: + with contextlib.suppress(AttributeError, exceptions.SymbolError): type_name = f"{symbol_table_name}{constants.BANG}_OBJECT_HEADER_{header}" header_type = context.symbol_space.get_type(type_name) headers.append(header) sizes.append(header_type.size) - except (AttributeError, exceptions.SymbolError): # Some of these may not exist, for example: # if build < 9200: PADDING_INFO else: AUDIT_INFO # if build == 10586: HANDLE_REVOCATION_INFO else EXTENDED_INFO # based on what's present and what's not, this list should be the right order and the right length - pass return headers, sizes def is_free_pool(self): @@ -233,7 +229,10 @@ class POOL_TRACKER_BIG_PAGES(objects.StructType): def is_valid(self) -> bool: return self.Key > 0 - # return self.Va > 0x1 + + def is_free(self) -> bool: + """Returns if the allocation is freed (True) or in-use (False)""" + return self.Va & 1 == 1 def get_key(self) -> str: """Returns the Key value as a 4 character string""" diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index cbd9052cd..c71fcf49b 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -1,14 +1,14 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import contextlib import enum import logging import struct -from typing import Optional, Iterable, Union +from typing import Iterable, Optional, Union -from volatility3.framework import constants, exceptions, objects, interfaces -from volatility3.framework.layers.registry import RegistryHive, RegistryInvalidIndex, RegistryFormatException +from volatility3.framework import constants, exceptions, interfaces, objects +from volatility3.framework.layers.registry import RegistryFormatException, RegistryHive, RegistryInvalidIndex vollog = logging.getLogger(__name__) @@ -75,10 +75,10 @@ class CMHIVE(objects.StructType): """ for attr in ["FileFullPath", "FileUserName", "HiveRootPath"]: - try: - return getattr(self, attr).get_string() - except (AttributeError, exceptions.InvalidAddressException): - pass + with contextlib.suppress(AttributeError, exceptions.InvalidAddressException): + name = getattr(self, attr) + if name.Length > 0: + return name.get_string() return None @@ -264,19 +264,22 @@ class CM_KEY_VALUE(objects.StructType): if self_type == RegValueTypes.REG_DWORD: if len(data) != struct.calcsize("L"): raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}") - return struct.unpack(">L", data)[0] + res, = struct.unpack(">L", data) + return res if self_type == RegValueTypes.REG_QWORD: if len(data) != struct.calcsize(" str: + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: """Creates symbol table for a module in the specified layer_name. Searches the memory section of the loaded module for its PDB GUID @@ -307,6 +316,19 @@ class PDBUtility(interfaces.configuration.VersionableInterface): Returns: The name of the constructed and loaded symbol table """ + _, symbol_table_name = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size) + return symbol_table_name + + @classmethod + def _modtable_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None, + create_module: bool = False) -> Tuple[Optional[str], Optional[str]]: + + if module_offset is None: + module_offset = context.layers[layer_name].minimum_address + if module_size is None: + module_size = context.layers[layer_name].maximum_address - module_offset guids = list( cls.pdbname_scan(context, @@ -323,12 +345,46 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - return cls.load_windows_symbol_table(context, - guid["GUID"], - guid["age"], - guid["pdb_name"], - "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path = config_path) + module_name = guid["pdb_name"].replace('.pdb', '') + + symbol_table_name = cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) + + new_module_name = None + if create_module: + new_module = contexts.Module.create(context, module_name, layer_name, offset = guid['mz_offset'], + symbol_table_name = symbol_table_name) + new_module_name = new_module.name + + return new_module_name, symbol_table_name + + @classmethod + def module_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int = None, module_size: int = None) -> str: + """Creates a module in the specified layer_name based on a pdb name. + + Searches the memory section of the loaded module for its PDB GUID + and loads the associated symbol table into the symbol space. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + config_path: The config path where to find symbol files + layer_name: The name of the layer on which to operate + module_offset: This memory dump's module image offset + module_size: The size of the module for this dump + + Returns: + The name of the constructed and loaded symbol table + """ + + module_name, _ = cls._modtable_from_pdb(context, config_path, layer_name, pdb_name, module_offset, + module_size, create_module = True) + + return module_name class PdbSignatureScanner(interfaces.layers.ScannerInterface): diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 96a7e3977..6029c0a5c 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,11 +1,14 @@ +import contextlib +import logging import struct -from typing import List, Iterator, Tuple +from typing import List, Iterator, Optional, Tuple, Type -from volatility3.framework import interfaces, renderers +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey +vollog = logging.getLogger(__name__) class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" @@ -15,12 +18,14 @@ class Certificates(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)) + requirements.PluginRequirement(name = 'printkey', plugin = printkey.PrintKey, version = (1, 0, 0)), + requirements.BooleanRequirement(name = 'dump', + description = "Extract listed certificates", + default = False, + optional = True) ] def parse_data(self, data: bytes) -> Tuple[str, bytes]: @@ -34,36 +39,51 @@ class Certificates(interfaces.plugins.PluginInterface): elif ctype == 0x100000020: certificate_data = cvalue return (name, certificate_data) + + @classmethod + def dump_certificate(cls, certificate_data: bytes, hive_offset: int, + reg_section: str, key_hash: str, + open_method: Type[interfaces.plugins.FileHandlerInterface]) -> \ + Optional[interfaces.plugins.FileHandlerInterface]: + try: + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) + file_handle = open_method(dump_name) + file_handle.write(certificate_data) + return file_handle + except exceptions.InvalidAddressException: + vollog.debug(f"Unable to certificate file at {hive_offset:#x}") + return None + def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: + kernel = self.context.modules[self.config['kernel']] + for hive in hivelist.HiveList.list_hives(self.context, base_config_path = self.config_path, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols']): + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name): for top_key in [ "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - try: + with contextlib.suppress(KeyError, exceptions.InvalidAddressException): # Walk it node_path = hive.get_key(top_key, return_list = True) - for (depth, is_key, last_write_time, key_path, volatility, - node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): + for (_depth, is_key, _last_write_time, key_path, _volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) - unique_key_offset = key_path.index(top_key) + len(top_key) + 1 + unique_key_offset = key_path.casefold().index(top_key.casefold()) + len(top_key) + 1 reg_section = key_path[unique_key_offset:key_path.index("\\", unique_key_offset)] key_hash = key_path[key_path.rindex("\\") + 1:] - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - with self.open("{} - {} - {}.crt".format(hex(hive.hive_offset), reg_section, - key_hash)) as file_data: - file_data.write(certificate_data) + if self.config['dump']: + file_handle = self.dump_certificate(certificate_data, hive.hive_offset, reg_section, key_hash, self.open) + if file_handle: + file_handle.close() + yield (0, (top_key, reg_section, key_hash, name)) - except KeyError: - # Key wasn't found in this hive, carry on - pass def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("Certificate path", str), ("Certificate section", str), ("Certificate ID", str), diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 65329a4f5..8666680b3 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -6,7 +6,7 @@ import hashlib import json import logging import os -from typing import Set, Any, Dict +from typing import Any, Dict, Optional, Set from volatility3.framework import constants @@ -51,9 +51,21 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: return valid(input, schema, use_cache) -def create_json_hash(input: Dict[str, Any], schema: Dict[str, Any]) -> str: +def create_json_hash(input: Dict[str, Any], schema: Optional[Dict[str, Any]] = None) -> Optional[str]: """Constructs the hash of the input and schema to create a unique identifier for a particular JSON file.""" + if schema is None: + format = input.get('metadata', {}).get('format', None) + if not format: + vollog.debug("No schema format defined") + return None + basepath = os.path.abspath(os.path.dirname(__file__)) + schema_path = os.path.join(basepath, 'schema-' + format + '.json') + if not os.path.exists(schema_path): + vollog.debug(f"Schema for format not found: {schema_path}") + return None + with open(schema_path, 'r') as s: + schema = json.load(s) return hashlib.sha1(bytes(json.dumps((input, schema), sort_keys = True), 'utf-8')).hexdigest()