diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 3a0cce8cc..2ccd4713c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -23,8 +23,10 @@ Steps to reproduce the behavior: **Expected behavior** A clear and concise description of what you expected to happen. -**Screenshots** -If applicable, add screenshots to help explain your problem. +**Example output** +Please copy and paste the text demonstrating the issue, ideally with verbose output turned on (`vol.py -vvv ...`). + +Text is preferred to screenshots for searching and to talk about specific parts of the output. **Additional information** Add any other information about the problem here. diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 77fe26931..061c8359a 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -15,14 +15,16 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.6"] steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.x - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..72bba07aa --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,74 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "develop" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "develop" ] + schedule: + - cron: '16 8 * * 0' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + queries: security-and-quality # ,security-extended + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2d3729981..b1a9dd31b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,14 +3,16 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.7"] steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.7 - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: '3.7' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | diff --git a/development/mac-kdk/parse_pbzx2.py b/development/mac-kdk/parse_pbzx2.py index 7ce9090d4..173a4d648 100644 --- a/development/mac-kdk/parse_pbzx2.py +++ b/development/mac-kdk/parse_pbzx2.py @@ -17,61 +17,54 @@ def seekread(f, offset = None, length = 0, relative = True): f.seek(offset, [0, 1, 2][relative]) if length: return f.read(length) + return None def parse_pbzx(pbzx_path): section = 0 xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) - f = open(pbzx_path, 'rb') - # pbzx = f.read() - # f.close() - magic = seekread(f, length = 4) - if magic != 'pbzx': - raise RuntimeError("Error: Not a pbzx file") - # Read 8 bytes for initial flags - flags = seekread(f, length = 8) - # Interpret the flags as a 64-bit big-endian unsigned int - flags = struct.unpack('>Q', flags)[0] - xar_f = open(xar_out_path, 'wb') - while flags & (1 << 24): - # Read in more flags + with open(pbzx_path, 'rb') as f: + # pbzx = f.read() + # f.close() + magic = seekread(f, length = 4) + if magic != 'pbzx': + raise RuntimeError("Error: Not a pbzx file") + # Read 8 bytes for initial flags flags = seekread(f, length = 8) + # Interpret the flags as a 64-bit big-endian unsigned int flags = struct.unpack('>Q', flags)[0] - # Read in length - f_length = seekread(f, length = 8) - f_length = struct.unpack('>Q', f_length)[0] - xzmagic = seekread(f, length = 6) - if xzmagic != '\xfd7zXZ\x00': - # This isn't xz content, this is actually _raw decompressed cpio_ chunk of 16MB in size... - # Let's back up ... - seekread(f, offset = -6, length = 0) - # ... and split it out ... - f_content = seekread(f, length = f_length) - section += 1 - decomp_out = '%s.part%02d.cpio' % (pbzx_path, section) - g = open(decomp_out, 'wb') - g.write(f_content) - g.close() - # Now to start the next section, which should hopefully be .xz (we'll just assume it is ...) - xar_f.close() - section += 1 - new_out = '%s.part%02d.cpio.xz' % (pbzx_path, section) - xar_f = open(new_out, 'wb') - else: - f_length -= 6 - # This part needs buffering - f_content = seekread(f, length = f_length) - tail = seekread(f, offset = -2, length = 2) - xar_f.write(xzmagic) - xar_f.write(f_content) - if tail != 'YZ': - xar_f.close() - raise RuntimeError("Error: Footer is not xar file footer") - try: - f.close() - xar_f.close() - except IOError: - pass + while flags & (1 << 24): + with open(xar_out_path, 'wb') as xar_f: + xar_f.seek(0, os.SEEK_END) + # Read in more flags + flags = seekread(f, length = 8) + flags = struct.unpack('>Q', flags)[0] + # Read in length + f_length = seekread(f, length = 8) + f_length = struct.unpack('>Q', f_length)[0] + xzmagic = seekread(f, length = 6) + if xzmagic != '\xfd7zXZ\x00': + # This isn't xz content, this is actually _raw decompressed cpio_ chunk of 16MB in size... + # Let's back up ... + seekread(f, offset = -6, length = 0) + # ... and split it out ... + f_content = seekread(f, length = f_length) + section += 1 + decomp_out = '%s.part%02d.cpio' % (pbzx_path, section) + with open(decomp_out, 'wb') as g: + g.write(f_content) + # Now to start the next section, which should hopefully be .xz (we'll just assume it is ...) + section += 1 + xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) + else: + f_length -= 6 + # This part needs buffering + f_content = seekread(f, length = f_length) + tail = seekread(f, offset = -2, length = 2) + xar_f.write(xzmagic) + xar_f.write(f_content) + if tail != 'YZ': + raise RuntimeError("Error: Footer is not xar file footer") def main(): diff --git a/doc/source/conf.py b/doc/source/conf.py index 731a73d56..895219b25 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -111,14 +111,17 @@ needs_sphinx = '2.0' # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'sphinx.ext.viewcode' + 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.autosectionlabel' ] +autosectionlabel_prefix_document = True + try: import sphinx_autodoc_typehints extensions.append('sphinx_autodoc_typehints') except ImportError: + # If the autodoc typehints extension isn't available, carry on regardless pass # Add any paths that contain templates here, relative to this directory. diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst new file mode 100644 index 000000000..26ad2c2e4 --- /dev/null +++ b/doc/source/getting-started-linux-tutorial.rst @@ -0,0 +1,192 @@ +Linux Tutorial +============== + +This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite. + +Acquiring memory +---------------- + +Volatility3 does not provide the ability to acquire memory. Below are some examples of tools that can be used to acquire memory, but more are available: + +* `AVML - Acquire Volatile Memory for Linux `_ +* `LiME - Linux Memory Extract `_ + + +Procedure to create symbol tables for linux +-------------------------------------------- + +To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`. + +.. tip:: It may be possible to locate pre-made ISF files from the `Linux ISF Server `_ , + which is built and maintained by `kevthehermit `_. + After creating the file or downloading it from the ISF server, place the file under the directory ``volatility3/symbols/linux``. + If necessary create a linux directory under the symbols directory (this will become unnecessary in future versions). + + +Listing plugins +--------------- + +The following is a sample of the linux plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. + +.. code-block:: shell-session + + $ python3 vol.py --help | grep -i linux. | head -n 5 + banners.Banners Attempts to identify potential linux banners in an + linux.bash.Bash Recovers bash command history from memory. + linux.check_afinfo.Check_afinfo + linux.check_creds.Check_creds + linux.check_idt.Check_idt + +.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of linux plugins. + + +Using plugins +------------- + +The following is the syntax to run the volatility CLI. + +.. code-block:: shell-session + + $ python3 vol.py -f + + +Example +------- + +banners +~~~~~~~ + +In this example we will be using a memory dump from the Insomni'hack teaser 2020 CTF Challenge called Getdents. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge. +Thanks go to `stuxnet `_ for providing this memory dump and `writeup `_. + + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem banners + + Volatility 3 Framework 2.0.1 + + Progress: 100.00 PDB scanning finished + Offset Banner + + 0x141c1390 Linux version 4.15.0-42-generic (buildd@lgw01-amd64-023) (gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)) #45-Ubuntu SMP Thu Nov 15 19:32:57 UTC 2018 (Ubuntu 4.15.0-42.45-generic 4.15.18) + 0x63a00160 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x6455c4d4 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x6e1e055f Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + 0x7fde0010 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18) + + +The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server. +If ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. + +.. tip:: Use the banner text which is most repeated to search from ISF Server. + +linux.pslist +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.pslist + + Volatility 3 Framework 2.0.1 Stacking attempts finished + + PID PPID COMM + + 1 0 systemd + 2 0 kthreadd + 3 2 kworker/0:0 + 4 2 kworker/0:0H + 5 2 kworker/u256:0 + 6 2 mm_percpu_wq + 7 2 ksoftirqd/0 + 8 2 rcu_sched + 9 2 rcu_bh + 10 2 migration/0 + 11 2 watchdog/0 + 12 2 cpuhp/0 + 13 2 kdevtmpfs + 14 2 netns + 15 2 rcu_tasks_kthre + 16 2 kauditd + ..... + +``linux.pslist`` helps us to list the processes which are running, their PIDs and PPIDs. + +linux.pstree +~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.pstree + Volatility 3 Framework 2.0.1 + Progress: 100.00 Stacking attempts finished + PID PPID COMM + + 1 0 systemd + * 636 1 polkitd + * 514 1 acpid + * 1411 1 pulseaudio + * 517 1 rsyslogd + * 637 1 cups-browsed + * 903 1 whoopsie + * 522 1 ModemManager + * 525 1 cron + * 526 1 avahi-daemon + ** 542 526 avahi-daemon + * 657 1 unattended-upgr + * 914 1 kerneloops + * 532 1 dbus-daemon + * 1429 1 ibus-x11 + * 929 1 kerneloops + * 1572 1 gsd-printer + * 933 1 upowerd + * 1071 1 rtkit-daemon + * 692 1 gdm3 + ** 1234 692 gdm-session-wor + *** 1255 1234 gdm-x-session + **** 1257 1255 Xorg + **** 1266 1255 gnome-session-b + ***** 1537 1266 gsd-clipboard + ***** 1539 1266 gsd-color + ***** 1542 1266 gsd-datetime + ***** 2950 1266 deja-dup-monito + ***** 1546 1266 gsd-housekeepin + ***** 1548 1266 gsd-keyboard + ***** 1550 1266 gsd-media-keys + +``linux.pstree`` helps us to display the parent child relationships between processes. + +linux.bash +~~~~~~~~~~ + +Now to find the commands that were run in the bash shell by using ``linux.bash``. + +.. code-block:: shell-session + + $ python3 vol.py -f memory.vmem linux.bash + + Volatility 3 Framework 2.0.1 + Progress: 100.00 Stacking attempts finished + PID Process CommandTime Command + + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 AWAVH�� + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 sudo apt update + 1733 bash 2020-01-16 14:00:36.000000 rub + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:36.000000 uname -a + 1733 bash 2020-01-16 14:00:36.000000 uname -a + 1733 bash 2020-01-16 14:00:36.000000 sudo apt autoclean + 1733 bash 2020-01-16 14:00:36.000000 sudo reboot + 1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade + 1733 bash 2020-01-16 14:00:41.000000 chmod +x meterpreter + 1733 bash 2020-01-16 14:00:42.000000 sudo ./meterpreter diff --git a/doc/source/getting-started-windows-tutorial.rst b/doc/source/getting-started-windows-tutorial.rst new file mode 100644 index 000000000..c89b065f5 --- /dev/null +++ b/doc/source/getting-started-windows-tutorial.rst @@ -0,0 +1,124 @@ +Windows Tutorial +================ + +This guide provides a brief introduction to how volatility3 works as a demonstration of several of the plugins available in the suite. + +Acquiring memory +---------------- + +Volatility does not provide the ability to acquire memory. +Memory can be acquired using a number of tools, below are some examples but others exist: + +* `WinPmem `_ +* `FTK Imager `_ + +Listing Plugins +--------------- + +The following is a sample of the windows plugins available for volatility3, it is not complete and more more plugins may +be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. +For plugin requests, please create an issue with a description of the requested plugin. + +.. code-block:: shell-session + + $ python3 vol.py --help | grep windows | head -n 5 + windows.bigpools.BigPools + windows.cmdline.CmdLine + windows.crashinfo.Crashinfo + windows.dlllist.DllList + +.. note:: Here the the command is piped to grep and head in-order to provide the start of a list of the available windows plugins. + +Using plugins +------------- + +The following is the syntax to run the volatility CLI. + +.. code-block:: shell-session + + $ python3 vol.py -f plugin_name plugin_option + + +Example +------- + +windows.pslist +~~~~~~~~~~~~~~ + +In this example we will be using a memory dump from the PragyanCTF'22. +We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenges. + +When using windows plugins in volatility 3, the required ISF file can often be generated from PDB files automatically +downloaded from Microsoft servers, and therefore does not require locating or adding specific ISF files to the volatility 3 symbols directory. + +.. code-block:: shell-session + + $ python3 vol.py -f MemDump.DMP windows.pslist | head -n 10 + + Volatility 3 Framework 2.0.1 PDB scanning finished + + PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output + + 4 0 System 0xfa8000cbc040 85 492 N/A False 2022-02-07 16:30:12.000000 N/A Disabled + 276 4 smss.exe 0xfa8001e04040 2 29 N/A False 2022-02-07 16:30:12.000000 N/A Disabled + 352 336 csrss.exe 0xfa8002110b30 9 375 0 False 2022-02-07 16:30:13.000000 N/A Disabled + 404 336 wininit.exe 0xfa800219f060 3 74 0 False 2022-02-07 16:30:13.000000 N/A Disabled + 412 396 csrss.exe 0xfa80021c5b30 9 224 1 False 2022-02-07 16:30:13.000000 N/A Disabled + 468 396 winlogon.exe 0xfa8002284060 5 113 1 False 2022-02-07 16:30:14.000000 N/A Disabled + +``windows.pslist`` helps list the processes running while the memory dump was taken. + +windows.pstree +~~~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f MemDump.DMP windows.pstree | head -n 20 + Volatility 3 Framework 2.0.1 PDB scanning finished + + PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime + + 4 0 System 0xfa8000cbc040 85 492 N/A False 2022-02-07 16:30:12.000000 N/A + * 276 4 smss.exe 0xfa8001e04040 2 29 N/A False 2022-02-07 16:30:12.000000 N/A + 352 336 csrss.exe 0xfa8002110b30 9 375 0 False 2022-02-07 16:30:13.000000 N/A + 404 336 wininit.exe 0xfa800219f060 3 74 0 False 2022-02-07 16:30:13.000000 N/A + * 504 404 services.exe 0xfa80022ccb30 7 190 0 False 2022-02-07 16:30:14.000000 N/A + ** 960 504 svchost.exe 0xfa8001c17b30 39 1003 0 False 2022-02-07 16:30:14.000000 N/A + ** 1216 504 svchost.exe 0xfa80026e0b30 18 311 0 False 2022-02-07 16:30:15.000000 N/A + ** 1312 504 svchost.exe 0xfa8002740380 19 287 0 False 2022-02-07 16:30:15.000000 N/A + ** 1984 504 taskhost.exe 0xfa8002eb1b30 8 129 1 False 2022-02-07 16:30:27.000000 N/A + ** 804 504 svchost.exe 0xfa80024ca5f0 20 450 0 False 2022-02-07 16:30:14.000000 N/A + *** 100 804 audiodg.exe 0xfa80025b4b30 6 131 0 False 2022-02-07 16:30:14.000000 N/A + ** 1568 504 SearchIndexer. 0xfa800254b480 12 616 0 False 2022-02-07 16:30:32.000000 N/A + ** 744 504 svchost.exe 0xfa8002477b30 8 265 0 False 2022-02-07 16:30:14.000000 N/A + ** 1096 504 svchost.exe 0xfa800260db30 14 357 0 False 2022-02-07 16:30:14.000000 N/A + ** 616 504 svchost.exe 0xfa8002b86ab0 13 314 0 False 2022-02-07 16:32:16.000000 N/A + ** 624 504 svchost.exe 0xfa8002410630 10 350 0 False 2022-02-07 16:30:14.000000 N/A + +``windows.pstree`` helps to display the parent child relationships between processes. + +.. note:: Here the the command is piped to head in-order to provide smaller output, here listing only the first 20. + +windows.hashdump +~~~~~~~~~~~~~~~~ + +.. code-block:: shell-session + + $ python3 vol.py -f MemDump.DMP windows.hashdump + Volatility 3 Framework 2.0.3 + Progress: 100.00 PDB scanning finished + User rid lmhash nthash + + Administrator 500 aad3b435b51404eeaad3b435b51404ee 31d6cfe0d16ae931b73c59d7e0c089c0 + Guest 501 aad3b435b51404eeaad3b435b51404ee 31d6cfe0d16ae931b73c59d7e0c089c0 + Frank Reynolds 1000 aad3b435b51404eeaad3b435b51404ee a88d1e18706d3aa676e01e5943d15911 + HomeGroupUser$ 1002 aad3b435b51404eeaad3b435b51404ee af10ecac6ea817d2bb56e3e5c33ce1cd + Dennis 1003 aad3b435b51404eeaad3b435b51404ee cf96684bbc7877920adaa9663698bf54 + +``windows.hashdump`` helps to list the hashes of the users in the system. + + + + + + diff --git a/doc/source/index.rst b/doc/source/index.rst index 3b5a5d2a8..9b1d05858 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -7,9 +7,10 @@ Volatility 3 is Open Source. :doc:`List of plugins ` -Here are some guidelines for using Volatility 3 effectively: +Below is the main documentation regarding volatility 3: .. toctree:: + :caption: Documentation basics development @@ -18,12 +19,21 @@ Here are some guidelines for using Volatility 3 effectively: volshell glossary -Python Packages -=============== +There is also some information to get you started quickly: .. toctree:: + :caption: Getting Started + + getting-started-linux-tutorial + getting-started-windows-tutorial + + +.. toctree:: + :caption: Python Packages + volatility3 + Indices and tables ================== diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index fd8b8933e..b7c26e046 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -40,8 +40,8 @@ following command: The :envvar:`PYTHONPATH` environment variable is not required if the Volatility library is installed in the system's library path or a virtual environment. -Mac/Linux symbol tables ------------------------ +Mac or Linux symbol tables +-------------------------- For Mac/Linux systems, both use the same mechanism for identification. The generated files contain an identifying string (the operating system banner), which Volatility's automagic can detect. Volatility caches the mapping between the strings and the symbol diff --git a/requirements-dev.txt b/requirements-dev.txt index 3ff7c50b8..7c372da2a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -16,7 +16,7 @@ pycryptodome # This can improve error messages regarding improperly configured ISF files, # but is only recommended for development -# jsonschema>=2.3.0 +jsonschema>=2.3.0 # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 diff --git a/setup.py b/setup.py index bce21ca66..a7456bf7a 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ setuptools.setup(name = "volatility3", '': ['development', 'development.*'], 'development': ['*'] }, - packages = setuptools.find_packages(exclude = ["development", "development.*"]), + packages = setuptools.find_namespace_packages(exclude = ["development", "development.*"]), entry_points = { 'console_scripts': [ 'vol = volatility3.cli:main', diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 5eeef77cf..f32a587e0 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -36,7 +36,6 @@ class VolShell(cli.CommandLine): def __init__(self): super().__init__() - self.output_dir = None def run(self): """Executes the command line module, taking the system arguments, diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 19e263a03..274b6ca17 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -324,11 +324,11 @@ class Volshell(interfaces.plugins.PluginInterface): " " * (longest_member - len_member), " ", member_type.vol.type_name) @classmethod - def _display_value(self, value: Any) -> str: + def _display_value(cls, value: Any) -> str: if isinstance(value, objects.PrimitiveObject): return repr(value) elif isinstance(value, objects.Array): - return repr([self._display_value(val) for val in value]) + return repr([cls._display_value(val) for val in value]) else: return hex(value.vol.offset) @@ -390,8 +390,8 @@ class Volshell(interfaces.plugins.PluginInterface): location = "file:" + request.pathname2url(location) print(f"Running code from {location}\n") accessor = resources.ResourceAccessor() - with io.TextIOWrapper(accessor.open(url = location), encoding = 'utf-8') as fp: - self.__console.runsource(fp.read(), symbol = 'exec') + with accessor.open(url = location) as fp: + self.__console.runsource(io.TextIOWrapper(fp.read(), encoding = 'utf-8'), symbol = 'exec') print("\nCode complete") def load_file(self, location: str): diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 5cbdbfe0e..36288ef90 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -181,6 +181,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): hex(kvo))) except exceptions.InvalidAddressException: vollog.debug(f"Potential kernel_virtual_offset caused a page fault: {hex(kvo)}") + return None vollog.debug("Kernel base determination - testing fixed base address") return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, False, True, progress_callback) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 1e0bba86e..164021340 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import base64 +import datetime import json import logging import os @@ -157,10 +158,10 @@ class SqliteCache(CacheManagerInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) - cache_period = '-3 days' def __init__(self, filename: str): super().__init__(filename) + self.cache_period = constants.SQLITE_CACHE_PERIOD try: self._database = self._connect_storage(filename) except sqlite3.DatabaseError: @@ -170,6 +171,7 @@ class SqliteCache(CacheManagerInterface): def _connect_storage(self, path: str) -> sqlite3.Connection: database = sqlite3.connect(path) database.row_factory = sqlite3.Row + database.cursor().execute( f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})') schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone() @@ -221,8 +223,7 @@ class SqliteCache(CacheManagerInterface): def is_url_local(self, url: str) -> bool: """Determines whether an url is local or not""" parsed = urllib.parse.urlparse(url) - if parsed.scheme in ['file', 'jar']: - return True + return parsed.scheme in ['file', 'jar'] def get_identifier(self, location: str) -> Optional[bytes]: results = self._database.cursor().execute('SELECT identifier FROM cache WHERE location = ?', @@ -244,6 +245,7 @@ class SqliteCache(CacheManagerInterface): (location,)).fetchall() for row in results: return row['hash'] + return None def update(self, progress_callback = None): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. @@ -259,10 +261,31 @@ class SqliteCache(CacheManagerInterface): cache_update = set() files_to_timestamp = on_disk_locations.intersection(cached_locations) if files_to_timestamp: - result = self._database.cursor().execute("SELECT location FROM cache WHERE local = 1 " + result = self._database.cursor().execute("SELECT location, cached FROM cache WHERE local = 1 " f"AND cached < date('now', '{self.cache_period}');") for row in result: - if row['location'] in files_to_timestamp: + location = row['location'] + stored_timestamp = datetime.datetime.fromisoformat(row['cached']) + timestamp = stored_timestamp # Default to requiring update + + # See if the file is a local URL type we can handle: + parsed = urllib.parse.urlparse(location) + pathname = None + if parsed.scheme == 'file': + pathname = urllib.request.url2pathname(parsed.path) + if parsed.scheme == 'jar': + inner_url = urllib.parse.urlparse(parsed.path) + if inner_url.scheme == 'file': + pathname = inner_url.path.split('!')[0] + + if pathname: + timestamp = datetime.datetime.fromtimestamp(os.stat(pathname).st_mtime) + else: + vollog.log(constants.LOGLEVEL_VVVV, + "File location in database classed as local but not file/jar URL") + + # If we're supposed to include it, and our last check is older than (or equal to) the file timestamp + if row['location'] in files_to_timestamp and stored_timestamp < timestamp: cache_update.add(row['location']) idextractors = list(framework.class_subclasses(IdentifierProcessor)) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index a8af538f9..79d67c61a 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -41,7 +41,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 4 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature @@ -64,6 +64,9 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" +SQLITE_CACHE_PERIOD = '-3 days' +"""SQLite time modifier for how long each item is valid in the cache for""" + if sys.platform == 'win32': CACHE_PATH = os.path.realpath(os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3")) os.makedirs(CACHE_PATH, exist_ok = True) diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index 713f91da0..4885645c3 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -113,7 +113,7 @@ class StackerLayerInterface(metaclass = ABCMeta): """The list operating systems/first-level plugin hierarchy that should exclude this stacker""" @classmethod - def stack(self, + def stack(cls, context: interfaces.context.ContextInterface, layer_name: str, progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index dca215c85..73f59bdbd 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -31,6 +31,7 @@ try: # Import so that the handler is found by the framework.class_subclasses callc import smb.SMBHandler # lgtm [py/unused-import] except ImportError: + # If we fail to import this, it means that SMB handling won't be available pass vollog = logging.getLogger(__name__) diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index ae4a7d55e..61b13eb88 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -154,7 +154,8 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): vmss_success = False with contextlib.suppress(IOError): - _ = resources.ResourceAccessor().open(vmss).read(10) + with resources.ResourceAccessor().open(vmss) as fp: + _ = fp.read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmss_success = True diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 62e6de553..2b026ccd1 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -9,7 +9,7 @@ import struct from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload from volatility3.framework import constants, interfaces -from volatility3.framework.objects import templates, utility +from volatility3.framework.objects import templates vollog = logging.getLogger(__name__) @@ -611,12 +611,10 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): @overload - def __getitem__(self, i: int) -> interfaces.objects.Template: - ... + def __getitem__(self, i: int) -> interfaces.objects.Template: ... @overload - def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: - ... + def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: ... def __getitem__(self, i): """Returns the i-th item from the array.""" @@ -749,10 +747,8 @@ class AggregateType(interfaces.objects.ObjectInterface): if isinstance(cls, agg_type): agg_name = agg_type.__name__ - assert isinstance(members, collections.abc.Mapping) - f"{agg_name} members parameter must be a mapping: {type(members)}" - assert all([(isinstance(member, tuple) and len(member) == 2) for member in members.values()]) - f"{agg_name} members must be a tuple of relative_offsets and templates" + assert isinstance(members, collections.abc.Mapping), f"{agg_name} members parameter must be a mapping: {type(members)}" + assert all([(isinstance(member, tuple) and len(member) == 2) for member in members.values()]), f"{agg_name} members must be a tuple of relative_offsets and templates" def member(self, attr: str = 'member') -> object: """Specifically named method for retrieving members.""" diff --git a/volatility3/framework/plugins/linux/check_modules.py b/volatility3/framework/plugins/linux/check_modules.py index 6af8dec96..2c478cebf 100644 --- a/volatility3/framework/plugins/linux/check_modules.py +++ b/volatility3/framework/plugins/linux/check_modules.py @@ -29,7 +29,7 @@ class Check_modules(plugins.PluginInterface): ] @classmethod - def get_kset_modules(self, context: interfaces.context.ContextInterface, vmlinux_name: str): + def get_kset_modules(cls, context: interfaces.context.ContextInterface, vmlinux_name: str): vmlinux = context.modules[vmlinux_name] diff --git a/volatility3/framework/plugins/mac/lsmod.py b/volatility3/framework/plugins/mac/lsmod.py index 095fbc663..345267fea 100644 --- a/volatility3/framework/plugins/mac/lsmod.py +++ b/volatility3/framework/plugins/mac/lsmod.py @@ -46,14 +46,14 @@ class Lsmod(plugins.PluginInterface): try: kmod = kmod_ptr.dereference().cast("kmod_info") except exceptions.InvalidAddressException: - return [] + return # Generation finished yield kmod try: kmod = kmod.next except exceptions.InvalidAddressException: - return [] + return # Generation finished seen: Set = set() @@ -74,6 +74,7 @@ class Lsmod(plugins.PluginInterface): kmod = kmod.next except exceptions.InvalidAddressException: return + return # Generation finished def _generator(self): for module in self.list_modules(self.context, self.config['kernel']): diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index f77c6257b..59ea656f3 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -83,6 +83,13 @@ class Cachedump(interfaces.plugins.PluginInterface): return (username, domain, domain_name, hashh) def _generator(self, syshive, sechive): + if not syshive or not sechive: + if syshive is None: + vollog.warning('Unable to locate SYSTEM hive') + if sechive is None: + vollog.warning('Unable to locate SECURITY hive') + return + bootkey = hashdump.Hashdump.get_bootkey(syshive) if not bootkey: vollog.warning('Unable to find bootkey') @@ -142,12 +149,5 @@ class Cachedump(interfaces.plugins.PluginInterface): if hive.get_name().split('\\')[-1].upper() == 'SECURITY': sechive = hive - if syshive is None or sechive is None: - if syshive is None: - vollog.warning('Unable to locate SYSTEM hive') - if sechive is None: - vollog.warning('Unable to locate SECURITY hive') - return - return renderers.TreeGrid([("Username", str), ("Domain", str), ("Domain name", str), ('Hash', bytes)], self._generator(syshive, sechive)) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index ab11d30d6..bdff88075 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -323,7 +323,7 @@ class Handles(interfaces.plugins.PluginInterface): obj_name = item.file_name_with_device() elif obj_type == "Process": item = entry.Body.cast("_EPROCESS") - obj_name = f"{utility.array_to_string(proc.ImageFileName)} Pid {item.UniqueProcessId}" + obj_name = f"{utility.array_to_string(item.ImageFileName)} Pid {item.UniqueProcessId}" elif obj_type == "Thread": item = entry.Body.cast("_ETHREAD") obj_name = f"Tid {item.Cid.UniqueThread} Pid {item.Cid.UniqueProcess}" diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 93ac3af93..651ca7696 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -42,7 +42,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ] @classmethod - def _decode_pointer(self, value): + def _decode_pointer(cls, value): """Copied from `windows.handles`. Windows encodes pointers to objects and decodes them on the fly @@ -427,6 +427,8 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self.config_path) tcpip_module = self.get_tcpip_module(self.context, kernel.layer_name, kernel.symbol_table_name) + if not tcpip_module: + vollog.error("Unable to locate symbols for the memory image's tcpip module") try: tcpip_symbol_table = pdbutil.PDBUtility.symbol_table_from_pdb( diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index cadccc5f1..6023d5f04 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -62,7 +62,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ file_handle = None + proc_id = 'Invalid process object' try: + proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() peb = context.object(kernel_table_name + constants.BANG + "_PEB", layer_name = proc_layer_name, @@ -76,7 +78,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_handle.seek(offset) file_handle.write(data) except Exception as excp: - vollog.debug(f"Unable to dump PE with pid {proc.UniqueProcessId}: {excp}") + vollog.debug(f"Unable to dump PE with pid {proc_id}: {excp}") return file_handle diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 335624672..00f96ff63 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Iterable, Callable, Tuple +from typing import Iterable, Callable, Optional, Tuple from volatility3.framework import renderers, interfaces, layers, exceptions from volatility3.framework.configuration import requirements @@ -78,7 +78,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): layer_name: str, symbol_table: str, proc: interfaces.objects.ObjectInterface) -> \ - Iterable[interfaces.objects.ObjectInterface]: + Optional[interfaces.objects.ObjectInterface]: """ Returns a virtual process from a physical addressed one Args: @@ -124,6 +124,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if virtual_process and \ proc.vol.offset == ph_offset: return virtual_process + return None @classmethod def get_osversion(cls, context: interfaces.context.ContextInterface, layer_name: str, diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 30b5db695..f31b7832e 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -33,7 +33,11 @@ class UserAssist(interfaces.plugins.PluginInterface): self._reg_table_name = None self._win7 = None # taken from http://msdn.microsoft.com/en-us/library/dd378457%28v=vs.85%29.aspx - self._folder_guids = json.load(open(os.path.join(os.path.dirname(__file__), "userassist.json"), "rb")) + try: + with open(os.path.join(os.path.dirname(__file__), "userassist.json"), "rb") as fp: + self._folder_guids = json.load(fp) + except IOError: + vollog.error("Usersassist data file not found") @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 1fceb1bcc..3fde0978d 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -102,10 +102,9 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): # Check there are no obvious errors # Open the file and test the version self._versions = dict([(x.version, x) for x in class_subclasses(ISFormatTable)]) - fp = resources.ResourceAccessor().open(isf_url) - reader = codecs.getreader("utf-8") - json_object = json.load(reader(fp)) # type: ignore - fp.close() + with resources.ResourceAccessor().open(isf_url) as fp: + reader = codecs.getreader("utf-8") + json_object = json.load(reader(fp)) # type: ignore # Validation is expensive, but we cache to store the hashes of successfully validated json objects if validate and not schemas.validate(json_object): diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 73f31115a..ce002b905 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -128,6 +128,7 @@ class module(generic.GenericIntelProcess): sym_addr = sym.st_value if wanted_sym_name == sym_name: return sym_addr + return None @property def section_symtab(self): diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index a66bfb534..45dc1db70 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -28,8 +28,11 @@ class proc(generic.GenericIntelProcess): if not isinstance(parent_layer, interfaces.layers.TranslationLayerInterface): raise TypeError("Parent layer is not a translation layer, unable to construct process layer") - with contextlib.suppress(exceptions.InvalidAddressException): + try: dtb = self.get_task().map.pmap.pm_cr3 + except exceptions.InvalidAddressException: + # Bail out because we couldn't find the DTB + return None if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.p_pid}" @@ -38,10 +41,8 @@ class proc(generic.GenericIntelProcess): return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - with contextlib.suppress(exceptions.InvalidAddressException): - task = self.get_task() - try: + task = self.get_task() current_map = task.map.hdr.links.next except exceptions.InvalidAddressException: return diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 350bb0a53..f42ac78fe 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Optional, Tuple +from typing import Optional, Tuple, Union from volatility3.framework import interfaces @@ -11,7 +11,7 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): """Class to handle the metadata from a Windows symbol table.""" @property - def pe_version(self) -> Optional[Tuple]: + def pe_version(self) -> Optional[Union[Tuple[int, int, int], Tuple[int, int, int, int]]]: build = self._json_data.get('pe', {}).get('build', None) revision = self._json_data.get('pe', {}).get('revision', None) minor = self._json_data.get('pe', {}).get('minor', None) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index a86f5b3cb..87e8e0f45 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -719,7 +719,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): envars = context.layers[process_space].read(block, block_size).decode("utf-16-le", errors = 'replace').split('\x00')[:-1] except exceptions.InvalidAddressException: - return renderers.UnreadableValue() + return # Generation finished for envar in envars: split_index = envar.find('=') @@ -729,6 +729,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): # Exclude parse problem with some types of env if env and var: yield env, var + return # Generation finished class LIST_ENTRY(objects.StructType, collections.abc.Iterable): diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index 15b5c733a..4809aafcf 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -926,14 +926,16 @@ class PdbRetreiver: try: vollog.debug(f"Attempting to retrieve {url + suffix}") # We have to cache this because the file is opened by a layer and we can't control whether that caches - result = resources.ResourceAccessor(progress_callback).open(url + suffix) + with resources.ResourceAccessor(progress_callback).open(url + suffix) as fp: + fp.read(10) + result = True except (error.HTTPError, error.URLError) as excp: vollog.debug(f"Failed with {excp}") if result: break if progress_callback is not None: progress_callback(100, f"Downloading {url + suffix}") - if result is None: + if not result: return None return url + suffix diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 2fa9bd591..25911e376 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -254,7 +254,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): pdb_names: List[bytes], progress_callback: constants.ProgressCallback = None, start: Optional[int] = None, - end: Optional[int] = None) -> Generator[Dict[str, Optional[Union[bytes, str, int]]], None, None]: + end: Optional[int] = None, + maximum_invalid_count: int = 100) -> Generator[Dict[str, Optional[Union[bytes, str, int]]], None, None]: """Scans through `layer_name` at `ctx` looking for RSDS headers that indicate one of four common pdb kernel names (as listed in `self.pdb_names`) and returns the tuple (GUID, age, pdb_name, @@ -264,6 +265,14 @@ class PDBUtility(interfaces.configuration.VersionableInterface): The UI should always provide the user an opportunity to specify the appropriate types and PDB values themselves + Args: + layer_name: The layer name to scan + page_size: Size of page constant + pdb_names: List of pdb names to scan + progress_callback: Means of providing the user with feedback during long processes + start: Start address to start scanning from the pdb_names + end: Minimum address to scan the pdb_names + maximum_invalid_count: Amount of pages that can be invalid during scanning before aborting signature search """ min_pfn = 0 @@ -279,11 +288,16 @@ class PDBUtility(interfaces.configuration.VersionableInterface): sections = [(start, end - start)]): mz_offset = None sig_pfn = signature_offset // page_size + current_invalid_counter = 0 for i in range(sig_pfn, min_pfn, -1): - if not ctx.layers[layer_name].is_valid(i * page_size, 2): + if current_invalid_counter > maximum_invalid_count: break + if not ctx.layers[layer_name].is_valid(i * page_size, 2): + current_invalid_counter += 1 + continue + data = ctx.layers[layer_name].read(i * page_size, 2) if data == b'MZ': mz_offset = i * page_size @@ -345,7 +359,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") - module_name = guid["pdb_name"].strip('.pdb') + module_name = guid["pdb_name"].replace('.pdb', '') symbol_table_name = cls.load_windows_symbol_table(context, guid["GUID"], diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 6029c0a5c..94962bc81 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -3,7 +3,7 @@ import logging import struct from typing import List, Iterator, Optional, Tuple, Type -from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes from volatility3.plugins.windows.registry import hivelist, printkey @@ -46,14 +46,13 @@ class Certificates(interfaces.plugins.PluginInterface): open_method: Type[interfaces.plugins.FileHandlerInterface]) -> \ Optional[interfaces.plugins.FileHandlerInterface]: try: - if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): - dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) - file_handle = open_method(dump_name) - file_handle.write(certificate_data) - return file_handle + dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) + file_handle = open_method(dump_name) + file_handle.write(certificate_data) + return file_handle except exceptions.InvalidAddressException: - vollog.debug(f"Unable to certificate file at {hive_offset:#x}") - return None + vollog.debug(f"Unable to dump certificate file at {hive_offset:#x}") + return None def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: @@ -79,9 +78,10 @@ class Certificates(interfaces.plugins.PluginInterface): key_hash = key_path[key_path.rindex("\\") + 1:] if self.config['dump']: - file_handle = self.dump_certificate(certificate_data, hive.hive_offset, reg_section, key_hash, self.open) - if file_handle: - file_handle.close() + if not isinstance(certificate_data, interfaces.renderers.BaseAbsentValue): + file_handle = self.dump_certificate(certificate_data, hive.hive_offset, reg_section, key_hash, self.open) + if file_handle: + file_handle.close() yield (0, (top_key, reg_section, key_hash, name))